From 9cfcc643758c1ff6483b98125867e9ed2d817f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 31 Oct 2026 12:03:40 +0100 Subject: [PATCH 01/29] fix(ios): book every surface on the lit panel, not the first window On an iPhone Duo home screen the application's first window is SpringBoard's status-bar window: a 223x317 box on a panel that may be dark. Taps, the snapshot viewport, the capture display and the synthesized record all took that window as "the screen", so every one of them answered for a screen nobody was looking at. One rule now names the window a command books its surface against (RunnerPanelSurface): the window the host reports as the lit panel shows. The host reads panel power it already has and sends the lit panel as a typed fact on the requests that resolve a screen; a request that names no panel keeps the historic first-window walk, so single-panel iPhone, iPad and tvOS behavior is byte-for-byte the request and the read it always were. Geometry never guesses a panel: the dark panel's windows have perfectly good frames. A regular capture additionally applies the panel fact to its tree (RunnerPanelSurfaceTree): the root carries the surface box the host reads as the payload's viewport, surfaces off the panel are dropped with their subtrees, and when the application's subtree carries no window for the lit panel the resolved window is read once and grafted, which is what makes the home screen's icons addressable. Raw retains reported frames. A device whose every window sits on the dark panel refuses with APP_SCREEN_OFF_LIT_PANEL rather than capturing black. Device evidence (iPhone Duo 9542F4F5, iPhone 17 Pro BC54AC11): - folded: lit=1(466x678) selection=litPanelWindow frame=(0,0 466x678); home-screen snapshot lists icons; icon long-press opens the icon menu; Safari tap opens Safari; SYNTHESIZED_DISPATCH reference=(0,0 466x678), SYNTHESIZED_RECORD displayID=1; screenshot 466x678. - unfolded: lit=3(669x951) frame=(0,0 951x669), root republished from the dark panel's 466x678, 4 off-panel surfaces dropped, tree grafted to 67 nodes; snapshot lists icons; long-press opens the context menu; Settings tap opens Settings; reference=(0,0 951x669), record displayID=3; screenshot 951x669. - iPhone 17 Pro: tap and snapshot unchanged, no panel fact on the wire, no panel markers in runner.log, reference=(0,0 402x874). --- .../RunnerAppScreenCapture.swift | 50 +- .../RunnerPanelSurface.h | 96 ++++ .../RunnerPanelSurface.m | 96 ++++ .../RunnerPanelSurfaceTree.swift | 158 ++++++ .../RunnerRequestPanel.swift | 51 ++ .../RunnerTests+CommandDispatch.swift | 9 +- .../RunnerTests+CommandExecution.swift | 5 +- .../RunnerTests+Interaction.swift | 47 +- .../RunnerTests+Keyboard.swift | 14 +- .../RunnerTests+Lifecycle.swift | 12 +- .../RunnerTests+Models.swift | 31 ++ .../RunnerTests+Navigation.swift | 2 +- .../RunnerTests+Snapshot.swift | 217 ++++++-- .../RunnerTests+SnapshotAcquisition.swift | 5 +- .../RunnerTests+SnapshotCapturePlan.swift | 28 +- .../RunnerTests.swift | 5 + .../RunnerXCTestEventBridge.h | 75 ++- .../RunnerXCTestEventBridge.m | 238 +++++++-- .../RunnerPanelSurfaceTreeTests.swift | 237 +++++++++ .../UnitTests/RunnerRequestPanelTests.swift | 55 ++ .../RunnerTests+AppScreenCaptureTests.swift | 35 +- .../UnitTests/RunnerTests+ModelsTests.swift | 6 + .../UnitTests/RunnerTests+SnapshotTests.swift | 4 +- .../RunnerTests+XCTestEventBridgeTests.swift | 490 +++++++++++++++--- .../SnapshotModels.swift | 6 +- contracts/fixtures/dispatch-disclosure.json | 6 + contracts/fixtures/runner-requests.json | 5 + docs/adr/0025-foldable-apple-panels.md | 31 +- docs/agents/device-foldable.md | 43 +- .../index-dispatch-disclosure.test.ts | 1 + .../runner-error-classification.test.ts | 1 + packages/apple-runner/src/index.ts | 1 + packages/apple-runner/src/runner-contract.ts | 17 +- packages/contracts/src/interactor-types.ts | 18 + .../src/__tests__/display-fixtures.ts | 47 ++ ...nteractor-foldable-runner-requests.test.ts | 86 +++ .../src/core/display-inventory.ts | 31 ++ .../platform-apple/src/core/runner-client.ts | 108 +++- .../src/foldable/orientation.test.ts | 59 +++ .../src/foldable/orientation.ts | 43 +- .../src/foldable/runner-panel.test.ts | 125 +++++ .../src/foldable/runner-panel.ts | 44 ++ packages/platform-apple/src/interactor.ts | 5 +- 43 files changed, 2428 insertions(+), 215 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift create mode 100644 packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts create mode 100644 packages/platform-apple/src/foldable/runner-panel.test.ts create mode 100644 packages/platform-apple/src/foldable/runner-panel.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index c71fde7267..bee77183b8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -21,11 +21,16 @@ enum RunnerAppScreenCaptureFailure: String, Error { case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED" case unresolvedScreen = "APP_SCREEN_UNRESOLVED" case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE" + /// The request named a lit panel and every window it found is on the other one. Guessing the bigger + /// window would capture a screen nobody is looking at, so the capture refuses with this instead. + case offLitPanel = "APP_SCREEN_OFF_LIT_PANEL" init(_ failure: RunnerApplicationScreenFailure) { switch failure { case .unresolvedWindow: self = .unresolvedWindow + case .offLitPanel: + self = .offLitPanel default: self = .unresolvedScreen } @@ -39,6 +44,8 @@ enum RunnerAppScreenCaptureFailure: String, Error { return "The resolved app window did not report the display it is shown on." case .unrenderableImage: return "The resolved display handed back an image that could not be drawn upright." + case .offLitPanel: + return "Every window the app and the system surface reported is on the panel the host reports as dark." } } @@ -50,19 +57,26 @@ enum RunnerAppScreenCaptureFailure: String, Error { return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved." case .unrenderableImage: return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question." + case .offLitPanel: + return "Read the device's fold pose and panel table: the host named a lit panel no window occupies. runner.log names the panel and the window frames it refused." } } } extension RunnerTests { - /// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved - /// window asks the system surface, and nothing else does. + /// The target rule, kept apart from the two queries so the rule itself is testable: a process that + /// names no window, or none on the lit panel, asks the system surface, and nothing else does. + /// + /// A panel refusal moves on for the same reason an unresolved window does: the home screen is + /// SpringBoard's window, so an app with nothing on the lit panel says nothing about where the lit + /// panel's content is. The system surface is asked, and if neither names a surface on it, the + /// panel refusal is the answer the host needs. func selectObservedScreenCapture( resolving: () -> Result, fallingBack: () -> Result ) -> Result { let outcome = resolving() - if case .failure(.unresolvedWindow) = outcome { + if case .failure(let failure) = outcome, failure == .unresolvedWindow || failure == .offLitPanel { return fallingBack() } return outcome @@ -82,14 +96,20 @@ extension RunnerTests { /// /// Never activates `app`: an observation must not change which app is foregrounded just to get a /// screen, so the caller passes the app context it already intends to observe. + /// + /// The panel arrives as an argument rather than being read here: an operation books its whole screen + /// on the panel its request named, and a snapshot of that fact cannot be replaced under it by the + /// command that follows. Callers that live for one command read `requestSurfacePanel` on the command + /// queue and pass what they got, so the panel survives any hop to main. func captureResolvedAppScreen( - app: XCUIApplication + app: XCUIApplication, + litPanel: RunnerLitPanel ) -> Result { var outcome: Result = .failure( .unresolvedScreen ) let capture = { - outcome = Self.resolveCapturedAppScreen(app: app) + outcome = Self.resolveCapturedAppScreen(app: app, litPanel: litPanel) } if Thread.isMainThread { capture() @@ -106,25 +126,29 @@ extension RunnerTests { /// keeps that capture honest — it names a display a window actually occupies instead of reaching /// for `XCUIScreen.main`, which on a foldable is one fixed panel whether or not it is dark (#2728). /// - /// Only an unresolved *window* moves on to the system surface. A window that resolved and then - /// refused to name its display is the failure the host has to see, and no second question about a - /// different process should bury it. - func captureObservedScreen(app: XCUIApplication) -> Result< + /// Only an unresolved window or a panel refusal moves on to the system surface. A window that + /// resolved on the right panel and then refused to name its display is the failure the host has to + /// see, and no second question about a different process should bury it. + func captureObservedScreen( + app: XCUIApplication, + litPanel: RunnerLitPanel + ) -> Result< CapturedAppScreen, RunnerAppScreenCaptureFailure > { selectObservedScreenCapture( - resolving: { captureResolvedAppScreen(app: app) }, - fallingBack: { captureResolvedAppScreen(app: springboard) } + resolving: { captureResolvedAppScreen(app: app, litPanel: litPanel) }, + fallingBack: { captureResolvedAppScreen(app: springboard, litPanel: litPanel) } ) } private static func resolveCapturedAppScreen( - app: XCUIApplication + app: XCUIApplication, + litPanel: RunnerLitPanel ) -> Result { var screen: AnyObject? var displayID: UInt = 0 var resolution: RunnerApplicationScreenFailure = .unresolvedScreen - guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution), + guard RunnerResolveApplicationScreen(app, litPanel, &screen, &displayID, &resolution), let captureScreen = screen as? XCUIScreen else { return .failure(RunnerAppScreenCaptureFailure(resolution)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h new file mode 100644 index 0000000000..770dd1d5c1 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h @@ -0,0 +1,96 @@ +// RunnerPanelSurface.h — the one rule that names the window a command books its surface against. +// +// Four runner paths need "the window the screen is": the interaction anchor and reference frame, the +// snapshot viewport, the display a capture names, and the display a synthesized event record routes +// to. Each used to take the application's first window. On a foldable home screen that is +// SpringBoard's status-bar window — a 223x317 box on a panel that may be dark — so taps, the +// snapshot viewport, and captures all landed on a screen nobody was looking at. They resolve through +// this rule instead. +// +// Only this file declares which window wins. The reads that feed it stay with their owners: the +// XCTest bridge reads an application's windows and their display identity, and hands the facts here, +// so a device the rule cannot settle is refused the same way by every caller. + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +// The panel the host reports as lit, in the units the panel table reports it: which display is +// powered, and that panel's own native box in points. The runner cannot read a hinge, and geometry +// cannot say which of two panels is powered, so only the host can name this (ADR 0025): it reads the +// panel table for the request and sends the answer with it. A request that carried no panel leaves +// `displayID` 0. +// +// The size is the panel's *native* box, not the box an app shows on it. The system presents the inner +// panel turned, so its interaction space swaps the side lengths (ADR 0025), which is why the rule +// compares a box either way up. +typedef struct { + NSUInteger displayID; + CGFloat widthPt; + CGFloat heightPt; +} RunnerLitPanel; + +// Whether a request named a panel at all. Only a real display identity counts: a panel with no +// display cannot be compared against a window, so it reads as no panel. +static inline BOOL RunnerLitPanelIsNamed(RunnerLitPanel panel) { + return panel.displayID != 0; +} + +// One window the runner measured: the display it is shown on, and the box it occupies in the app's +// coordinate space. +typedef struct { + NSUInteger displayID; + CGRect frame; +} RunnerPanelWindowFact; + +// What the rule made of an application's windows. Every outcome but `NoWindow` names the window it +// chose or the first one it looked at, through `selected`, so a refusal can report the frame it +// refused instead of returning a silent nil. +typedef NS_ENUM(NSUInteger, RunnerPanelSurfaceSelection) { + // A window on the lit display whose box is that panel's box, either way up: the screen to book. + RunnerPanelSurfaceSelectionLitPanelWindow, + // Windows exist on the lit display but none carries the panel's box, so the app occupies part of + // it (Split View, a stage, a sheet). The largest is the surface to book: being on the lit panel is + // the fact a first-window walk could never establish. + RunnerPanelSurfaceSelectionLitDisplayWindow, + // Every window the application reported is on a display the host reports as dark. + RunnerPanelSurfaceSelectionOffLitPanel, + // No window answered a usable frame, so there is no surface to book right now. + RunnerPanelSurfaceSelectionNoWindow, +}; + +// How many windows one walk reads. A device presenting more qualifying windows than this is not a +// surface this rule can settle, and reading past the cap would only widen the risk of reading a +// window that has already vanished (#2995). +#define AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS 8 + +// Half a point: the rounding tolerance the full-screen proof already uses. Sub-pixel layout is not a +// different panel. +#define AGENT_DEVICE_PANEL_SURFACE_TOLERANCE 0.5 + +// Whether `frame` is the lit panel's surface: anchored at the origin, and the panel's size either way +// up. A frame that is not a measurement at all never qualifies. +FOUNDATION_EXPORT BOOL RunnerPanelFrameIsLitPanelSurface( + CGRect frame, + RunnerLitPanel litPanel +); + +// Decides which of `windows` a command books against, from the facts alone: no queries, no device +// state, no clock. `litPanel` must name a display: a caller whose request carried no panel has +// nothing to compare a window against and keeps its historic first-window route instead of calling +// this. `count` is capped by the caller at `AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS`. +// +// `selected` reports the window the decision was made from and `selectedIndex` its slot in `windows`, +// so a caller keeps the window element it measured instead of reading it again. On a refusal +// `selected` is the first window it looked at, or a null frame when there were none, so the caller's +// refusal names what it saw. +FOUNDATION_EXPORT RunnerPanelSurfaceSelection RunnerPanelSurfaceSelectWindow( + const RunnerPanelWindowFact *windows, + NSUInteger count, + RunnerLitPanel litPanel, + RunnerPanelWindowFact *selected, + NSUInteger *selectedIndex +); + +NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m new file mode 100644 index 0000000000..f8d5560718 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m @@ -0,0 +1,96 @@ +#import "RunnerPanelSurface.h" +#import "ForegroundApplicationGeometry.h" +#import + +BOOL RunnerPanelFrameIsLitPanelSurface(CGRect frame, RunnerLitPanel litPanel) { + if (!AgentDeviceForegroundFrameIsMeasurable(frame)) { + return NO; + } + if (fabs(frame.origin.x) > AGENT_DEVICE_PANEL_SURFACE_TOLERANCE || + fabs(frame.origin.y) > AGENT_DEVICE_PANEL_SURFACE_TOLERANCE) { + return NO; + } + if (!isfinite(litPanel.widthPt) || !isfinite(litPanel.heightPt)) { + return NO; + } + BOOL upright = + fabs(frame.size.width - litPanel.widthPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE && + fabs(frame.size.height - litPanel.heightPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE; + // A panel the system presents turned reports its window in that panel's own native space, so the + // iPhone Duo inner panel's 951x669 interaction box is the transpose of the 669x951 the panel table + // names for it (ADR 0025). + BOOL turned = + fabs(frame.size.width - litPanel.heightPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE && + fabs(frame.size.height - litPanel.widthPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE; + return upright || turned; +} + +static CGFloat RunnerPanelWindowFrameArea(CGRect frame) { + CGFloat width = frame.size.width > 1 ? frame.size.width : 1; + CGFloat height = frame.size.height > 1 ? frame.size.height : 1; + return width * height; +} + +RunnerPanelSurfaceSelection RunnerPanelSurfaceSelectWindow( + const RunnerPanelWindowFact *windows, + NSUInteger count, + RunnerLitPanel litPanel, + RunnerPanelWindowFact *selected, + NSUInteger *selectedIndex +) { + if (selected != NULL) { + *selected = (RunnerPanelWindowFact){.displayID = 0, .frame = CGRectNull}; + } + if (selectedIndex != NULL) { + *selectedIndex = NSUIntegerMax; + } + if (windows == NULL || count == 0) { + return RunnerPanelSurfaceSelectionNoWindow; + } + if (selected != NULL) { + *selected = windows[0]; + } + if (selectedIndex != NULL) { + *selectedIndex = 0; + } + + RunnerPanelWindowFact largestPlaced = windows[0]; + NSUInteger largestPlacedIndex = 0; + BOOL sawPlaced = NO; + CGFloat largestPlacedArea = 0; + for (NSUInteger index = 0; index < count; index += 1) { + RunnerPanelWindowFact candidate = windows[index]; + if (candidate.displayID != litPanel.displayID) { + continue; + } + if (RunnerPanelFrameIsLitPanelSurface(candidate.frame, litPanel)) { + if (selected != NULL) { + *selected = candidate; + } + if (selectedIndex != NULL) { + *selectedIndex = index; + } + return RunnerPanelSurfaceSelectionLitPanelWindow; + } + CGFloat area = RunnerPanelWindowFrameArea(candidate.frame); + if (!sawPlaced || area > largestPlacedArea) { + sawPlaced = YES; + largestPlacedArea = area; + largestPlaced = candidate; + largestPlacedIndex = index; + } + } + if (sawPlaced) { + if (selected != NULL) { + *selected = largestPlaced; + } + if (selectedIndex != NULL) { + *selectedIndex = largestPlacedIndex; + } + return RunnerPanelSurfaceSelectionLitDisplayWindow; + } + // The lit display named no window, so every window the app has is on the dark panel. Taking the + // biggest of them by geometry would book a gesture or a capture on a screen nobody is looking at, + // so this refuses and leaves the caller to report the panel it looked at. + return RunnerPanelSurfaceSelectionOffLitPanel; +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift new file mode 100644 index 0000000000..7e048cd987 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift @@ -0,0 +1,158 @@ +import Foundation +import AgentDeviceSnapshotPresentation + +/// The tree a multi-panel device publishes. +/// +/// A capture walks the application element, and on a foldable that hands back whichever window the +/// platform serializes. On an iPhone Duo home screen, the application's subtree reported SpringBoard's +/// outer-panel window and the menu that had already closed on it, while the icons on the lit panel were +/// absent. Booking the viewport against the lit panel, as the surface rule does, would then make every +/// published node a claim about a screen nobody is looking at. +/// +/// Three answers come from the panel the request named, and this type is the only place that applies +/// them (ADR 0025): +/// +/// 1. The root node carries the panel's box, because the host reads that node as the box the payload is +/// measured in. On a multi-panel device the application element's own frame names a panel, not the +/// panel in use. +/// 2. Surfaces that cannot be seen on that panel are dropped, because their refs would name points on a +/// dark screen and the fold can only clip them into a shape nobody can act on. +/// 3. When the tree carries no window for the panel at all, the caller reads the resolved surface window +/// once and grafts it, which is what makes the lit panel's icons addressable. +/// +/// A request that names no panel reaches none of this: on a single-panel device the application element's +/// frame is the surface box and every window is on the one panel, so the historic tree is published as it +/// was read. +enum RunnerPanelSurfaceTree { + /// One point of drift between two reported window boxes is layout, not a different panel. This is + /// looser than the surface rule's half point (`AGENT_DEVICE_PANEL_SURFACE_TOLERANCE`) on purpose: + /// that rule compares an app's box against the panel fact the host divided out of pixels, while this + /// type compares two boxes the runtime reported itself. + static let tolerance: CGFloat = 1 + + /// The wire types a whole surface arrives as, in the payload's own vocabulary. `Other` is + /// deliberately absent: XCTest reports it for most containers, and treating it as a surface would + /// drop content the fold can clip. `Sheet` and `Drawer` are what XCTest would name them; today its + /// tree reports them as `Element()`, so in practice only `Window` is dropped, which is the + /// conservative direction. The graft asks the platform's element types instead + /// (`RunnerTests.surfaceElementTypes`), because it reads live elements rather than a payload. + static let surfaceTypeNames: Set = ["Application", "Window", "Sheet", "Drawer"] + + /// Whether a reported box is that panel's box, either way up: the system presents the inner panel + /// turned, so its window arrives in the panel's own native space (ADR 0025). + static func matchesSurfaceBox(_ frame: CGRect, _ surface: CGRect) -> Bool { + guard isMeasurable(frame), isMeasurable(surface) else { return false } + guard abs(frame.minX) <= tolerance, abs(frame.minY) <= tolerance else { return false } + return (near(frame.width, surface.width) && near(frame.height, surface.height)) + || (near(frame.width, surface.height) && near(frame.height, surface.width)) + } + + /// Whether a reported box can be seen on the named panel. + static func liesOnSurface(_ frame: CGRect, _ surface: CGRect) -> Bool { + guard isMeasurable(frame), isMeasurable(surface) else { return false } + return frame.minX >= surface.minX - tolerance + && frame.minY >= surface.minY - tolerance + && frame.maxX <= surface.maxX + tolerance + && frame.maxY <= surface.maxY + tolerance + } + + /// Whether one already-read layer of a capture carries a window on the named panel. That is the + /// capture's own evidence that it already shows the lit panel, so no second read is needed. + static func carriesSurfaceWindow( + _ layer: [Layer], + surface: CGRect, + isSurface: (Layer) -> Bool, + frame: (Layer) -> CGRect + ) -> Bool { + layer.contains { isSurface($0) && matchesSurfaceBox(frame($0), surface) } + } + + /// What narrowing a payload to one panel changed, so the caller can report it and a test can read it. + struct Published { + let nodes: [RawAXNode] + /// Whether the root moved onto the panel's box. + let republishedRoot: Bool + /// How many surfaces were removed with their subtrees. + let droppedSurfaces: Int + } + + /// The tree as published on the named panel: the root carrying the panel's box and the off-panel + /// surfaces removed with their subtrees, re-indexed for the payload. + /// + /// `nil` means the tree already said exactly this, which is the common case and costs no copy. Only a + /// payload rooted at the application is rewritten: that root is the node whose children are the app's + /// whole window list, so it is the one shape where a child naming a surface names a screen. A payload + /// already rooted at a window was read from one surface and cannot hold another one's content. + /// + /// Only a root's own children are dropped: a surface is a whole screen, so its subtree stands or falls + /// with it, while a node deeper in the tree that reaches past the panel is content the fold clips. + static func publishing(nodes: [RawAXNode], surface: CGRect) -> Published? { + guard let root = nodes.first, root.parentIndex == nil, root.type == "Application", + nodes.dropFirst().contains(where: { $0.parentIndex == nil }) == false + else { + return nil + } + let offPanel = nodes.filter { node in + node.parentIndex == root.index + && node.index != root.index + && surfaceTypeNames.contains(node.type) + && !liesOnSurface(node.rect.cgRect, surface) + } + let panelBox = SnapshotRect(surface) + let republishRoot = root.rect != panelBox + guard republishRoot || !offPanel.isEmpty else { return nil } + + let dropped = subtreeIndexes(of: offPanel.map(\.index), in: nodes) + var kept: [RawAXNode] = [] + kept.reserveCapacity(nodes.count - dropped.count) + var reindex: [Int: Int] = [:] + reindex.reserveCapacity(nodes.count - dropped.count) + for node in nodes where !dropped.contains(node.index) { + reindex[node.index] = kept.count + kept.append(node) + } + let published = kept.map { node in + let republished = node.parentIndex == nil + ? node.replacing(rect: panelBox, hittable: node.hittable) + : node + return republished.reindexed( + index: reindex[node.index] ?? node.index, + depth: node.depth, + parentIndex: node.parentIndex.flatMap { reindex[$0] } + ) + } + return Published( + nodes: published, + republishedRoot: republishRoot, + droppedSurfaces: offPanel.count + ) + } + + /// Depth counts ancestors and only whole sibling subtrees are removed, so a kept node keeps its depth. + /// Indexes shift because the payload's wire array is positional. + private static func subtreeIndexes(of roots: [Int], in nodes: [RawAXNode]) -> Set { + var childrenByParent: [Int: [Int]] = [:] + for node in nodes { + if let parent = node.parentIndex { + childrenByParent[parent, default: []].append(node.index) + } + } + var dropped = Set(roots) + var frontier = roots + while let index = frontier.popLast() { + for child in childrenByParent[index] ?? [] where dropped.insert(child).inserted { + frontier.append(child) + } + } + return dropped + } + + private static func isMeasurable(_ frame: CGRect) -> Bool { + frame.minX.isFinite && frame.minY.isFinite && frame.size.width.isFinite && frame.size.height.isFinite + && frame.size.width > 0 && frame.size.height > 0 + } + + private static func near(_ left: CGFloat, _ right: CGFloat) -> Bool { + abs(left - right) <= tolerance + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift new file mode 100644 index 0000000000..1814701a9f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -0,0 +1,51 @@ +import XCTest + +// MARK: - Request-scoped lit panel (#2728, ADR 0025) +// +// A foldable lights one panel at a time, and only the host can say which: the runner has no hinge to +// read, and geometry cannot settle it, because the dark panel's windows have perfectly good frames. +// So the host reads the panel table for the request and names the lit panel in it (`Command.litPanel`), +// and every surface resolver books its window against that fact rather than against the application's +// first window — which on a foldable home screen is SpringBoard's status-bar window. +// +// The resolvers that need the fact sit on the gesture, snapshot, capture and synthesized-dispatch +// paths. Each of them runs the command that carries the fact, and none of them has the `Command` in +// scope: the viewport read is a hop inside a capture plan, the capture answers through a result type, +// and a gesture path is handed a context rather than a request. So the runner holds the panel of the +// command in flight in one place, which is also the only place that answers the question. +// +// Commands are served one at a time on `commandExecutionQueue`, which is where the slot is written. +// Readers run on main, so the slot is lock-guarded rather than plain main-owned state, the same split +// `SnapshotProbePenaltyIdentity` makes. Main-queue work abandoned by the execution watchdog can +// therefore read the panel of the command that replaced it; the resolution marker the bridge logs +// names the panel a window was booked against, so such a resolution is visible, not silent. +// +// An operation that keeps sampling after its request is served does not read the slot per sample: it +// reads it once on the command queue and carries that panel as an argument for its whole life. A clip +// records on one panel and a settle loop compares samples of one screen; either one reading the slot +// again would be answering a question about a command it is not part of. + +extension RunnerTests { + /// Records the panel the command about to be served names as lit. A request that names none clears + /// the slot, so a command for a single-panel device can never inherit a foldable's panel. + func setRequestLitPanel(from command: Command) { + requestPanelLock.lock() + requestLitPanel = command.litPanel + requestPanelLock.unlock() + } + + /// The lit panel in the units the surface rule compares a window against. An unnamed or unusable + /// panel answers `displayID` 0, which every resolver reads as "keep the historic first-window + /// route": a panel that cannot be named cannot be compared with a window. + var requestSurfacePanel: RunnerLitPanel { + requestPanelLock.lock() + let panel = requestLitPanel + requestPanelLock.unlock() + guard let panel, panel.displayId != 0, panel.widthPt.isFinite, panel.heightPt.isFinite, + panel.widthPt > 0, panel.heightPt > 0 + else { + return RunnerLitPanel(displayID: 0, widthPt: 0, heightPt: 0) + } + return panel.surfacePanel + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index 7f77331073..88031d1076 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -7,6 +7,9 @@ extension RunnerTests { func executeAccepted(command: Command) throws -> Response { commandJournal.start(command: command) pendingTargetActivation = nil + // Before anything the command resolves: every surface resolver this command reaches reads the + // panel it named, and one that named none must not inherit the previous command's. + setRequestLitPanel(from: command) do { let response = try executeDispatched(command: command) commandJournal.finish(command: command, response: response) @@ -432,9 +435,13 @@ extension RunnerTests { outputPath: resolvedOutPath, fps: command.fps.map { Int32($0) } ) + // One panel for the whole clip, taken while this request still owns the slot: the writer is + // sized from the bootstrap frame, so a pump that asked again later could book the remaining + // frames on the panel of whatever command happened to follow `recordStart`. + let clipLitPanel = requestSurfacePanel try startRecording(recorder) { [weak self] in guard let self else { return .failure(.unresolvedScreen) } - return self.captureRunnerFrameResult(app: activeApp) + return self.captureRunnerFrameResult(app: activeApp, litPanel: clipLitPanel) } activeRecording = recorder return Response(ok: true, data: DataPayload(message: "recording started")) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 526d334a2e..b6754691fc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -487,7 +487,10 @@ extension RunnerTests { // session app, so on a fresh runner process `activeApp` is still the runner host app. The // requested bundle id is resolved here and never activated: an observation must not change // which app is foregrounded in order to see which display it is on (#2728). - switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) { + switch captureObservedScreen( + app: resolveAppWithoutActivation(command: command), + litPanel: requestSurfacePanel + ) { case .failure(let failure): return Response( ok: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 3684adfee4..109572747c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -243,17 +243,16 @@ extension RunnerTests { // transformed subtrees, so a closed drawer at negative x inflates it and // out-of-window coordinates still pass containment. Falls back to app.frame // when no window frame is readable. - /// The app window interactions and captures are booked against: the first window the app - /// reports with a non-empty frame, and no window at all when none qualifies. The viewport, - /// the interaction anchor, and the synthesized reference frame all resolve through here, so a - /// fold, a sheet, or a rotation moves every consumer to the same window in the same pass. - /// Synthesized gestures route their display ID through the returned window so the record's + /// The app window interactions and captures are booked against, and no window at all when none + /// qualifies. The viewport, the interaction anchor, and the synthesized reference frame all resolve + /// through here, so a fold, a sheet, or a rotation moves every consumer to the same window in the + /// same pass. Synthesized gestures route their display ID through the returned window so the record's /// display can never name a different window than the one the reference frame was measured on. /// - /// Stops at the first qualifying window. Reading a window that will be discarded is not free: a - /// transient candidate that disappears between its `exists` answer and its `frame` fetch makes - /// XCTest record an issue, and an issue `record(_:)` does not mute ends the runner session - /// (ADR 0004, ADR 0005, #2995). + /// Which window qualifies is `RunnerResolveSurfaceWindow`'s decision, and this only asks it: on a + /// single-panel device that is the first window with a non-empty frame, and on a foldable it is a + /// window on the panel the request named as lit, which is how a home-screen tap stops landing on + /// SpringBoard's status-bar window on the dark panel. /// /// The read that remains is not atomic: a window vanishing while it is the one being resolved still /// records, and the chosen window is read again on its way to a display ID @@ -265,17 +264,27 @@ extension RunnerTests { return (resolved.window, resolved.frame) } - /// The app's first window with a usable frame, or nil. Unlike `resolveRunnerWindow` it never falls - /// back to `app.frame`: when the windows query fails because XCTest cannot serialize the app's - /// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). - func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { - for window in app.windows.allElementsBoundByIndex where window.exists { - let frame = window.frame - if !frame.isEmpty { - return (window, frame) - } + /// The app's surface window, or nil. Unlike `resolveRunnerWindow` it never falls back to + /// `app.frame`: when the windows query fails because XCTest cannot serialize the app's tree, that + /// read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). + func resolveRunnerWindowElement( + app: XCUIApplication, + litPanel: RunnerLitPanel? = nil + ) -> (window: XCUIElement, frame: CGRect)? { + var resolvedWindow: AnyObject? + var windowFrame = CGRect.null + var observedFrame = CGRect.null + let failure = RunnerResolveSurfaceWindow( + app, + litPanel ?? requestSurfacePanel, + &resolvedWindow, + &windowFrame, + &observedFrame + ) + guard failure == .none, let window = resolvedWindow as? XCUIElement else { + return nil } - return nil + return (window, windowFrame) } func onScreenWindowFrame(app: XCUIApplication) -> CGRect { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index 05f3b5b00d..9ace71ff93 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -241,10 +241,13 @@ extension RunnerTests { requiredConsecutiveMatches: Int ) { let deadline = Date().addingTimeInterval(timeout) - var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app)] + // One panel for the whole settle loop: samples taken against two panels could never match, and a + // loop that outlives its request must not follow the command that replaced it. + let litPanel = requestSurfacePanel + var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app, litPanel: litPanel)] while Date() < deadline { sleepFor(sampleInterval) - samples.append(screenshotFingerprintForStabilityCheck(app: app)) + samples.append(screenshotFingerprintForStabilityCheck(app: app, litPanel: litPanel)) if runnerScreenshotStabilitySettled(samples, requiredConsecutiveMatches: requiredConsecutiveMatches) { logScreenshotStability(samples: samples, settled: true) return @@ -265,8 +268,11 @@ extension RunnerTests { ) } - private func screenshotFingerprintForStabilityCheck(app: XCUIApplication) -> Data? { - guard let image = captureRunnerFrame(app: app) else { return nil } + private func screenshotFingerprintForStabilityCheck( + app: XCUIApplication, + litPanel: RunnerLitPanel + ) -> Data? { + guard let image = captureRunnerFrame(app: app, litPanel: litPanel) else { return nil } return runnerPngData(for: image) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index f6e4d6d9bb..63657867c6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -51,8 +51,8 @@ extension RunnerTests { /// identical black frames would then read as a settled screen (#2728). An observation with no /// session window falls to the system surface's window, which is what the home screen is. macOS /// keeps the host display it always recorded. - func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? { - switch captureRunnerFrameResult(app: app) { + func captureRunnerFrame(app: XCUIApplication, litPanel: RunnerLitPanel) -> RunnerImage? { + switch captureRunnerFrameResult(app: app, litPanel: litPanel) { case .success(let captured): return captured.image case .failure: @@ -64,11 +64,15 @@ extension RunnerTests { /// frame — a recording's bootstrap, which sizes the whole writer from it — fails closed with a /// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal /// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728). + /// The panel is a parameter and not a read here for the same reason it is one for the capture: a + /// recording is one observation of one panel, and its pump runs long after the request that started + /// it was served. A caller that lives for one command may read `requestSurfacePanel` itself. func captureRunnerFrameResult( - app: XCUIApplication + app: XCUIApplication, + litPanel: RunnerLitPanel ) -> Result { #if os(iOS) - return captureObservedScreen(app: app) + return captureObservedScreen(app: app, litPanel: litPanel) #else var outcome: Result = .failure( .unrenderableImage diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index bd58bed816..1a9d2dee6b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -230,6 +230,33 @@ struct CoordinateTapTextInputEvidence: Codable { let textInputRects: [RunnerGestureViewport] } +/// The panel the host reports as lit, in the units the host's panel table reports it: which display is +/// powered and that panel's own native box in points. +/// +/// Only the host can name this. The runner has no hinge to read, and geometry cannot say which of two +/// panels is powered — the dark panel's window has a perfectly good frame — so the host reads the +/// panel table for the request and sends the answer with it (ADR 0025). A request that carries no +/// panel is a single-panel device's request, and keeps the runner's historic first-window route. +/// +/// `widthPt`/`heightPt` are the panel's *native* box, not the box an app shows on it: the system +/// presents the inner panel turned, so its interaction space swaps the side lengths. +/// +/// The field names are the wire spelling and the request side's own convention (`commandId`, +/// `appBundleId`), so the id keeps its lowercase `d` here even though the surface rule's own type +/// spells the runtime's `XCUIScreen.displayID` property the runtime's way. +struct RunnerLitPanelFact: Codable { + let displayId: UInt + let widthPt: Double + let heightPt: Double +} + +extension RunnerLitPanelFact { + /// The panel in the units the surface rule compares a window against. + var surfacePanel: RunnerLitPanel { + RunnerLitPanel(displayID: displayId, widthPt: CGFloat(widthPt), heightPt: CGFloat(heightPt)) + } +} + struct Command: Codable { let command: CommandType let commandId: String? @@ -269,6 +296,10 @@ struct Command: Codable { let inlineScreenshot: Bool? let synthesized: Bool? let coordinateTapTextInputEvidence: CoordinateTapTextInputEvidence? + /// The lit panel, sent by the host only for a device whose panel table reports more than one panel. + /// Absent means "single-panel device": every surface resolver then takes its historic first-window + /// route and reads no display identity at all. + let litPanel: RunnerLitPanelFact? let steps: [SequenceStep]? } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 4bb91cab25..1687991d0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -257,7 +257,7 @@ extension RunnerTests { return Self.navigationVisualSample( from: context.resolvedWindow == nil ? captureSoleAppScreen(app: app) - : captureResolvedAppScreen(app: app), + : captureResolvedAppScreen(app: app, litPanel: requestSurfacePanel), encoding: { runnerPngData(for: $0.image) } ) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 035d676c9e..e36bf2f2ce 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -185,6 +185,7 @@ extension RunnerTests { @MainActor func recursiveTreeSnapshotAcquisition( + app: XCUIApplication, context: SnapshotTraversalContext, hint: CaptureHint, processingDeadline: Date @@ -214,22 +215,88 @@ extension RunnerTests { parentIndex: nil ) ) - if Self.canDescendAtRawDepth(0, hint: hint) { - appendCollapsedTabFallbackNodes( - to: &nodes, - containerSnapshot: context.rootSnapshot, - resolveCandidates: collapsedTabCandidatesOnce, - depth: 1, - parentIndex: 0 + var seen = Set() + try walkSnapshotChildren( + of: context.rootSnapshot, + parentIndex: 0, + depth: 0, + into: &nodes, + seen: &seen, + hint: hint, + collapsedTabStep: { [self] nodes, container, depth, parentIndex in + self.appendCollapsedTabFallbackNodes( + to: &nodes, + containerSnapshot: container, + resolveCandidates: collapsedTabCandidatesOnce, + depth: depth, + parentIndex: parentIndex + ) + } + ) + + // A foldable's application subtree can hold only another panel's window: on a Duo home screen it + // reported the outer panel's window and the lit panel's icons were absent. The window the surface + // rule resolved is then the only remaining way to read the screen the request named, and one read + // of it is what makes its content addressable (ADR 0025). Only a request that named a panel asks + // this question: a single-panel app legitimately has no window child for its own frame, and a + // second read there would be a cost the historic route never paid. + let surface = context.viewport.rect + // One panel read for the tier: the decision below and the graft it triggers must name the same + // panel the request did, wherever the graft's own hop to main runs. + let tierLitPanel = requestSurfacePanel + if let surface, tierLitPanel.displayID != 0, + !RunnerPanelSurfaceTree.carriesSurfaceWindow( + context.rootSnapshot.children, + surface: surface, + isSurface: { Self.surfaceElementTypes.contains($0.elementType) }, + frame: { $0.frame } + ) + { + outcome = graftSurfaceWindowNodes( + app: app, + surface: surface, + litPanel: tierLitPanel, + into: &nodes, + seen: seen, + hint: hint, + outcome: outcome, + processingDeadline: processingDeadline ) } - var seen = Set() - var stack: [SnapshotTraversalEntry] = [] - if Self.canDescendAtRawDepth(0, hint: hint) { - stack = context.rootSnapshot.children.map { - SnapshotTraversalEntry(snapshot: $0, depth: 1, parentIndex: 0) - } + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: outcome == .deadlineExhausted, + effectiveDepth: nil, + viewport: context.viewport + ), + outcome + ) + } + + /// The element types a whole surface arrives as, in the platform's own vocabulary. The tree reports + /// `Other` for most containers, so it is deliberately absent: a container is not a screen. + static let surfaceElementTypes: Set = [.window, .sheet, .drawer] + + /// One subtree of the walk, applying the walk's own bounds: raw traversal depth, the node cap, and + /// identity de-duplication. Shared by the application walk and by the panel-surface graft so the two + /// cannot disagree about what a payload's nodes mean (#2661). + @MainActor + private func walkSnapshotChildren( + of snapshot: XCUIElementSnapshot, + parentIndex: Int, + depth: Int, + into nodes: inout [RawAXNode], + seen: inout Set, + hint: CaptureHint, + collapsedTabStep: ((inout [RawAXNode], XCUIElementSnapshot, Int, Int) -> Void)? + ) throws { + guard Self.canDescendAtRawDepth(depth, hint: hint) else { return } + collapsedTabStep?(&nodes, snapshot, depth + 1, parentIndex) + var stack: [SnapshotTraversalEntry] = snapshot.children.map { + SnapshotTraversalEntry(snapshot: $0, depth: depth + 1, parentIndex: parentIndex) } while let entry = stack.popLast() { @@ -272,28 +339,120 @@ extension RunnerTests { throw regularSnapshotTooLargeFailure(nodeCount: nodes.count) } if Self.canDescendAtRawDepth(depth, hint: hint) { - appendCollapsedTabFallbackNodes( - to: &nodes, - containerSnapshot: snapshot, - resolveCandidates: collapsedTabCandidatesOnce, - depth: depth + 1, - parentIndex: node.index - ) + collapsedTabStep?(&nodes, snapshot, depth + 1, node.index) } } + } - return ( - SnapshotAcquisition( + /// The window the surface rule resolved, read once and grafted onto the application root as a sibling + /// of the windows the application subtree already carried. + /// + /// The read is bounded by what the tier has left: a snapshot that cannot start inside the remaining + /// slice is not started, because XCTest cannot cancel one and an abandoned read stalls main for the + /// whole plan. A graft that cannot finish leaves the tree exactly as it was walked, with the nodes and + /// dedup keys its abandoned read produced discarded, so the payload stays one the host can validate: + /// the tier that already read the screen is never failed or marked truncated by a graft on top of it. + @MainActor + private func graftSurfaceWindowNodes( + app: XCUIApplication, + surface: CGRect, + litPanel: RunnerLitPanel, + into nodes: inout [RawAXNode], + seen: Set, + hint: CaptureHint, + outcome: SnapshotTierOutcome, + processingDeadline: Date + ) -> SnapshotTierOutcome { + guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: Self.surfaceWindowReadFloor) else { + // A graft that cannot start leaves the tier's own outcome alone: the tree already walked is + // complete, and calling it `deadlineExhausted` would discard it and penalize the backend. + NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=budget_exhausted") + return outcome + } + guard let resolved = resolveRunnerWindowElement(app: app, litPanel: litPanel) else { + NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=window-unresolved") + return outcome + } + guard resolved.frame.width.isFinite, resolved.frame.height.isFinite, + RunnerPanelSurfaceTree.liesOnSurface(resolved.frame, surface) + else { + NSLog( + "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=window-off-panel frame=(%g,%g %gx%g)", + resolved.frame.origin.x, + resolved.frame.origin.y, + resolved.frame.size.width, + resolved.frame.size.height + ) + return outcome + } + // The window walk above is itself an unbounded XCTest read, so the slice is tested again here: + // what the tier has left after it, and not before, is what the snapshot may still use. + let sliceLeft = processingDeadline.timeIntervalSinceNow + guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: Self.surfaceWindowReadFloor) else { + NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=budget_exhausted") + return outcome + } + // `seen` arrives as a copy on purpose: the walk's dedup keys are kept only when its nodes are, and + // keys for nodes nobody published would hide content that arrives later by the app route. + let graftStart = nodes.count + // The grafted root is the one node this tier appends outside the walk's own cap check, so it is + // asked for here: a tree already at the cap gets no graft, and the read that could not be + // published is not paid for. + guard graftStart < Self.regularSnapshotMaxNodes else { + NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=node-cap nodes=\(graftStart)") + return outcome + } + var seenByGraft = seen + do { + guard let windowSnapshot = try captureSnapshotRootBounded( + resolved.window, + sliceSeconds: sliceLeft + ) else { + NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=read-failed") + return outcome + } + let windowNode = makeSnapshotNode( + snapshot: windowSnapshot, + evaluation: evaluateSnapshot(windowSnapshot), + depth: 1, + index: nodes.count, + parentIndex: 0 + ) + nodes.append(windowNode) + try walkSnapshotChildren( + of: windowSnapshot, + parentIndex: windowNode.index, + depth: 1, + into: &nodes, + seen: &seenByGraft, hint: hint, - nodes: nodes, - truncated: outcome == .deadlineExhausted, - effectiveDepth: nil, - viewport: context.viewport - ), - outcome - ) + collapsedTabStep: nil + ) + NSLog( + "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_GRAFTED frame=(%g,%g %gx%g) nodes=%ld", + resolved.frame.origin.x, + resolved.frame.origin.y, + resolved.frame.size.width, + resolved.frame.size.height, + nodes.count + ) + } catch { + // The tree walked before the graft was publishable as it stands, so a graft that cannot be + // finished never fails the tier: the payload goes out without it, and without the dedup keys its + // abandoned read had added, since those live only in this scope's copy of `seen`. + nodes.removeLast(nodes.count - graftStart) + let failure = error as? SnapshotCaptureFailure + NSLog( + "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=graft-failed code=\(failure?.code ?? "unknown") nodes=\(graftStart)" + ) + } + return outcome } + /// The least a second snapshot can usefully be given. Below it the read would be abandoned on its way + /// in, which costs the plan more than the tree it would have added. + static let surfaceWindowReadFloor: TimeInterval = 0.25 + // See `snapshotFast` above: the single production entry point, no unit-test overload. func snapshotRaw(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 6f31243dea..594717adf8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -76,7 +76,7 @@ extension RunnerTests { /// penalizes the tree backend for this bundle (unless the fresh-process warmup exemption /// applies), and the plan moves to the platform's independent recovery tier when one exists /// (#1105/#1122). - private func captureSnapshotRootBounded( + func captureSnapshotRootBounded( _ element: XCUIElement, sliceSeconds: TimeInterval ) throws -> XCUIElementSnapshot? { @@ -259,6 +259,9 @@ extension RunnerTests { private func snapshotAppFrame(app: XCUIApplication) -> CGRect { #if os(iOS) + // The same resolved surface a tap books against: on a foldable the first window is SpringBoard's + // status-bar window on a panel that may be dark, and a viewport that small makes the presenter + // refuse a tree whose nodes fall outside it. return onScreenWindowFrame(app: app) #else return app.frame diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index c47b77842a..4363e69969 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -360,6 +360,7 @@ extension RunnerTests { hint.isRaw ? (try self.rawTreeSnapshotAcquisition(context: context, hint: hint), .completed) : try self.recursiveTreeSnapshotAcquisition( + app: app, context: context, hint: hint, processingDeadline: processingDeadline @@ -417,12 +418,29 @@ extension RunnerTests { // The one coordinate-space pass (#2661): reported frames become the app's orientation space // before presentation reads them. A backend with unknown orientation (the windowless sweep) // turns nothing. - let normalizedAcquisition = acquisition.replacingNodes( - SnapshotGeometrySpace.normalized( - nodes: acquisition.nodes, - viewport: acquisition.viewport - ) + let normalizedNodes = SnapshotGeometrySpace.normalized( + nodes: acquisition.nodes, + viewport: acquisition.viewport ) + // On a multi-panel device the app element's frame names a panel, not the panel in use, and the host + // reads that root node as the box the whole payload is measured in. Publishing the resolved surface + // box, and dropping what cannot be seen on it, is what the request's panel fact says about this tree. + // Raw is exempt by its own contract: it retains reported frames, and publishes no viewport box the + // host would have to agree with, so a caller asking for the platform's report keeps the whole report. + var publishedNodes = normalizedNodes + if requestSurfacePanel.displayID != 0, !hint.isRaw, let surface = acquisition.viewport.rect, + let published = RunnerPanelSurfaceTree.publishing(nodes: publishedNodes, surface: surface) + { + NSLog( + "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE republishedRoot=%@ droppedSurfaces=%ld nodes=%ld->%ld", + published.republishedRoot ? "yes" : "no", + published.droppedSurfaces, + normalizedNodes.count, + published.nodes.count + ) + publishedNodes = published.nodes + } + let normalizedAcquisition = acquisition.replacingNodes(publishedNodes) let presented: SnapshotBackendCapture do { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 143ac75555..c6be23f12b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -92,6 +92,11 @@ final class RunnerTests: XCTestCase { // Set while serving a command that had to re-activate the bound app, and stamped onto that // command's response before it leaves the execution queue (#2682). var pendingTargetActivation: TargetActivationFactPayload? + // The lit panel the command being served names, and the lock that guards it. A foldable's surface + // resolvers read it through `requestSurfacePanel`; see RunnerRequestPanel.swift for why it is not + // main-owned state and what an abandoned command's read can cost. + let requestPanelLock = NSLock() + var requestLitPanel: RunnerLitPanelFact? // iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when // hardware-keyboard input hides the software keyboard. A successful tap on a concrete text // input is a scoped witness for the immediately-following bare type; lifecycle and non-text diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 8e0c1c6b4c..80139d51d6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -1,5 +1,7 @@ #import #import +// The `RunnerLitPanel` the screen resolvers take, and the panel rule their walk applies. +#import "RunnerPanelSurface.h" NS_ASSUME_NONNULL_BEGIN @@ -29,6 +31,9 @@ typedef NS_ENUM(NSUInteger, RunnerApplicationScreenFailure) { RunnerApplicationScreenFailureUnresolvedWindow, // A resolved window whose screen or display identity the XCTest runtime would not answer. RunnerApplicationScreenFailureUnresolvedScreen, + // The request named a lit panel and every window the app reported is on the other one, so no + // surface on the screen the user is looking at exists to book against. + RunnerApplicationScreenFailureOffLitPanel, }; // Resolves the shared core against the live XCTest runtime. Returns nil on success, or a @@ -40,18 +45,78 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveXCTestEventBridge( RunnerXCTestEventBridge *bridge ); -// Resolves the `XCUIScreen` hosting the application's window and its `displayID`, the only display -// identity that follows an app to the panel it lights. Reading the window frame must precede -// reading its screen: an unresolved element still reports the main screen, and neither read is -// ever selected by number or position. Private KVC raises rather than returns nil, so both reads -// are guarded here and any failure surfaces as the typed `failure`, never an exception. +// Resolves the application's surface window and returns it, so every consumer of the resolved window +// — the capture's display, the interaction anchor and reference frame, the snapshot viewport, and the +// synthesized record's display — books against one window that one rule chose. +// +// `litPanel` is the panel the request named, or a `displayID` of 0 when it named none. With no panel +// this is the historic read: the first window the application reports with a usable frame, and no +// display identity is read at all, so a single-panel device pays nothing it did not pay before. With a +// panel, the walk reads each candidate's frame and display and keeps the window +// `RunnerPanelSurfaceSelectWindow` names, which is how a foldable home screen stops resolving to +// SpringBoard's status-bar window on whichever panel the runtime happens to list first. +// +// `*window` is the window element the runtime handed out, ready for `RunnerReadResolvedWindowScreen` +// or `RunnerResolveWindowDisplayID`; `*windowFrame` is the box the rule booked it for. `*observedFrame` +// is the last frame the walk looked at, so a refusal names what it refused. Private KVC raises rather +// than returns nil, so every read is guarded here and any failure surfaces as the typed `failure`, +// never an exception. The walk is bounded: reading a window that has just vanished makes XCTest record +// an issue that outlives the call (#2995). +FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveSurfaceWindow( + id application, + RunnerLitPanel litPanel, + id _Nullable *window, + CGRect *windowFrame, + CGRect *observedFrame +); + +// Reads the `XCUIScreen` and `displayID` of a window the caller has already resolved and measured. +// The panel walk holds the frame; asking a window it just measured for the same measurement again +// would only add a read of a window that may already be gone. `screen` may be NULL for a caller that +// routes by display identity alone. +FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( + id window, + CGRect observedFrame, + id _Nullable * _Nullable screen, + NSUInteger *displayID +); + +// The application's window elements, at most `maxCount` of them, read through the query the runtime +// exposes rather than a typed property, so a renamed key answers nothing instead of raising. Returns +// nil when the query itself cannot be answered, which a caller reads as an application with no +// windows. `reportedCount`, when asked for, is how many the query named before the cap, so a device +// the cap would silently shorten stays visible. Only the panel walk reads from this list, and it +// reads each element's attributes at most once, so a candidate the rule discards is never asked +// anything again (#2995). +FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyApplicationWindowElements( + id application, + NSUInteger maxCount, + NSUInteger * _Nullable reportedCount +); + +// Resolves the display hosting the application's surface window: the surface window, then the +// `XCUIScreen` that window reports and its `displayID` — the only display identity that follows an +// app to the panel it lights. Reading the window frame must precede reading its screen, and neither +// read is ever selected by number or position. A request that names no panel reads the historic first +// window and still asks that window for its screen, so single-panel captures are unchanged. FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( id application, + RunnerLitPanel litPanel, id _Nullable *screen, NSUInteger *displayID, RunnerApplicationScreenFailure *failure ); +// One window read: does it exist, and what box does it occupy. Both answers are required — an +// application with no window answers `frame` with `(0,0 0x0)` without raising, so only `exists` says +// whether the window is there. The frame is reported even when refused. Returns whether the window is +// there with a measurable box. +FOUNDATION_EXPORT BOOL RunnerReadWindowExistsAndFrame( + id window, + CGRect *observedFrame +); + + // Reads the display ID off an already-resolved window, keeping a synthesized record's display and // the caller's booked reference frame on the same window. The frame is still read before the // screen; a window that answers empty or raises fails closed with a refusal message naming the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index b32a285818..4d07c8122e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -3,6 +3,8 @@ #import "RunnerAXSnapshotBridge.h" // The shared full-screen rule, from the snapshot bridge; found via HEADER_SEARCH_PATHS. #import "ForegroundApplicationGeometry.h" +// The panel rule this file's window walk applies. +#import "RunnerPanelSurface.h" #import #import #import @@ -109,51 +111,69 @@ return @"no resolved application window"; case RunnerApplicationScreenFailureUnresolvedScreen: return @"no resolved window display ID"; + case RunnerApplicationScreenFailureOffLitPanel: + return @"every application window is off the lit panel"; case RunnerApplicationScreenFailureNone: return @"application screen resolved"; } } -// One window read. Both answers are required: an app with no window answers `frame` with -// `(0,0 0x0)` without raising, so only `exists` says whether the window is there. The window -// object is handed back because resolving this instance is what its `screen` will answer from. -// The frame is reported even when refused, so a refusal names what it looked at. -static RunnerApplicationScreenFailure RunnerResolveUsableWindow( - id application, - id _Nullable *window, - CGRect *observedFrame -) { - __block id readWindow = nil; +BOOL RunnerReadWindowExistsAndFrame(id window, CGRect *observedFrame) { __block CGRect frame = CGRectNull; __block BOOL resolved = NO; NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ - id candidate = [[application valueForKey:@"windows"] valueForKey:@"firstMatch"]; - NSNumber *present = [candidate valueForKey:@"exists"]; + NSNumber *present = [window valueForKey:@"exists"]; resolved = [present isKindOfClass:NSNumber.class] && present.boolValue; - NSValue *frameValue = [candidate valueForKey:@"frame"]; + // A window that says it is not there is not asked for its frame. XCTest records an issue for a + // frame read on a vanished element, an unmuted issue ends the runner session, and the walk asks + // several candidates (#2995). + if (!resolved) { + return; + } + NSValue *frameValue = [window valueForKey:@"frame"]; [frameValue getValue:&frame size:sizeof(frame)]; - readWindow = candidate; }]; *observedFrame = frame; - if (exception != nil || !resolved || !AgentDeviceForegroundFrameIsMeasurable(frame)) { - NSLog( - @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) raised=%@ resolved=%@", - frame.origin.x, - frame.origin.y, - frame.size.width, - frame.size.height, - exception != nil ? @"yes" : @"no", - resolved ? @"yes" : @"no" - ); - return RunnerApplicationScreenFailureUnresolvedWindow; + if (exception != nil) { + NSLog(@"AGENT_DEVICE_RUNNER_APP_WINDOW_FRAME_RAISED frame raised=%@", exception); + return NO; } - *window = readWindow; - return RunnerApplicationScreenFailureNone; + return resolved && AgentDeviceForegroundFrameIsMeasurable(frame); +} + +NSArray *RunnerCopyApplicationWindowElements( + id application, + NSUInteger maxCount, + NSUInteger *reportedCount +) { + if (reportedCount != NULL) { + *reportedCount = 0; + } + __block NSArray *windows = nil; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id elements = [[application valueForKey:@"windows"] valueForKey:@"allElementsBoundByIndex"]; + if ([elements isKindOfClass:NSArray.class]) { + windows = elements; + } + }]; + if (exception != nil) { + NSLog(@"AGENT_DEVICE_RUNNER_APP_WINDOWS_UNRESOLVED raised=%@", exception); + return nil; + } + if (windows == nil) { + return nil; + } + // The count is the list the query named, before the cap trims it, so a device whose window list + // outgrows the walk is visible to whoever reads the marker. + if (reportedCount != NULL) { + *reportedCount = windows.count; + } + return windows.count <= maxCount ? windows : [windows subarrayWithRange:NSMakeRange(0, maxCount)]; } // The screen read that must follow window resolution on that same window instance: before the frame // is asked for, `screen` can still name main, so this never merges into the read ahead of it. -static RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( +RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( id window, CGRect windowFrame, id _Nullable *screen, @@ -181,13 +201,170 @@ static RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( ); return RunnerApplicationScreenFailureUnresolvedScreen; } - *screen = readScreen; + if (screen != NULL) { + *screen = readScreen; + } *displayID = readDisplayID; return RunnerApplicationScreenFailureNone; } +static RunnerApplicationScreenFailure RunnerSurfaceFailureFromPanelSelection( + RunnerPanelSurfaceSelection selection +) { + switch (selection) { + case RunnerPanelSurfaceSelectionLitPanelWindow: + case RunnerPanelSurfaceSelectionLitDisplayWindow: + return RunnerApplicationScreenFailureNone; + case RunnerPanelSurfaceSelectionOffLitPanel: + return RunnerApplicationScreenFailureOffLitPanel; + case RunnerPanelSurfaceSelectionNoWindow: + return RunnerApplicationScreenFailureUnresolvedWindow; + } +} + +RunnerApplicationScreenFailure RunnerResolveSurfaceWindow( + id application, + RunnerLitPanel litPanel, + id _Nullable *window, + CGRect *windowFrame, + CGRect *observedFrame +) { + *window = nil; + *windowFrame = CGRectNull; + *observedFrame = CGRectNull; + + if (!RunnerLitPanelIsNamed(litPanel)) { + // The historic read, and the only one a single-panel device ever takes: the first window the + // query names that exists with a measurable box, and no display identity is asked for at all. + // The list is uncapped here for the same reason the historic read was uncapped: an app may own + // more windows than the panel walk's budget, and stopping early would refuse a screen the + // historic route resolved. Each candidate is asked twice at most, and the walk stops at the + // answer, so a transient window that vanishes mid-read is the only thing that can widen it + // (#2995). + __block CGRect refusedFrame = CGRectNull; + for (id candidate in RunnerCopyApplicationWindowElements( + application, NSUIntegerMax, NULL)) { + CGRect frame = CGRectNull; + if (!RunnerReadWindowExistsAndFrame(candidate, &frame)) { + refusedFrame = frame; + continue; + } + *window = candidate; + *windowFrame = frame; + *observedFrame = frame; + return RunnerApplicationScreenFailureNone; + } + NSLog( + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) resolved=no", + refusedFrame.origin.x, + refusedFrame.origin.y, + refusedFrame.size.width, + refusedFrame.size.height + ); + *observedFrame = refusedFrame; + return RunnerApplicationScreenFailureUnresolvedWindow; + } + + NSUInteger reported = 0; + NSArray *candidates = RunnerCopyApplicationWindowElements( + application, + AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS, + &reported + ); + // The walk reads at most the cap's worth of windows, so it may not have seen the whole list. That + // matters only for a refusal: a refusal that claims the lit panel holds nothing is honest only + // when nothing had to be left unread. + BOOL listTruncated = candidates.count < reported; + NSMutableArray *placed = [NSMutableArray array]; + RunnerPanelWindowFact facts[AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS]; + NSUInteger factCount = 0; + BOOL sawUnplaceableWindow = NO; + for (id candidate in candidates) { + if (factCount == AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS) { + break; + } + CGRect frame = CGRectNull; + if (!RunnerReadWindowExistsAndFrame(candidate, &frame)) { + continue; + } + NSUInteger displayID = 0; + if (RunnerReadResolvedWindowScreen(candidate, frame, NULL, &displayID) != + RunnerApplicationScreenFailureNone) { + // The window is there but the runtime will not say which display shows it, so the panel rule + // cannot place it. The rest of the walk still gets its chance. + sawUnplaceableWindow = YES; + *observedFrame = frame; + continue; + } + facts[factCount] = (RunnerPanelWindowFact){.displayID = displayID, .frame = frame}; + factCount += 1; + [placed addObject:candidate]; + } + + RunnerPanelWindowFact chosen = {.displayID = 0, .frame = CGRectNull}; + NSUInteger chosenIndex = NSUIntegerMax; + RunnerPanelSurfaceSelection selection = RunnerPanelSurfaceSelectionNoWindow; + if (factCount > 0) { + selection = RunnerPanelSurfaceSelectWindow( + facts, + factCount, + litPanel, + &chosen, + &chosenIndex + ); + } + NSLog( + @"AGENT_DEVICE_RUNNER_PANEL_SURFACE lit=%lu(%.1fx%.1f) reported=%lu placed=%lu unplaceable=%@ selection=%lu frame=(%g,%g %gx%g)", + (unsigned long)litPanel.displayID, + litPanel.widthPt, + litPanel.heightPt, + (unsigned long)reported, + (unsigned long)factCount, + sawUnplaceableWindow ? @"yes" : @"no", + (unsigned long)selection, + chosen.frame.origin.x, + chosen.frame.origin.y, + chosen.frame.size.width, + chosen.frame.size.height + ); + // A refusal names what the walk looked at: the frame the rule held for the list it was given. + if (factCount > 0) { + *observedFrame = chosen.frame; + } + // A claim about the whole window list is honest only when the walk read the whole list. An + // unplaceable window, or a list the cap cut short, leaves part of it unread, and the unread part + // could hold the lit panel's surface. Both of the rule's answers that depend on the rest of the + // list refuse here: `offLitPanel`, which says the lit panel holds nothing, and `litDisplayWindow`, + // which books the largest window the lit display names — with an unread tail, the panel's own + // full-screen window may be the one that went unread and the largest survivor a strip in a corner. + // A gesture booked to the right panel at the wrong box is the failure this fix exists to remove. + BOOL listUnread = sawUnplaceableWindow || listTruncated; + if (listUnread && + (selection == RunnerPanelSurfaceSelectionOffLitPanel || + selection == RunnerPanelSurfaceSelectionLitDisplayWindow)) { + return RunnerApplicationScreenFailureUnresolvedScreen; + } + RunnerApplicationScreenFailure failure = RunnerSurfaceFailureFromPanelSelection(selection); + if (failure != RunnerApplicationScreenFailureNone) { + if (factCount == 0 && sawUnplaceableWindow) { + return RunnerApplicationScreenFailureUnresolvedScreen; + } + return failure; + } + *observedFrame = chosen.frame; + // The rule indexes the windows it was handed, so this cannot trip; it bounds the subscript that + // follows rather than trusting the index across the rule boundary. + if (chosenIndex >= placed.count) { + return RunnerApplicationScreenFailureUnresolvedWindow; + } + *window = placed[chosenIndex]; + *windowFrame = chosen.frame; + return RunnerApplicationScreenFailureNone; +} + BOOL RunnerResolveApplicationScreen( id application, + RunnerLitPanel litPanel, id _Nullable *screen, NSUInteger *displayID, RunnerApplicationScreenFailure *failure @@ -196,7 +373,8 @@ BOOL RunnerResolveApplicationScreen( *displayID = 0; id window = nil; CGRect windowFrame = CGRectNull; - *failure = RunnerResolveUsableWindow(application, &window, &windowFrame); + CGRect observedFrame = CGRectNull; + *failure = RunnerResolveSurfaceWindow(application, litPanel, &window, &windowFrame, &observedFrame); if (*failure != RunnerApplicationScreenFailureNone) { return NO; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift new file mode 100644 index 0000000000..6e73272bec --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift @@ -0,0 +1,237 @@ +import XCTest +import AgentDeviceSnapshotPresentation + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - What one panel's fact says about a captured tree (ADR 0025) + +extension RunnerTests { + /// The Duo's lit inner panel as the interaction space reports it: the panel table names 669x951 and + /// the system presents that panel turned, so its surface box swaps the side lengths. + private static let litSurface = CGRect(x: 0, y: 0, width: 951, height: 669) + /// SpringBoard's outer-panel window, which the platform handed back as the app's only window on the + /// run that found this bug. The device was unfolded, so nothing in it can be seen. + private static let offPanelWindow = CGRect(x: 0, y: 0, width: 466, height: 678) + + private static func panelNode( + _ index: Int, + type: String, + rect: CGRect, + parentIndex: Int? + ) -> RawAXNode { + RawAXNode( + index: index, + type: type, + label: nil, + identifier: nil, + value: nil, + rect: SnapshotRect(rect), + enabled: true, + focused: nil, + selected: nil, + hittable: parentIndex == nil ? false : true, + depth: parentIndex == nil ? 0 : 1, + parentIndex: parentIndex, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) + } + + /// The tree the bug produced: an app root naming the dark panel, one window naming the dark panel, and + /// content under it. + private static func darkPanelOnlyTree() -> [RawAXNode] { + [ + panelNode(0, type: "Application", rect: Self.offPanelWindow, parentIndex: nil), + panelNode(1, type: "Window", rect: Self.offPanelWindow, parentIndex: 0), + panelNode(2, type: "Button", rect: CGRect(x: 20, y: 120, width: 250, height: 42), parentIndex: 1), + ] + } + + func testPanelBoxMatchesItsOwnBoxAndThatBoxTurned() { + XCTAssertTrue( + RunnerPanelSurfaceTree.matchesSurfaceBox(Self.litSurface, Self.litSurface), + "the panel's own box is its surface" + ) + XCTAssertTrue( + RunnerPanelSurfaceTree.matchesSurfaceBox( + CGRect(x: 0, y: 0, width: 669, height: 951), + Self.litSurface + ), + "the panel table reports native points and the system presents the panel turned" + ) + XCTAssertFalse( + RunnerPanelSurfaceTree.matchesSurfaceBox(Self.offPanelWindow, Self.litSurface), + "another panel's box is not this panel's surface" + ) + XCTAssertFalse( + RunnerPanelSurfaceTree.matchesSurfaceBox( + CGRect(x: 0, y: 0, width: 951, height: 600), + Self.litSurface + ), + "a window that stops short of the panel is an app occupying part of it, not the surface" + ) + } + + func testASurfaceLiesOnThePanelItCanBeSeenOn() { + XCTAssertTrue( + RunnerPanelSurfaceTree.liesOnSurface(CGRect(x: 0, y: 0, width: 951, height: 669), Self.litSurface) + ) + XCTAssertTrue( + RunnerPanelSurfaceTree.liesOnSurface( + CGRect(x: 100, y: 50, width: 300, height: 200), + Self.litSurface + ), + "a sheet inside the panel's box is on the panel" + ) + XCTAssertFalse( + RunnerPanelSurfaceTree.liesOnSurface(Self.offPanelWindow, Self.litSurface), + "a box reaching past the panel's bottom edge is on another display" + ) + XCTAssertFalse( + RunnerPanelSurfaceTree.liesOnSurface(CGRect(x: 0, y: 0, width: 0, height: 669), Self.litSurface), + "a box with no width names no surface at all" + ) + } + + /// The root node is the box the host reads as the payload's viewport, so an app root that reports a + /// panel's box is republished with the surface box the capture resolved. + func testPanelTreePublishesTheSurfaceBoxOnItsRoot() throws { + let published = try XCTUnwrap( + RunnerPanelSurfaceTree.publishing(nodes: Self.darkPanelOnlyTree(), surface: Self.litSurface) + ) + XCTAssertTrue(published.republishedRoot) + XCTAssertEqual(published.droppedSurfaces, 1) + XCTAssertEqual(published.nodes.count, 1, "the dark panel's window and its content are both gone") + XCTAssertEqual(published.nodes[0].rect.cgRect, Self.litSurface) + XCTAssertNil(published.nodes[0].parentIndex) + } + + /// Dropping a surface drops its subtree: a ref for a control on a dark panel would name a point no + /// gesture can land on, and the host's own containment rule refuses the node anyway. + func testPanelTreeDropsAWindowsWholeSubtree() throws { + var nodes = Self.darkPanelOnlyTree() + nodes.append( + Self.panelNode( + 3, + type: "Cell", + rect: CGRect(x: 20, y: 170, width: 250, height: 42), + parentIndex: 1 + ) + ) + nodes.append( + Self.panelNode( + 4, + type: "Window", + rect: Self.litSurface, + parentIndex: 0 + ) + ) + nodes.append( + Self.panelNode( + 5, + type: "Button", + rect: CGRect(x: 470, y: 54, width: 60, height: 60), + parentIndex: 4 + ) + ) + let published = try XCTUnwrap(RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface)) + XCTAssertEqual(published.droppedSurfaces, 1) + XCTAssertEqual( + published.nodes.map(\.type), + ["Application", "Window", "Button"], + "the lit panel's window and its control survive" + ) + XCTAssertEqual(published.nodes.map(\.index), Array(0..<3), "the wire array stays positional") + XCTAssertEqual(published.nodes[1].parentIndex, 0, "the kept window is still the app's child") + XCTAssertEqual(published.nodes[2].parentIndex, 1, "the kept control is still its window's child") + XCTAssertEqual( + published.nodes[2].depth, + nodes[5].depth, + "whole siblings were removed, so no kept node changed depth" + ) + } + + /// The common case must cost nothing: an app whose window fills the lit panel already publishes the + /// right root box, and there is nothing to drop. + func testPanelTreeLeavesATreeOfTheLitPanelAlone() { + let nodes = [ + Self.panelNode(0, type: "Application", rect: Self.litSurface, parentIndex: nil), + Self.panelNode(1, type: "Window", rect: Self.litSurface, parentIndex: 0), + ] + XCTAssertNil(RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface)) + } + + /// A keyboard band, an alert, or a hosted surface that reaches past the panel's edge is content the + /// fold clips, so only a root's own children are ever dropped. + func testPanelTreeKeepsContentThatReachesPastThePanel() { + let nodes = [ + Self.panelNode(0, type: "Application", rect: Self.litSurface, parentIndex: nil), + Self.panelNode(1, type: "Window", rect: Self.litSurface, parentIndex: 0), + Self.panelNode( + 2, + type: "ScrollView", + rect: CGRect(x: -57, y: -40, width: 1065, height: 749), + parentIndex: 1 + ), + ] + XCTAssertNil(RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface)) + } + + /// The private-AX tier's tree and a scoped capture are rooted at what they read rather than at an + /// application element. Such a payload was read from one surface, so its nodes are left as they were + /// read: narrowing them would be a guess about which screen they came from. + func testPanelTreeLeavesATreeNotRootedAtAnApplicationAlone() { + for nodes in [ + [Self.panelNode(0, type: "Window", rect: Self.offPanelWindow, parentIndex: nil)], + [ + Self.panelNode(0, type: "Other", rect: Self.offPanelWindow, parentIndex: nil), + Self.panelNode(1, type: "Window", rect: Self.offPanelWindow, parentIndex: 0), + ], + ] { + XCTAssertNil( + RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface), + "no application root, so no window list to narrow" + ) + } + } + + /// The capture's own evidence that its tree already shows the lit panel, which is what decides whether + /// a second read of the resolved window is needed at all. + func testCarriesSurfaceWindowReadsOneLayerOnly() { + struct Layer { + let type: String + let rect: CGRect + } + let layers = [ + Layer(type: "Other", rect: Self.litSurface), + Layer(type: "Window", rect: Self.offPanelWindow), + ] + XCTAssertFalse( + RunnerPanelSurfaceTree.carriesSurfaceWindow( + layers, + surface: Self.litSurface, + isSurface: { $0.type == "Window" }, + frame: { $0.rect } + ), + "the only window is on another panel" + ) + XCTAssertTrue( + RunnerPanelSurfaceTree.carriesSurfaceWindow( + layers + [Layer(type: "Window", rect: CGRect(x: 0, y: 0, width: 669, height: 951))], + surface: Self.litSurface, + isSurface: { $0.type == "Window" }, + frame: { $0.rect } + ), + "a turned window on the panel counts" + ) + XCTAssertFalse( + RunnerPanelSurfaceTree.carriesSurfaceWindow( + [Layer(type: "Other", rect: Self.litSurface)], + surface: Self.litSurface, + isSurface: { $0.type == "Window" }, + frame: { $0.rect } + ), + "a container is not a surface" + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift new file mode 100644 index 0000000000..57f55802f3 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -0,0 +1,55 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - The request-scoped lit panel (#2728, ADR 0025) + +extension RunnerTests { + /// The panel a command names is the panel that command's resolvers see, and a request that names + /// none clears whatever the previous command left behind. One runner process can serve a foldable's + /// `tap` and then a single-panel device's, and the second must not resolve against the first's panel. + func testRequestPanelFollowsTheCommandAndAnUnnamedPanelClearsIt() throws { + setRequestLitPanel(from: try command(namingLitPanel: #"{"displayId":3,"widthPt":669,"heightPt":951}"#)) + XCTAssertEqual(requestSurfacePanel.displayID, 3) + XCTAssertEqual(requestSurfacePanel.widthPt, 669) + XCTAssertEqual(requestSurfacePanel.heightPt, 951) + + setRequestLitPanel(from: try command(namingLitPanel: nil)) + XCTAssertEqual( + requestSurfacePanel.displayID, + 0, + "a request that names no panel must not inherit the previous command's" + ) + } + + /// A panel that cannot describe a box cannot be compared with a window, so it answers as no panel + /// and the resolver keeps the historic route rather than refusing every window it reads. The rows + /// are the ways a panel table can report a panel without naming one: no display, or a display whose + /// box has no size. + func testRequestPanelTreatsAPanelNamingNoUsableBoxAsNoPanel() throws { + for panel in [ + #"{"displayId":0,"widthPt":669,"heightPt":951}"#, + #"{"displayId":3,"widthPt":0,"heightPt":951}"#, + #"{"displayId":3,"widthPt":669,"heightPt":0}"#, + #"{"displayId":3,"widthPt":-1,"heightPt":-951}"#, + ] { + setRequestLitPanel(from: try command(namingLitPanel: nil)) + setRequestLitPanel(from: try command(namingLitPanel: panel)) + XCTAssertEqual( + requestSurfacePanel.displayID, + 0, + "display 0 is what every resolver reads as the historic first-window route (\(panel))" + ) + } + } + + /// Built from JSON, because the slot's input is the wire shape the host sends; a Swift literal would + /// keep compiling if the field were renamed. + private func command(namingLitPanel panel: String?) throws -> Command { + let fields = ["\"command\":\"tap\"", panel.map { "\"litPanel\":\($0)" }].compactMap { $0 } + return try JSONDecoder().decode( + Command.self, + from: Data("{\(fields.joined(separator: ","))}".utf8) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift index 74b65986a2..08ab26edd5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift @@ -49,6 +49,12 @@ extension RunnerTests { RunnerAppScreenCaptureFailure.unrenderableImage.rawValue, "APP_SCREEN_CAPTURE_UNRENDERABLE" ) + // A panel refusal has a system surface left to ask, so it is its own code too: "the app has + // nothing on the lit panel" and "no window anywhere" send an operator to different places. + XCTAssertEqual( + RunnerAppScreenCaptureFailure(.offLitPanel).rawValue, + "APP_SCREEN_OFF_LIT_PANEL" + ) // A success-shaped reason cannot become a success: the only way here is a caller that forgot to // branch, and the capture it describes did not happen. XCTAssertEqual( @@ -118,6 +124,30 @@ extension RunnerTests { ) } + /// Every window the session app has is on the dark panel: that says nothing about where the lit + /// panel's content is, because the home screen is SpringBoard's window. So the panel refusal moves + /// the question on exactly like an unresolved window does. + func testObservedScreenCaptureAsksTheSystemSurfaceWhenEveryAppWindowIsOffTheLitPanel() { + var askedSystemSurface = false + let outcome = selectObservedScreenCapture( + resolving: { .failure(.offLitPanel) }, + fallingBack: { + askedSystemSurface = true + return .success( + CapturedAppScreen( + image: UIImage(), + displayID: 3, + pixelWidth: 2852, + pixelHeight: 2006, + pixelsPerPoint: 3 + ) + ) + } + ) + XCTAssertTrue(askedSystemSurface, "the lit panel's content belongs to another process") + XCTAssertEqual(try? outcome.get().displayID, 3) + } + /// A window that resolved and then refused to name its display is the failure the host has to /// see; asking a second process would replace a real refusal with an unrelated capture. func testObservedScreenCaptureReportsADisplayRefusalWithoutAskingTheSystemSurface() { @@ -129,7 +159,10 @@ extension RunnerTests { return .failure(.unrenderableImage) } ) - XCTAssertFalse(askedSystemSurface, "only an unresolved window may reach the system surface") + XCTAssertFalse( + askedSystemSurface, + "only an unresolved window or a panel refusal may reach the system surface" + ) guard case .failure(let refusal) = outcome else { return XCTFail("a display that refused must not become a capture") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index 7c603af668..d2a3644c56 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -76,16 +76,22 @@ extension RunnerTests { let plans = requests.compactMap { $0["gesturePlan"] as? [String: Any] } let pointers = plans.flatMap { $0["pointers"] as? [[String: Any]] ?? [] } let samples = pointers.flatMap { $0["samples"] as? [[String: Any]] ?? [] } + let litPanels = requests.compactMap { $0["litPanel"] as? [String: Any] } let commands = try entries.map { try decodeProductionRunnerRequest($0.request, $0.name) } let command = try XCTUnwrap(commands.first, "no production request") let plan = try XCTUnwrap( commands.compactMap(\.gesturePlan).first, "no production request carries a gesturePlan" ) + let litPanel = try XCTUnwrap( + commands.compactMap(\.litPanel).first, + "no production request carries a litPanel, so a foldable's surface route would go unpinned" + ) let step = try JSONDecoder().decode(SequenceStep.self, from: Data(#"{"kind":"tap"}"#.utf8)) let sample = try XCTUnwrap(plan.pointers.first?.samples.first) assertEveryStoredField(of: command, appearsIn: requests, "Command") assertEveryStoredField(of: step, appearsIn: steps, "SequenceStep") + assertEveryStoredField(of: litPanel, appearsIn: litPanels, "RunnerLitPanelFact") assertEveryStoredField(of: plan, appearsIn: plans, "RunnerGesturePlan") assertEveryStoredField( of: plan.viewport, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift index e71ab44527..6b5dcb0ae0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -392,6 +392,7 @@ extension RunnerTests { let context = try XCTUnwrap(makeSnapshotTraversalContext(app: app, hint: hint)) let startedAt = Date() let capture = try recursiveTreeSnapshotAcquisition( + app: app, context: context, hint: hint, processingDeadline: startedAt.addingTimeInterval(treeCaptureSliceBudget) @@ -430,10 +431,11 @@ extension RunnerTests { let context = try XCTUnwrap(makeSnapshotTraversalContext(app: app, hint: hint)) let tabBar = try XCTUnwrap(Self.snapshotDescendants(of: context.rootSnapshot).first { $0.elementType == .tabBar }) XCTAssertTrue(tabBar.children.isEmpty, "the fixture must exercise collapsed-tab recovery") - let unstarted = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: Date()) + let unstarted = try recursiveTreeSnapshotAcquisition(app: app, context: context, hint: hint, processingDeadline: Date()) XCTAssertEqual(unstarted.outcome, .deadlineExhausted) XCTAssertTrue(unstarted.acquisition.truncated) let capture = try recursiveTreeSnapshotAcquisition( + app: app, context: context, hint: hint, processingDeadline: Date().addingTimeInterval(treeCaptureSliceBudget) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 289012ccf7..b46575da06 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -32,7 +32,15 @@ private final class DisplayWindowFixture: NSObject { private final class DisplayWindowsFixture: NSObject { @objc let firstMatch: NSObject - init(_ window: NSObject) { firstMatch = window } + @objc let allElementsBoundByIndex: [NSObject] + init(_ window: NSObject) { + firstMatch = window + allElementsBoundByIndex = [window] + } + init(windows: [NSObject]) { + firstMatch = windows.first ?? NSObject() + allElementsBoundByIndex = windows + } } private final class DisplayApplicationFixture: NSObject { @@ -60,6 +68,101 @@ private final class FramelessWindowFixture: NSObject { } extension RunnerTests { + /// The no-panel request a single-panel device sends. Every existing case below reads this: a + /// request that names no panel keeps the historic first-window route. + static var noPanelFixture: RunnerLitPanel { RunnerLitPanel(displayID: 0, widthPt: 0, heightPt: 0) } + + /// The Duo's lit inner panel as a request names it: the panel table reports 669x951 native points. + static var duoInnerPanel: RunnerLitPanel { RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) } + + /// The inner panel's home-screen window as the runtime reports it: presented turned, so the + /// side lengths swap and it reads 951x669. + static let duoHomeScreenFrame = CGRect(x: 0, y: 0, width: 951, height: 669) + + fileprivate static func duoHomeScreenWindow() -> DisplayWindowFixture { + DisplayWindowFixture(frame: duoHomeScreenFrame, displayID: 3) + } + + /// SpringBoard's first reported window: the 226x155.3 status-bar window, on the outer panel. + fileprivate static func springBoardStatusBarWindow() -> DisplayWindowFixture { + DisplayWindowFixture(frame: CGRect(x: 452, y: 0, width: 226, height: 155.3), displayID: 1) + } + + /// `count` full-screen windows on the outer panel, which is the dark one while the inner panel is + /// lit. SpringBoard really names dozens, so a case that needs the walk's budget spent uses this. + fileprivate static func outerPanelWindows(_ count: Int) -> [NSObject] { + (0.. DisplayWindowFixture { + DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 226, height: 155.3), displayID: 3) + } + + /// A folded Duo's home screen seen while the inner panel is lit: every window it names is on the + /// outer panel, so nothing here may become the surface. + fileprivate static func darkPanelOnlyWindows() -> [NSObject] { + [springBoardStatusBarWindow()] + outerPanelWindows(1) + } + + fileprivate static func application(_ windows: [NSObject]) -> DisplayApplicationFixture { + DisplayApplicationFixture(windows: DisplayWindowsFixture(windows: windows)) + } + + /// One screen read through the full route, holding the three in-out boxes every caller declares. + /// `screenSeed` lets a refusal case prove the resolver clears a screen object it was handed. + fileprivate func resolveScreen( + _ application: NSObject, + litPanel: RunnerLitPanel, + screenSeed: AnyObject? = nil + ) -> (succeeded: Bool, screen: AnyObject?, displayID: UInt, failure: RunnerApplicationScreenFailure) { + var screen = screenSeed + var displayID: UInt = 0 + var failure = RunnerApplicationScreenFailure.none + let succeeded = RunnerResolveApplicationScreen( + application, litPanel, &screen, &displayID, &failure + ) + return (succeeded, screen, displayID, failure) + } + + /// One screen read that must refuse, and the three facts every refusal holds: the typed failure, + /// no screen object, and no display named. + fileprivate func assertScreenReadRefuses( + _ application: NSObject, + expected: RunnerApplicationScreenFailure, + litPanel: RunnerLitPanel, + screenMessage: String? = nil + ) { + let read = resolveScreen(application, litPanel: litPanel, screenSeed: NSObject()) + XCTAssertFalse(read.succeeded) + XCTAssertEqual(read.failure, expected) + if let screenMessage { + XCTAssertNil(read.screen, screenMessage) + } else { + XCTAssertNil(read.screen) + } + XCTAssertEqual(read.displayID, 0, "a refusal names no display") + } + + /// One surface walk, holding the same in-out boxes, so a case states only the shape it feeds and + /// the window it expects back. + fileprivate func resolveSurfaceWindow( + _ application: NSObject, + litPanel: RunnerLitPanel + ) -> (failure: RunnerApplicationScreenFailure, window: AnyObject?, windowFrame: CGRect, observedFrame: CGRect) { + var window: AnyObject? + var windowFrame = CGRect.null + var observedFrame = CGRect.null + let failure = RunnerResolveSurfaceWindow( + application, litPanel, &window, &windowFrame, &observedFrame + ) + return (failure, window, windowFrame, observedFrame) + } + + func testSynthesizedDisplayRoutesByTheResolvedWindow() { let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 3) var displayID: UInt = 0 @@ -111,98 +214,357 @@ extension RunnerTests { } func testResolvedScreenHandsBackTheScreenTheResolvedWindowReported() { - let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 3) - var screen: AnyObject? - var displayID: UInt = 0 - var failure = RunnerApplicationScreenFailure.unresolvedScreen - XCTAssertTrue( - RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) - ) - XCTAssertEqual(displayID, 3) + let window = Self.duoHomeScreenWindow() + let read = resolveScreen(DisplayApplicationFixture(window), litPanel: Self.noPanelFixture) + XCTAssertTrue(read.succeeded) + XCTAssertEqual(read.displayID, 3) XCTAssertTrue(window.resolved, "the window's screen only answers once its query resolved") XCTAssertTrue( - screen === window.screen, + read.screen === window.screen, "the capture must go to the screen object this window reported, not a re-queried one" ) } func testResolvedScreenRefusesAWindowLookupThatRaises() { - var screen: AnyObject? = NSObject() - var displayID: UInt = 99 - var failure = RunnerApplicationScreenFailure.none - XCTAssertFalse( - RunnerResolveApplicationScreen( - DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), - &screen, - &displayID, - &failure - ) + assertScreenReadRefuses( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), + expected: .unresolvedWindow, + litPanel: Self.noPanelFixture ) - XCTAssertEqual(failure, .unresolvedWindow) - XCTAssertNil(screen) - XCTAssertEqual(displayID, 0) } func testResolvedScreenRefusesAResolvedWindowWithNoAnswerableScreen() { - var screen: AnyObject? = NSObject() - var displayID: UInt = 99 - var failure = RunnerApplicationScreenFailure.none - XCTAssertFalse( - RunnerResolveApplicationScreen( - DisplayApplicationFixture(ScreenlessWindowFixture()), - &screen, - &displayID, - &failure - ) + assertScreenReadRefuses( + DisplayApplicationFixture(ScreenlessWindowFixture()), + expected: .unresolvedScreen, + litPanel: Self.noPanelFixture ) - XCTAssertEqual(failure, .unresolvedScreen) - XCTAssertNil(screen) - XCTAssertEqual(displayID, 0) } func testResolvedScreenRefusesAWindowReportingNoDisplayID() { - let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 0) - var screen: AnyObject? = NSObject() - var displayID: UInt = 99 - var failure = RunnerApplicationScreenFailure.none - XCTAssertFalse( - RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + let outerPanel = DisplayWindowFixture( + frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 0) + assertScreenReadRefuses( + DisplayApplicationFixture(outerPanel), expected: .unresolvedScreen, litPanel: Self.noPanelFixture ) - XCTAssertEqual(failure, .unresolvedScreen) - XCTAssertNil(screen) - XCTAssertEqual(displayID, 0) } func testResolvedScreenRefusesAnEmptyWindowFrameWithoutNamingAnyDisplay() { - let window = DisplayWindowFixture(frame: .zero, displayID: 3) - var screen: AnyObject? = NSObject() - var displayID: UInt = 99 - var failure = RunnerApplicationScreenFailure.none - XCTAssertFalse( - RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + assertScreenReadRefuses( + DisplayApplicationFixture(DisplayWindowFixture(frame: .zero, displayID: 3)), + expected: .unresolvedWindow, + litPanel: Self.noPanelFixture, + screenMessage: "a dark panel must never become the capture target by default" ) - XCTAssertEqual(failure, .unresolvedWindow) - XCTAssertNil(screen, "a dark panel must never become the capture target by default") } /// A cold query answers `frame` with a usable rect on some runtimes while its `exists` says the /// window is not there. Geometry alone must not be trusted into capturing a panel nobody resolved. func testResolvedScreenRefusesAWindowTheQuerySaysIsNotThere() { - let window = DisplayWindowFixture( - frame: CGRect(x: 0, y: 0, width: 951, height: 669), - displayID: 3, - present: false + let absent = DisplayWindowFixture( + frame: Self.duoHomeScreenFrame, displayID: 3, present: false) + assertScreenReadRefuses( + DisplayApplicationFixture(absent), expected: .unresolvedWindow, litPanel: Self.noPanelFixture + ) + } + + /// #2995: a `frame` read on an element the query says is not there can make XCTest record an issue, + /// and an unmuted issue ends the runner session. The walk reads `exists` and stops there, on both + /// routes, so a candidate that vanished between listing and reading costs one read, not two. + func testSurfaceWalkNeverAsksAnAbsentWindowForItsFrame() { + for panel in [Self.noPanelFixture, Self.duoInnerPanel] { + let absent = DisplayWindowFixture( + frame: Self.duoHomeScreenFrame, displayID: 3, present: false) + let walk = resolveSurfaceWindow(Self.application([absent]), litPanel: panel) + XCTAssertEqual(walk.failure, .unresolvedWindow) + XCTAssertFalse( + absent.resolved, + "an absent window must not be asked for its frame (panel=\(panel.displayID))" + ) + } + } + + /// The panel rule's own decision, from facts alone: no queries, no device, no clock. The rows are + /// the shapes a foldable home screen actually presents — SpringBoard's status-bar window is a + /// 226x155.3 box, and the inner panel reports 951x669 for the 669x951 the panel table names. + func testPanelSurfaceRuleSelectsTheWindowOnTheLitPanel() { + let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + let statusBar = RunnerPanelWindowFact( + displayID: 1, frame: CGRect(x: 452, y: 0, width: 226, height: 155.3)) + let home = RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 0, width: 951, height: 669)) + var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) + var index: UInt = UInt.max + + XCTAssertEqual( + RunnerPanelSurfaceSelectWindow([statusBar, home], 2, lit, &selected, &index), + .litPanelWindow + ) + XCTAssertEqual(index, 1, "the window on the lit panel wins over the one listed first") + XCTAssertEqual(selected.displayID, 3) + XCTAssertEqual(selected.frame, home.frame) + } + + func testPanelSurfaceRuleAcceptsAPanelPresentedTurned() { + // The panel table names the inner panel 669x951; the system presents it turned, so its window + // reports 951x669. Both ways up are the same panel. + let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + let upright = RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 0, width: 669, height: 951)) + let turned = RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 0, width: 951, height: 669)) + for candidate in [upright, turned] { + let facts = [candidate] + var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) + var index: UInt = UInt.max + XCTAssertEqual( + RunnerPanelSurfaceSelectWindow(facts, 1, lit, &selected, &index), + .litPanelWindow, + "expected \(candidate.frame) to qualify as the lit panel's surface" + ) + XCTAssertEqual(selected.frame, candidate.frame) + XCTAssertEqual(index, 0) + } + } + + func testPanelSurfaceRuleRefusesWindowsThatAreAllOffTheLitPanel() { + // A full-screen window on the *dark* panel is the trap: geometry alone would happily book a + // gesture on a screen nobody is looking at. + let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + let facts = [ + RunnerPanelWindowFact(displayID: 1, frame: CGRect(x: 452, y: 0, width: 226, height: 155.3)), + RunnerPanelWindowFact(displayID: 1, frame: CGRect(x: 0, y: 0, width: 466, height: 678)), + ] + var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) + var index: UInt = UInt.max + XCTAssertEqual( + RunnerPanelSurfaceSelectWindow(facts, 2, lit, &selected, &index), + .offLitPanel + ) + XCTAssertEqual(selected.displayID, 1, "a refusal reports the first window it looked at") + XCTAssertEqual(index, 0) + } + + func testPanelSurfaceRuleBooksTheLargestWindowPlacedOnTheLitPanel() { + // An app that occupies part of the lit panel (Split View, a stage, a sheet) has no box equal to + // the panel's, and being on the lit panel is still the fact a first-window walk could not + // establish. The status-bar window is on the same display and loses on size. + let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + let facts = [ + RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 452, y: 0, width: 226, height: 155.3)), + RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 90, width: 300, height: 400)), + RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 20, y: 20, width: 120, height: 900)), + ] + var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) + var index: UInt = UInt.max + XCTAssertEqual( + RunnerPanelSurfaceSelectWindow(facts, 3, lit, &selected, &index), + .litDisplayWindow + ) + XCTAssertEqual(index, 1) + XCTAssertEqual(selected.frame, CGRect(x: 0, y: 90, width: 300, height: 400)) + } + + func testPanelSurfaceRuleRefusesNoWindows() { + let facts: [RunnerPanelWindowFact] = [] + var selected = RunnerPanelWindowFact(displayID: 7, frame: CGRect(x: 1, y: 2, width: 3, height: 4)) + var index: UInt = UInt.max + XCTAssertEqual( + RunnerPanelSurfaceSelectWindow( + facts, 0, RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951), &selected, &index), + .noWindow + ) + XCTAssertTrue(CGRectIsNull(selected.frame), "a refusal with nothing to report names no frame") + XCTAssertEqual(index, UInt.max) + } + + func testPanelSurfaceRuleRequiresTheOriginAndRejectsUnmeasurableFrames() { + let lit = RunnerLitPanel(displayID: 1, widthPt: 466, heightPt: 678) + XCTAssertTrue( + RunnerPanelFrameIsLitPanelSurface(CGRect(x: 0, y: 0, width: 466, height: 678), lit) + ) + // Half a point is rounding; a whole point is a different box. + XCTAssertTrue( + RunnerPanelFrameIsLitPanelSurface(CGRect(x: 0.4, y: -0.4, width: 466, height: 678), lit) + ) + XCTAssertFalse( + RunnerPanelFrameIsLitPanelSurface(CGRect(x: 1, y: 0, width: 466, height: 678), lit) + ) + // The same size in the wrong place, and a panel's size on the wrong panel's shape. + XCTAssertFalse( + RunnerPanelFrameIsLitPanelSurface(CGRect(x: 452, y: 0, width: 466, height: 678), lit) ) - var screen: AnyObject? = NSObject() - var displayID: UInt = 99 - var failure = RunnerApplicationScreenFailure.none XCTAssertFalse( - RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + RunnerPanelFrameIsLitPanelSurface(CGRect(x: 0, y: 0, width: 223, height: 317), lit) + ) + for frame in [CGRect.zero, .null, .infinite] { + XCTAssertFalse(RunnerPanelFrameIsLitPanelSurface(frame, lit), "\(frame) is not a measurement") + } + } + + /// The walk the four surface paths run: with a panel, the app's first window is not the answer. + /// The foldable home screen below is the shape that made a long press open nothing — the runtime + /// lists SpringBoard's status-bar window first, on the dark panel. + func testSurfaceWindowWalkWithAPanelSkipsTheFirstWindowOnTheDarkPanel() { + let statusBar = Self.springBoardStatusBarWindow() + let home = Self.duoHomeScreenWindow() + let walk = resolveSurfaceWindow(Self.application([statusBar, home]), litPanel: Self.duoInnerPanel) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === home, "the surface is the window on the lit panel") + XCTAssertEqual(walk.windowFrame, Self.duoHomeScreenFrame) + } + + /// A request that names no panel is a single-panel device's request: the walk stops at the first + /// window the query names and never asks a window which display it is on. + func testSurfaceWindowWalkWithoutAPanelKeepsTheFirstWindowAndReadsNoDisplay() { + let first = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 390, height: 844), displayID: 1) + let walk = resolveSurfaceWindow(Self.application([first]), litPanel: Self.noPanelFixture) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === first) + XCTAssertEqual(walk.windowFrame, CGRect(x: 0, y: 0, width: 390, height: 844)) + } + + func testSurfaceWindowWalkRefusesAWindowOnTheDarkPanel() { + let walk = resolveSurfaceWindow( + Self.application(Self.darkPanelOnlyWindows()), litPanel: Self.duoInnerPanel) + XCTAssertEqual(walk.failure, .offLitPanel, "no window is on the lit panel, so nothing may be booked") + XCTAssertNil(walk.window, "a dark panel must never become the surface by default") + XCTAssertEqual(walk.windowFrame, .null) + } + + /// A window whose display the runtime will not name cannot be placed on a panel. It is skipped, + /// and a window that does answer still resolves. + func testSurfaceWindowWalkSkipsAWindowThatWillNotNameItsDisplay() { let home = Self.duoHomeScreenWindow() + let walk = resolveSurfaceWindow( + Self.application([ScreenlessWindowFixture(), home]), litPanel: Self.duoInnerPanel) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === home) + } + + /// When every window that has a frame refuses to name its display, the walk has nothing to place, + /// and the host must see the display refusal rather than a window it can retry. + func testSurfaceWindowWalkReportsADisplayRefusalWhenNoWindowCanBePlaced() { + let walk = resolveSurfaceWindow( + Self.application([ScreenlessWindowFixture()]), litPanel: Self.duoInnerPanel) + XCTAssertEqual(walk.failure, .unresolvedScreen) + XCTAssertNil(walk.window) + } + + /// `offLitPanel` says the lit panel holds nothing, which is a claim about the whole window list. A + /// window the runtime would not place leaves part of that list unread, so the walk reports the + /// display refusal it can support instead of a panel it only partly saw. + func testSurfaceWindowWalkReportsADisplayRefusalWhenAnUnreadableWindowRemains() { + let walk = resolveSurfaceWindow( + Self.application(Self.darkPanelOnlyWindows() + [ScreenlessWindowFixture()]), + litPanel: Self.duoInnerPanel + ) + // An unreadable window could be the lit panel's surface, so nothing may be claimed about it. + XCTAssertEqual(walk.failure, .unresolvedScreen) + XCTAssertNil(walk.window) + } + + /// `litDisplayWindow` books the largest window the lit display names. When the query named more + /// windows than the walk may read, the panel's own full-screen window could be the one left unread, + /// and the largest survivor could be a strip in a corner. That is the right panel at the wrong box, + /// so an unread list refuses instead of booking there. + func testSurfaceWindowWalkRefusesAWindowOnTheLitPanelWhenTheListWasCutShort() { + // Seven full-screen windows on the dark panel, then a strip the lit panel does name. The cap + // reads all eight and never reaches the ninth and tenth, where the lit panel's own full-screen + // window would sit. + let unreadHome = DisplayWindowFixture( + frame: Self.duoHomeScreenFrame, displayID: Self.duoInnerPanel.displayID) + let walk = resolveSurfaceWindow( + Self.application( + Self.outerPanelWindows(7) + [Self.innerPanelStripWindow(), unreadHome, unreadHome]), + litPanel: Self.duoInnerPanel + ) + XCTAssertEqual( + walk.failure, .unresolvedScreen, + "the lit panel's surface could be the window the cap left unread" + ) + XCTAssertNil(walk.window) + } + + /// The same list the case above refuses, short enough that the walk read all of it: the strip is + /// then the honest answer, which is what makes the refusal above about the unread windows and + /// nothing else. + func testSurfaceWindowWalkBooksTheLargestWindowOnTheLitPanelWhenTheListWasReadFully() { + let walk = resolveSurfaceWindow( + Self.application(Self.outerPanelWindows(6) + [Self.innerPanelStripWindow()]), + litPanel: Self.duoInnerPanel + ) + XCTAssertEqual(walk.failure, .none) + XCTAssertEqual(walk.windowFrame, CGRect(x: 0, y: 0, width: 226, height: 155.3)) + } + + /// A request naming no panel keeps the historic uncapped read, so an app whose usable window sits + /// behind the panel walk's budget still resolves the window the historic route resolved. + func testSurfaceWindowWalkWithoutAPanelReadsPastThePanelWalkBudget() { + let unusable = (0..<9).map { _ in + DisplayWindowFixture(frame: .zero, displayID: 1) + } + let usable = DisplayWindowFixture( + frame: CGRect(x: 0, y: 0, width: 390, height: 844), displayID: 1) + let walk = resolveSurfaceWindow( + Self.application(unusable + [usable]), litPanel: Self.noPanelFixture) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue( + walk.window === usable, + "a no-panel request must not refuse a screen the historic uncapped walk resolved" ) - XCTAssertEqual(failure, .unresolvedWindow) - XCTAssertNil(screen) - XCTAssertEqual(displayID, 0) } + + func testSurfaceWindowWalkRefusesAnApplicationThatAnswersNoWindows() { + for panel in [Self.noPanelFixture, Self.duoInnerPanel] { + let walk = resolveSurfaceWindow( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), litPanel: panel) + XCTAssertEqual(walk.failure, .unresolvedWindow) + XCTAssertNil(walk.window) + } + } + + /// The full route a capture takes: the panel decides which window's `XCUIScreen` the capture uses, + /// and it is the screen object that window reported. + func testResolvedScreenFollowsTheLitPanelNotTheFirstWindow() { + let statusBar = Self.springBoardStatusBarWindow() + let home = Self.duoHomeScreenWindow() + let read = resolveScreen( + Self.application([statusBar, home]), litPanel: Self.duoInnerPanel) + XCTAssertTrue(read.succeeded) + XCTAssertEqual(read.failure, .none) + XCTAssertEqual(read.displayID, 3, "the capture must name the lit panel, not the display listed first") + XCTAssertTrue(read.screen === home.screen) + } + + /// A capture's screen read is the one consumer that needs the `XCUIScreen` object; a caller that + /// routes by display identity alone must be able to ask without one. + func testResolvedWindowScreenAnswersDisplayIDWithoutARequestedScreen() { + let window = Self.duoHomeScreenWindow() + var displayID: UInt = 0 + XCTAssertEqual( + RunnerReadResolvedWindowScreen(window, window.frame, nil, &displayID), + .none + ) + XCTAssertEqual(displayID, 3) + } + + /// The walk is bounded: reading more windows than the cap would widen the risk of asking a window + /// that has already vanished for anything (#2995). + func testWindowElementCopyIsCappedAtThePanelWalkBudget() { + let windows: [NSObject] = (0..<12).map { + DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 100, height: 100), displayID: UInt($0 + 1)) + } + let application = DisplayApplicationFixture(windows: DisplayWindowsFixture(windows: windows)) + var reported: UInt = 0 + XCTAssertEqual(RunnerCopyApplicationWindowElements(application, 8, &reported)?.count, 8) + XCTAssertEqual( + reported, 12, "the marker reports the list the query named, not the cap that trimmed it" + ) + XCTAssertNotNil(RunnerCopyApplicationWindowElements(application, 20, &reported)) + XCTAssertNil( + RunnerCopyApplicationWindowElements( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), 8, nil), + "a windows query that raises answers no windows, and does not raise here" + ) + } + /// The sole-screen rule's decision, at the seam the production probe feeds: the values the /// AXRuntime probe reads in, never a device list. The windowed/unknown rows below fail without /// the fills-screen proof, on any family and on the macOS host lane, and each keeps its exact diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift index 5ef97086db..02864f4d7b 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift @@ -86,7 +86,11 @@ public struct RawAXNode: Equatable { self.actions = actions } - func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode { + /// The node with a new box and a new actionability verdict, every other fact kept. Published because + /// the runner's surface rule rewrites a root's box against the box it resolved, the same way the + /// coordinate-space pass rewrites frames in the app's orientation space (#2661); both need the + /// keyboard-ownership and custom-action fields to survive the copy. + public func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode { var updated = self updated.rect = rect updated.hittable = hittable diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 66616ff5ba..446bd0283a 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -247,6 +247,12 @@ "trigger": "structured runner reply with code APP_SCREEN_CAPTURE_UNRENDERABLE: a refusal that ran nothing", "dispatched": "no" }, + { + "id": "ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL", + "producer": "ios-runner", + "trigger": "structured runner reply with code APP_SCREEN_OFF_LIT_PANEL: a refusal that ran nothing", + "dispatched": "no" + }, { "id": "ios-runner.reply.ELEMENT_NOT_FOUND", "producer": "ios-runner", diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index 90a22d0ac9..ce15c0d5e3 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -7,6 +7,11 @@ {"name": "ios-device.recording-start.fps", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "recordStart", "outPath": "agent-device-recording-1700000000000.mp4", "fps": 30, "appBundleId": "com.example.app"}}, {"name": "ios-device.recording-stop.no-app", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "recordStop"}}, {"name": "ios-device.recording-stop.session", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "recordStop", "appBundleId": "com.example.app"}}, + {"name": "ios-foldable.alert.get", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "alert", "action": "get", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, + {"name": "ios-foldable.interactions-tap.synthesized", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-keyboard-dismiss.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-read-text.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-snapshot.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, {"name": "ios-simulator.alert.accept", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "alert", "action": "accept", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, {"name": "ios-simulator.alert.dismiss", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "alert", "action": "dismiss", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, {"name": "ios-simulator.alert.get", "producer": "packages/platform-apple/src/__tests__/index-runner-requests.test.ts", "request": {"command": "alert", "action": "get", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 0e744dc228..2b8efd2134 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -364,7 +364,7 @@ Rejected alternatives: `devicectl device orientation set` reports success and has no effect on a simulator that `simctl boot` started. `orientation` therefore rotates a multi-panel simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). The control has no upside-down value, so `portrait-upside-down` is refused with `unsupported-foldable-orientation`. -Routing uses the `fold` admission and the panel table. A resolved panel count is a hardware fact, so the daemon probes each simulator once. +Routing uses the `fold` admission and the panel table. A resolved panel count is a hardware fact, so the daemon probes each simulator once. A rotation asks that table itself and never answers from the short window a surface probe opens after a table that would not reply: the two routes differ in correctness, because on a multi-panel simulator the historic runner route has no effect and its readback repeats the request, so answering from that window would report a rotation the device never performed. The verdict comes from `devicectl device orientation get`, a process that never sent the HID event. CoreDevice answers each query with the value that it sampled at the previous query, so the first read after a dispatch is discarded. @@ -708,3 +708,32 @@ It is refused as `single-panel-device`, the same typed reason as a single-panel Keyframes and panel geometry remain Apple-only. The emulator console supports only fixed postures. Android reports the app window on the next snapshot instead of a panel. + +## Amendment: the lit panel chooses the window, not the window list (2026-10-31) + +Resolving a screen had two authorities. Capture asked CoreDevice which panel is lit; the runner took the application's first window. The first changes when the device folds and the second does not, so on a foldable the two disagreed. + +SpringBoard keeps a window per panel. On a Duo home screen, its first reported window is the 226x155.3 status-bar window on the outer panel, whichever panel is lit. Four paths booked against that window: the interaction viewport, the synthesized reference frame, the snapshot viewport, and the capture's display read. Every home-screen gesture therefore entered a panel that might be dark, and the snapshot viewport was 223x317 on a 951x669 screen. One resolver owns all four paths: `RunnerResolveSurfaceWindow` (`RunnerXCTestEventBridge.m`). It reads each window's `exists`, frame, and `screen.displayID` once, and applies the rule in `RunnerPanelSurface.m`. + +Which panel is lit is a host fact. The runner has no hinge to read, and geometry cannot settle it, because the dark panel's window has a full-size frame at the origin. So the request carries it. The host reads the panel table per request (`resolveRunnerRequestLitPanel`) and names the panel in `litPanel` (`displayId`, `widthPt`, `heightPt`) only for a device whose table reports more than one integrated panel, and only on a command that resolves a screen. Selection requires a named display. A frame equal to the panel's box, or that box transposed, is the panel's surface; otherwise the largest window on that display is the surface. Both sizes are the panel's native points, never rotated, because the system presents the inner panel turned and its interaction space swaps the side lengths. + +A request naming no panel takes the historic read: the first window with a usable frame, and no display identity is asked for at all. Two kinds of device send that request, and only one of them is single-panel. A single-panel device is the common case. A multi-panel device also sends it whenever the panel table could not be read, or decided nothing, because a surface fact has no pressure to name some panel and guess: a capture must name a display or refuse, a surface booking can fall back. So an absent `litPanel` is never evidence that a device has one panel. This is why the fact travels per request rather than as runner state. An iPhone pays nothing it did not pay before, and a fold cannot be answered from a stale panel, because the next request asks again. + +Rejected alternatives: + +- **The largest window: rejected.** It guesses power from size. It picks the 951x669 inner window while the device is closed, the same class of error as the implicit capture default. +- **Panel points derived in the runner: rejected.** The host already measures panel power, and the runner's own screen read can name the dark panel. +- **Threading one optional panel through every Swift signature: rejected.** Forty call sites, four of which have no `Command` in scope, for one answer per command. The runner holds the panel of the command in flight in one lock-guarded slot (`RunnerRequestPanel.swift`), which is the only place that answers the question. An operation that outlives its request takes the panel as an argument instead of reading the slot per sample: a recording books every frame on the panel its `recordStart` named, because the writer is sized from the bootstrap frame, and a keyboard settle loop compares samples taken on one panel. Reading the slot again from such a loop would follow whatever command replaced the request. +- **Choosing a panel when the table decides nothing: rejected.** Capture may fall back to the `primary` panel because it must name some display or refuse; a surface fact has no such pressure, and naming the wrong panel would refuse the screen the user is looking at. Such a request names no panel. +- **A `--screen outer|inner` flag for gestures: rejected again, for the same reason as for capture.** The panel is measured. It is not asked for. + +Consequences: + +- A multi-panel device whose every window is on the dark panel refuses instead of booking there, with the typed code `APP_SCREEN_OFF_LIT_PANEL`. The system surface is asked first, as for an unresolved window, so only a device with nothing on the lit panel refuses. +- `runner.log` names the decision: `AGENT_DEVICE_RUNNER_PANEL_SURFACE lit=(x) reported= placed= unplaceable= selection= frame=(…)` reports the panel a window was booked against and the frame that won. `reported` is what the window query named before the cap, `placed` is how many of those the runtime would say a display for. `selection` is the rule's answer: 0 `litPanelWindow`, 1 `litDisplayWindow`, 2 `offLitPanel`, 3 `noWindow`. A single-panel device takes the historic walk and prints none of this. +- The two synthesized-input markers say what the booked surface produced. `AGENT_DEVICE_RUNNER_SYNTHESIZED_DISPATCH kind= point=(x,y) reference=(, x) orientation=` names no display: `reference` is the booked surface's own box, so on a foldable it must be the lit panel's presented box and `reference=(452,0 226x155.3)` is the status-bar window this fix removes. Compare that box with `PANEL_SURFACE frame=`. `AGENT_DEVICE_RUNNER_SYNTHESIZED_RECORD name= displayID= interfaceOrientation=` names the track's display: the display ID of the window the gesture was booked on, so it must equal `PANEL_SURFACE lit=`. +- A Duo's SpringBoard names 40-43 windows while the walk books at most eight, so the marker reads `reported=40 placed=8`. The lit panel's full-screen window sits inside the first eight in every pose observed. A refusal that claims something about the whole list is refused instead when the walk could not read the whole list: `offLitPanel` becomes "could not decide" when a window was unplaceable or the cap cut the list short, and so does `litDisplayWindow`, whose largest survivor could be a corner strip while the panel's real surface went unread. `selection=3` means no window the walk read could be placed on a display, which is always `placed=0`: it says nothing about windows the cap left unread. +- A regular capture of a multi-panel device reports what the panel fact said about its tree: `AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE republishedRoot= droppedSurfaces= nodes=->`, and `…_TREE_GRAFTED frame=(…) nodes=` when the application's own subtree carried no window for the lit panel. `…_TREE_SKIPPED reason=` names why a graft did not happen, and `graft-failed` carries the capture code that refused it. A skipped graft leaves the tree as it was walked, with the nodes and dedup keys its abandoned read produced discarded; it never fails the tier and never marks it exhausted. +- Main-queue work abandoned by the execution watchdog can read the panel of the command that replaced it. The log line makes such a resolution visible. A single-panel device reads no panel, so it has nothing to repair. +- Recording, keyboard settling, and a read by point name their panel too: `recordStart`, `keyboardDismiss`, and `readText` resolve a screen (each samples it or places a point in it), so they carry the fact. Without it a clip of a foldable's dark panel is a black video whose bootstrap succeeded. +- The wire field is a two-declaration contract like the capture metadata: `AppleLitPanel` in `@agent-device/contracts/interactor-types` and `RunnerLitPanelFact` in `RunnerTests+Models.swift`. diff --git a/docs/agents/device-foldable.md b/docs/agents/device-foldable.md index 0719bb2e99..5bf57f56a6 100644 --- a/docs/agents/device-foldable.md +++ b/docs/agents/device-foldable.md @@ -1,30 +1,25 @@ # Foldable Apple verification Before you change capture behavior on a multi-panel device, read ADR 0025. -To list the panels, run `xcrun devicectl device info displays --device `. -In a scoped set, run `xcrun simctl --set spawn ~/.agent-device/fold-helper//fold-helper panels`. - -- Panels: the list shows each integrated panel with `backlightState`. Only the lit panel permits capture. - A capture of the dark panel exits 0 and writes an all-black PNG. -- Input routing: after each pose change, verify a fresh control. - A successful synthesis acknowledgement does not prove a hit. - Examine the runner and simulator `testmanagerd`/BackBoard logs for display identity and delivery. - The resolved app window owns gesture coordinates and its target screen. -- Pose: `agent-device fold closed|half-open|open` sends private HID inside the simulator. - The same helper reads the hinge back in `hinge` mode. - After a fold, take another snapshot. Folds invalidate refs and coordinates. - Verify locally with Device Hub stopped and the simulator booted through `simctl boot`. - Test these items in the default set and with `--ios-simulator-device-set`: - - All three poses. - - Capture of the active panel. - - An app interaction. - Until GHA supports the runtime, Duo coverage remains local. - After a fresh boot, a closed hinge can keep the inner panel lit for more than 30 seconds. - The lit panel is evidence only. It cannot separate `half-open` from `open`. -- Touch overlays export at the captured track size. - The compositor checks its output size and non-black content against the raw capture. - If that check fails, `record stop` keeps the raw file. It reports `overlayWarning`. - `record start --hide-touches` is only for the fastest raw capture. +- Panels: `xcrun devicectl device info displays --device ` shows each integrated panel with + `backlightState`; only the lit panel permits capture. In a scoped set, `simctl --set spawn + ~/.agent-device/fold-helper//fold-helper panels` reads it. +- Input routing: after each pose change, verify a fresh control. A successful synthesis + acknowledgement does not prove a hit. Examine the runner and simulator `testmanagerd`/BackBoard + logs for display identity and delivery. +- Surface identity: verify the window the runner booked, not only the panel the host named; it owns + gesture coordinates and its target screen. Its `runner.log` markers are listed in ADR 0025 and + each must name the lit panel, except `SYNTHESIZED_DISPATCH reference=(…)`, which names no display: + compare it with the booked surface. A 226x155.3 reference frame is the status-bar window: the bug. +- Pose: `agent-device fold closed|half-open|open` sends private HID inside the simulator; the same + helper reads the hinge back in `hinge` mode. After a fold, take another snapshot: folds invalidate + refs and coordinates. Verify locally, Device Hub stopped and the simulator booted through `simctl + boot`, in the default set and with `--ios-simulator-device-set`: all three poses, capture of the + active panel, and an app interaction. Duo coverage stays local until GHA supports the runtime. + After a fresh boot a closed hinge can keep the inner panel lit for over 30 seconds, so lit-panel + state never settles a pose. +- Touch overlays export at the captured track size. A failed check keeps the raw file and `record + stop` reports `overlayWarning`; `--hide-touches` is the raw-capture fallback. - To launch on iOS 27.0/27.1, iOS 27 SDK apps need UIScene lifecycle. - The iOS 27.1 runtime (24A94425) accepts only the iPhone Duo device type. For a non-foldable control, use another runtime. diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index 3fcb985970..60a123b8db 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -178,6 +178,7 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_CAPTURE_UNRENDERABLE': () => replyFailure('APP_SCREEN_CAPTURE_UNRENDERABLE'), + 'ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL': () => replyFailure('APP_SCREEN_OFF_LIT_PANEL'), 'ios-runner.reply.ELEMENT_NOT_FOUND': () => replyFailure('ELEMENT_NOT_FOUND'), 'ios-runner.reply.ELEMENT_OFFSCREEN': () => replyFailure('ELEMENT_OFFSCREEN'), 'ios-runner.reply.AMBIGUOUS_MATCH': () => replyFailure('AMBIGUOUS_MATCH'), diff --git a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts index 7c881cc936..65e4fd1853 100644 --- a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts +++ b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts @@ -426,6 +426,7 @@ test('runner_endpoint_probe_exhausted is decided by the typed reason, not the me test('a refused screen capture keeps the runner reason and stays off the wire code', () => { assert.deepEqual([...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].sort(), [ 'APP_SCREEN_CAPTURE_UNRENDERABLE', + 'APP_SCREEN_OFF_LIT_PANEL', 'APP_SCREEN_UNRESOLVED', 'APP_SCREEN_WINDOW_UNRESOLVED', ]); diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index cf544dcfc5..c77a021966 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -26,6 +26,7 @@ export { SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, buildRunnerSequenceCommand, parseRunnerSequenceResult, + resolveRunnerRequestSignal, type RunnerCommand, } from './runner-contract.ts'; export { isRunnerMainThreadTimeoutError } from './runner-error-classification.ts'; diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 24c7bfa56c..cf7dafe38f 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -15,6 +15,7 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot'; import type { ClickButton } from '@agent-device/contracts/click-button'; import type { + AppleLitPanel, CoordinateTapTextInputEvidence, ElementSelectorKey, } from '@agent-device/contracts/interactor-types'; @@ -163,6 +164,13 @@ export type RunnerCommand = { fullscreen?: boolean; inlineScreenshot?: boolean; synthesized?: boolean; + /** + * The panel the host reports as lit, sent only for a device whose panel table reports more than one + * integrated panel and only on a command that resolves a screen. Absent means "single-panel device" + * or "the panel table decided nothing": the runner then keeps its historic first-window surface + * route rather than guessing a panel (ADR 0025). + */ + litPanel?: AppleLitPanel; steps?: RunnerSequenceStep[]; }; @@ -270,12 +278,19 @@ export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCC const RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_WINDOW_UNRESOLVED'; const RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_UNRESOLVED'; const RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE = 'APP_SCREEN_CAPTURE_UNRENDERABLE'; +const RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE = 'APP_SCREEN_OFF_LIT_PANEL'; -/** Every runner code meaning "this capture did not happen", covering the display and the image. */ +/** + * Every runner code meaning "this capture did not happen", covering the display, the image, and the + * panel. The last one is a foldable's own refusal: the request named a lit panel and neither the + * target app nor the system surface had a window on it. The capture does not fall back to the dark + * panel, which would answer with a valid all-black PNG, so this code is what the host sees. + */ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = new Set([ RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE, + RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE, ]); /** diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 0de826f22b..475486f370 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -107,6 +107,24 @@ export type PressPointOptions = Readonly<{ coordinateTapTextInputEvidence?: CoordinateTapTextInputEvidence; }>; +/** + * The panel the host reports as lit, in the units the panel table reports it: which display is + * powered, and that panel's own native box in points (`nativeSize` divided by `pointScale`). + * + * A runner on a multi-panel device cannot decide which panel is powered — it has no hinge to read, + * and geometry decides nothing because the dark panel's windows have perfectly good frames — so a + * request that resolves a screen names the panel here and the runner books its surface against it + * (ADR 0025). A request that carries no panel is a single-panel device's request. + * + * The size is the panel's native box, deliberately not the box an app is showing on it: the system + * presents the iPhone Duo inner panel turned, so its interaction space swaps the side lengths. + */ +export type AppleLitPanel = Readonly<{ + displayId: number; + widthPt: number; + heightPt: number; +}>; + /** * Legacy success text retained for compatibility when the XCTest runner used * the Maestro non-hittable coordinate fallback. Usage itself is carried by diff --git a/packages/platform-apple/src/__tests__/display-fixtures.ts b/packages/platform-apple/src/__tests__/display-fixtures.ts index 75135bf6f4..33a3f340f5 100644 --- a/packages/platform-apple/src/__tests__/display-fixtures.ts +++ b/packages/platform-apple/src/__tests__/display-fixtures.ts @@ -41,6 +41,53 @@ export const IPHONE_DUO_CLOSED = { }, }; +/** + * The same Duo with the hinge open: the inner panel is lit and the outer one is dark. The `rot90` + * the inner panel reports in a portrait hold is what makes its interaction space 951x669 for a + * 669x951 native panel (ADR 0025). + */ +export const IPHONE_DUO_OPEN = { + result: { + displays: [ + { + active: false, + backlightState: 'off', + bounds: [ + [0, 0], + [1398, 2034], + ], + currentOrientation: 'rot0', + displayId: 1, + name: 'LCD', + nativeOrientation: 'rot0', + nativeSize: [1398, 2034], + pointScale: 3, + primary: true, + type: { integrated: {} }, + uniqueId: '2C570360-1D02-40A1-BFBF-11197BB1159C', + }, + { + active: true, + backlightState: 'activeOn', + bounds: [ + [0, 0], + [2007, 2853], + ], + currentOrientation: 'rot90', + displayId: 3, + name: 'LCD-1', + nativeOrientation: 'rot0', + nativeSize: [2007, 2853], + pointScale: 3, + primary: false, + type: { integrated: {} }, + uniqueId: '162CC14D-0995-4D13-9C83-B609458243C8', + }, + ], + orientation: { currentDeviceOrientation: 'portrait' }, + }, +}; + /** Measured on a real iPhone 17: CoreDevice omits `active` entirely for one panel. */ export const IPHONE_17_SINGLE_PANEL = { result: { diff --git a/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts b/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts new file mode 100644 index 0000000000..6e3994e6e3 --- /dev/null +++ b/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts @@ -0,0 +1,86 @@ +import { + assertProducedRunnerRequests, + bindTestCommandHost, + IOS_SIMULATOR, +} from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; +import type { Interactor } from '@agent-device/contracts/interactor-types'; +import type { RunnerCommand } from '@agent-device/apple-runner'; +import type { DeviceInfo } from '@agent-device/kernel/device'; + +vi.mock('../foldable/fold-helper-cache.ts', () => ({ + ensureFoldHelperBinary: vi.fn(async () => ({ path: '/cache/fold-helper' })), +})); +vi.mock('../core/devicectl.ts', async (importOriginal) => { + const actual = await importOriginal(); + // The mock factory is hoisted above this file's imports, so the panel table it answers with is + // read through the dynamic import the mock itself controls. + const { IPHONE_DUO_OPEN } = await import('./display-fixtures.ts'); + return { + ...actual, + runIosDevicectlJsonRequest: vi.fn(async () => ({ + ok: true as const, + payload: IPHONE_DUO_OPEN, + })), + }; +}); + +import { createAppleInteractor } from '../interactor.ts'; +import { recordingRunnerProvider, type RecordedRunnerCall } from './recording-runner-provider.ts'; + +bindTestCommandHost(); + +// The multi-panel half of the Apple runner request pin: the same production entry points as +// ./index-runner-requests.test.ts, driven for a device whose +// panel table reports two integrated panels, so the wire shape that carries the lit panel is pinned +// too. That file's entries are the single-panel requests, and they must stay panel-free. + +const APP = 'com.example.app'; + +/** A device the `fold` fact admits: a foldable iPhone simulator in the default device set. */ +const DUO_SIMULATOR: DeviceInfo = { ...IOS_SIMULATOR, name: 'iPhone Duo' }; + +type InteractorDrive = readonly [DeviceInfo, (interactor: Interactor) => Promise]; + +const INTERACTOR_SITES: Record = { + 'ios-foldable.interactions-tap.synthesized': [DUO_SIMULATOR, (i) => i.tap(10, 20)], + 'ios-foldable.interactor-snapshot.lit-panel': [ + DUO_SIMULATOR, + (i) => i.snapshot({ appBundleId: APP, interactiveOnly: true }), + ], + // `keyboardDismiss` settles by fingerprinting screenshots, so it resolves a screen even though it + // reads nothing: a clip or a settle loop taken from the dark panel reports success on nothing. + 'ios-foldable.interactor-keyboard-dismiss.lit-panel': [ + DUO_SIMULATOR, + (i) => i.keyboardDismiss!(), + ], + // `readText` resolves the surface window to place the point it was handed, so the point means the + // same screen on a foldable that the ref it came from was measured in. + 'ios-foldable.interactor-read-text.lit-panel': [ + DUO_SIMULATOR, + (i) => i.readTextAtPoint!({ x: 10, y: 20 }, { appBundleId: APP }), + ], + // A request that resolves no screen carries no panel: alert, lifecycle, readiness and clipboard + // traffic must stay byte-for-byte the request it was. + 'ios-foldable.alert.get': [DUO_SIMULATOR, (i) => i.readAlert!({ appBundleId: APP })], +}; + +test('a multi-panel device pins the runner request that carries its lit panel', async () => { + const captured: Array = []; + for (const [name, [device, drive]] of Object.entries(INTERACTOR_SITES)) { + const calls: RecordedRunnerCall[] = []; + await drive( + createAppleInteractor( + device, + { appBundleId: APP }, + { env: process.env, cwd: process.cwd() }, + recordingRunnerProvider(calls, { + alert: { message: 'Allow', items: ['Allow'] }, + }), + ), + ); + if (calls.length !== 1) throw new Error(`${name} sent ${calls.length} runner requests`); + captured.push([name, calls[0]!.command]); + } + assertProducedRunnerRequests(import.meta.filename, captured); +}); diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index b3db960c66..eee9d33fd8 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,4 +1,5 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; +import type { AppleLitPanel } from '@agent-device/contracts/interactor-types'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; @@ -409,6 +410,36 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi }); } +/** + * The panel to name in a runner request, from a panel table read. + * + * Absent unless panel power itself decided one panel. A capture may still name the `primary` panel + * when panel power decides nothing (ADR 0025), because a capture has to name some display or refuse; + * a surface fact has no such pressure, and naming a panel the table did not settle would have the + * runner refuse the panel the user is actually looking at. The `primary` flag plays no part here: a + * payload can decide which panel is lit and still omit that flag. + * + * A panel the runtime names display 0 cannot be sent: 0 is the wire's "no panel named" identity, so + * such a display would look panel-carrying on the request and behave single-panel in the runner. + */ +export function appleLitPanel(inventory: AppleDisplayInventory): AppleLitPanel | undefined { + // `activeDisplay` is set only by a table that named a panel to capture, and an incomplete table + // names none: a panel the parser dropped may be the lit one, and naming the survivor would send + // every surface request to a display that can be dark. + if (!inventory.multiScreen || inventory.unresolved || !inventory.activeDisplay) { + return undefined; + } + const litPanels = inventory.displays.filter((display) => display.power === 'lit'); + const panel = litPanels.length === 1 ? litPanels[0]! : undefined; + if (!panel || panel.displayId <= 0) return undefined; + if (!panel.pointScale || !Number.isFinite(panel.pointScale)) return undefined; + return { + displayId: panel.displayId, + widthPt: panel.widthPx / panel.pointScale, + heightPt: panel.heightPx / panel.pointScale, + }; +} + /** * Resolves the display a capture must target. * diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 01673064f1..9afbfacf2d 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -1,16 +1,109 @@ -import { bindAppleRunnerHost } from '@agent-device/apple-runner'; +import { + bindAppleRunnerHost, + resolveRunnerRequestSignal, + runAppleRunnerCommand as runHostBoundAppleRunnerCommand, + type AppleRunnerCommandOptions, + type RunnerCommand, +} from '@agent-device/apple-runner'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { AppleLitPanel } from '@agent-device/contracts/interactor-types'; import { appleRunnerHost } from './runner-host.ts'; /** - * Composition root for the Apple runner operations: the only module that binds - * the real host capabilities from `runner-host.ts`, and the module daemon, - * platform, and CLI consumers import the host-bound operations from, so every - * such consumer evaluates after the binding. Runner modules must not reach the - * host while they evaluate: they load before this module binds it. Types and + * Composition root for the Apple runner operations: the only module that binds the real host + * capabilities from `runner-host.ts`, and the module daemon, platform, and CLI consumers import the + * host-bound operations from, so every such consumer evaluates after the binding. Runner modules + * must not reach the host while they evaluate: they load before this module binds it. Types and * host-free helpers come from the package façade directly. + * + * Because every Apple runner request leaves the platform through here, this is also where a request + * that resolves a screen gains the one fact a runner cannot read for itself: which panel is lit. */ bindAppleRunnerHost(appleRunnerHost); +/** + * Runner commands whose execution resolves a screen — a window to book a gesture against, a viewport + * to place a capture or a snapshot in, or the display to capture. Each of them used the application's + * first window, which on a foldable home screen is SpringBoard's status-bar window on a panel that + * may be dark, so each carries the panel the host read as lit for the request. + * + * A command outside this set resolves no surface and carries no panel: readiness, lifecycle, + * clipboard and element- or selector-targeted traffic stays byte-for-byte the request it was. + */ +const RUNNER_SURFACE_RESOLVING_COMMANDS: ReadonlySet = new Set([ + 'tap', + 'longPress', + 'drag', + 'type', + 'swipe', + 'scroll', + 'gesture', + 'sequence', + 'backInApp', + 'backSystem', + 'appSwitcher', + 'gestureViewport', + 'snapshot', + 'screenshot', + // Both sample the screen from the runner: `recordStart` sizes and pumps its frames, and + // `keyboardDismiss` fingerprints screenshots to settle. Without the panel they would read the + // dark panel and report success — an all-black clip, or a keyboard that "settled" on a screen + // that never changed. `readText` resolves the surface window to place the point it was handed. + 'recordStart', + 'keyboardDismiss', + 'readText', +]); + +/** + * The request as the wire carries it: a surface-resolving command for a multi-panel device names the + * lit panel, and anything else is the command it was. A command that already names a panel is left + * alone, so a caller that resolved one for its whole sequence is not overruled mid-flight. + */ +function withRunnerRequestLitPanel( + command: RunnerCommand, + litPanel: AppleLitPanel | undefined, +): RunnerCommand { + if (!litPanel || command.litPanel || !RUNNER_SURFACE_RESOLVING_COMMANDS.has(command.command)) { + return command; + } + return { ...command, litPanel }; +} + +/** + * Sends `command` to the runner, naming the lit panel when the device presents more than one panel and + * the command resolves a screen. A single-panel device, an unreadable panel table, and a multi-panel + * device whose table decides nothing all send no panel, and the runner keeps its historic surface + * route. + * + * The panel owner is loaded on first use, not at module load: this module sits in the eager closure + * of the Apple façades, and the panel probe is host tooling only a multi-panel request needs (ADR + * 0019). + */ +export async function runAppleRunnerCommand( + device: DeviceInfo, + command: RunnerCommand, + options: AppleRunnerCommandOptions = {}, +): Promise> { + if (!RUNNER_SURFACE_RESOLVING_COMMANDS.has(command.command)) { + return await runHostBoundAppleRunnerCommand(device, command, options); + } + // A command that already names a panel is not re-probed: a caller that resolved one for its whole + // sequence keeps the answer it resolved, and the probe is not paid twice for one request. + if (command.litPanel) { + return await runHostBoundAppleRunnerCommand(device, command, options); + } + const { resolveRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); + // The request's own cancellation covers the probe too: a cancelled surface command must not wait + // out the panel table's timeout before the transport sees the cancellation. + const signal = resolveRunnerRequestSignal(options); + const litPanel = await resolveRunnerRequestLitPanel(device, { signal }); + return await runHostBoundAppleRunnerCommand( + device, + withRunnerRequestLitPanel(command, litPanel), + options, + ); +} + export { applyXctestRunnerAppIconFromDerivedPath, cleanupRunnerLeasesForOwner, @@ -38,9 +131,10 @@ export { resolveRunnerSandboxBuildArgs, resolveRunnerSigningBuildSettings, runApplePressSeries, - runAppleRunnerCommand, stopAllIosRunnerSessions, stopIosRunnerSession, verifyLeaseRunnerPidIdentity, writeRunnerCacheMetadataForArtifacts, } from '@agent-device/apple-runner'; + +export type { AppleRunnerCommandOptions, RunnerCommand }; diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 0d695b074e..eaa8991150 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -15,6 +15,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/tool-provider.ts'; import { createAppleInteractor } from '../interactor.ts'; +import { isAppleFoldableSimulator } from './orientation.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; import { IPHONE_17_SINGLE_PANEL, IPHONE_DUO_CLOSED } from '../__tests__/display-fixtures.ts'; @@ -200,6 +201,64 @@ test('a foldable rotation that CoreDevice never reports is refused as unverified expect(sim.helpers).toHaveLength(1); }); +/** + * A cancelled panel-table read is the caller going away, not a toolchain that will not answer, so it + * must not spend the retry window the `unresolved` answer opens: the next request asks again. + */ +test('a cancelled panel-table probe does not keep the device single-panel', async () => { + const device = simulator(); + let reads = 0; + const provider = createLocalAppleToolProvider({ + runCommand: async (_command, args, options) => { + expect(args.slice(0, 4)).toEqual(['devicectl', 'device', 'info', 'displays']); + if (options?.signal?.aborted) throw new Error('devicectl aborted by the caller'); + reads += 1; + fs.writeFileSync(args.at(-1)!, JSON.stringify({ result: { displays: DUO_DISPLAYS } })); + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }); + const ask = (signal?: AbortSignal) => + withAppleToolProvider(provider, () => isAppleFoldableSimulator(device, signal)); + + const cancelled = new AbortController(); + cancelled.abort(); + await expect(ask(cancelled.signal)).resolves.toBe(false); + // The request that follows is free to ask again, and the answer it gets is the device's own. + await expect(ask()).resolves.toBe(true); + // One read: the cancelled probe never took the retry window the `unresolved` answer opens. + expect(reads).toBe(1); +}); + +/** + * A rotation is decided by the panel table, never by the window a surface probe opened: the historic + * runner route has no effect on a foldable and its readback repeats the request, so answering from + * that window would report a rotation the device never performed. + */ +test('a rotation is not routed by a pending panel-table retry', async () => { + const device = simulator(); + let tableFailures = 0; + const sim = fakeSimulator(device, DUO_DISPLAYS); + // A surface probe finds a panel table that will not answer, which opens the retry window. + const failing = createLocalAppleToolProvider({ + runCommand: async (_command, args) => { + if (args.slice(0, 4).join(' ') === 'devicectl device info displays') { + tableFailures += 1; + throw new AppError('COMMAND_FAILED', 'devicectl does not report displays', { + timeoutMs: 2_000, + }); + } + throw new Error(`unexpected tool call: ${args.join(' ')}`); + }, + }); + await withAppleToolProvider(failing, () => isAppleFoldableSimulator(device, undefined)); + expect(tableFailures).toBe(1); + + // The rotation that follows pays its own read, gets the device's real answer, and folds through HID. + await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); + expect(sim.helpers).toEqual([['rotate', 'landscape-left']]); + expect(mockRunner).not.toHaveBeenCalled(); +}); + test('a stale CoreDevice sample cannot verify a rotation that did not happen', async () => { const sim = fakeSimulator(simulator(), DUO_DISPLAYS, { followsHid: false, diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index f9c78cc97d..5d01d8d251 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -2,6 +2,7 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { createTtlMemo } from '@agent-device/kernel/ttl-memo'; import { IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS, @@ -25,20 +26,60 @@ const ORIENTATION_UNVERIFIED_HINT = /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); +// A panel table the runtime would not answer is retried, but not once per command: every runner +// request that resolves a surface asks this question, and an unbounded retry pays the probe cost on +// each of them. A few seconds absorbs one command burst; it is short enough that the next request +// minutes after a transient CoreDevice failure asks the panel table again. `apple_panel_table_retry_ +// pending` names the window during which a foldable is treated as single-panel. +const PANEL_TABLE_RETRY_MS = 5_000; +const panelTableRetriesAt = createTtlMemo({ + ttlMs: PANEL_TABLE_RETRY_MS, + scheduleExpiry: true, +}); + /** * Whether a rotation must go through guest HID: a simulator the `fold` fact admits that has more * than one integrated panel. XCUIDevice rotation has no effect there, and its readback repeats * the requested value. + * + * `askPanelTableAgain` skips the retry window and reads the table now. A caller that asks once per + * command, such as a rotation, takes that route: a surface probe is paid on every runner request, so + * only it may answer from the window, and holding a rotation there would send a foldable through the + * route whose readback repeats the request and reports a rotation that never happened. */ export async function isAppleFoldableSimulator( device: DeviceInfo, signal: AbortSignal | undefined, + options: { askPanelTableAgain?: boolean } = {}, ): Promise { if (!appleFoldableFacts(device).setFoldPose.available) return false; const known = multiPanelByDevice.get(device.id); if (known !== undefined) return known; + if (!options.askPanelTableAgain && panelTableRetriesAt.get(device.id)) { + // The caller cannot tell this answer from a device it proved single-panel, so the reason has to + // be in the log: a runner request that follows takes its historic first-window surface route. + emitDiagnostic({ + level: 'debug', + phase: 'apple_panel_table_retry_pending', + data: { + deviceId: device.id, + retryInMs: PANEL_TABLE_RETRY_MS, + hint: 'The panel table would not answer, so until it is asked again this device is treated as single-panel: a runner request carries no lit panel.', + }, + }); + return false; + } const inventory = await queryAppleDisplayInventory(device, { signal }); - if (inventory.unresolved) return false; + if (inventory.unresolved) { + // A probe the caller cancelled is not evidence that the panel table will not answer, so it must + // not spend the retry window: the next request asks again instead of inheriting this one's + // single-panel fallback for the whole window. The caller's own cancellation surfaces downstream, + // where the request it belongs to reaches the transport. + if (!signal?.aborted) { + panelTableRetriesAt.set(device.id, true); + } + return false; + } multiPanelByDevice.set(device.id, inventory.multiScreen); return inventory.multiScreen; } diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts new file mode 100644 index 0000000000..b99df12425 --- /dev/null +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict'; +import { beforeEach, test, vi } from 'vitest'; +import { bindTestCommandHost } from '@agent-device/test-kit'; +import type { DeviceInfo } from '@agent-device/kernel/device'; + +vi.mock('../foldable/fold-helper-cache.ts', () => ({ + ensureFoldHelperBinary: vi.fn(async () => ({ path: '/cache/fold-helper' })), +})); +vi.mock('../core/devicectl.ts', () => ({ + runIosDevicectlJsonRequest: vi.fn(async () => ({ + ok: true as const, + payload: { result: { displays: [] } }, + })), +})); + +import { runIosDevicectlJsonRequest } from '../core/devicectl.ts'; +import { resolveRunnerRequestLitPanel } from './runner-panel.ts'; +import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; +import { + IPHONE_17_SINGLE_PANEL, + IPHONE_DUO_CLOSED, + IPHONE_DUO_OPEN, +} from '../__tests__/display-fixtures.ts'; + +bindTestCommandHost(); + +const mockPanelTable = vi.mocked(runIosDevicectlJsonRequest); + +beforeEach(() => { + mockPanelTable.mockReset(); + mockPanelTable.mockResolvedValue({ ok: true as const, payload: { result: { displays: [] } } }); +}); + +/** + * A fresh simulator id per case: the panel count is cached per simulator for the life of the + * process, and that cache is one of the facts under test. + */ +let simulatorCount = 0; +function simulator(overrides: Partial = {}): DeviceInfo { + simulatorCount += 1; + return { ...IOS_SIMULATOR, id: `${IOS_SIMULATOR.id}-${simulatorCount}`, ...overrides }; +} + +function answering(payload: unknown): void { + mockPanelTable.mockImplementation(async () => ({ ok: true as const, payload })); +} + +test('a multi-panel request names the lit panel in the units the runner compares a window against', async () => { + answering(IPHONE_DUO_CLOSED); + // A closed Duo lights its outer panel: 1398x2034 px at pointScale 3, which is the 466x678 the + // runner finds as that panel's SpringBoard window. + assert.deepEqual(await resolveRunnerRequestLitPanel(simulator(), {}), { + displayId: 1, + widthPt: 466, + heightPt: 678, + }); +}); + +test('the panel is read again for the next request, so a fold moves the surface', async () => { + const device = simulator(); + answering(IPHONE_DUO_CLOSED); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); + // The first request also paid for the panel COUNT probe; every later one pays only for the read + // that answers which panel is lit now. + const readsAfterFirst = mockPanelTable.mock.calls.length; + answering(IPHONE_DUO_OPEN); + assert.equal( + (await resolveRunnerRequestLitPanel(device, {}))?.displayId, + 3, + 'an opened device must be asked again, not answered from the panel that just went dark', + ); + assert.equal(mockPanelTable.mock.calls.length - readsAfterFirst, 1); +}); + +test('a device with one panel sends no panel and is never asked again', async () => { + const device = simulator(); + answering(IPHONE_17_SINGLE_PANEL); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + const reads = mockPanelTable.mock.calls.length; + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(mockPanelTable.mock.calls.length, reads, 'one panel is hardware, probed once'); +}); + +test('a device whose panel table cannot be read sends no panel', async () => { + mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); +}); + +test('a multi-panel device whose panel power decides nothing sends no panel', async () => { + // Both panels lit: the panel table settles no single panel. `activeDisplay` still names the + // primary panel for a capture, which has to name some display or refuse, but a surface fact is + // not forced to guess, and naming the wrong one would have the runner refuse the panel the user + // is looking at. + answering({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((display) => ({ + ...display, + active: true, + backlightState: 'activeOn', + })), + }, + }); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); +}); + +test('a multi-panel device whose panel table is incomplete sends no panel', async () => { + // The table named two integrated panels and the parser could read only one of them, so the + // unread one may be the lit one. Its capture still names the parsed panel because a capture must + // name a display or refuse; a surface fact is not allowed to book every gesture on a display that + // can be dark. + const [outer, inner] = IPHONE_DUO_CLOSED.result.displays; + answering({ + result: { + displays: [outer, { ...inner, nativeSize: undefined }], + }, + }); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); +}); + +test('a device the fold fact refuses never costs a panel table read', async () => { + // A physical iPhone is folded by hand (ADR 0025) and no physical foldable leaf is in the + // supported matrix, so its runner keeps the surface route the request always sent. + assert.equal(await resolveRunnerRequestLitPanel(IOS_DEVICE, {}), undefined); + assert.equal(mockPanelTable.mock.calls.length, 0); +}); diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts new file mode 100644 index 0000000000..a902f1b31f --- /dev/null +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -0,0 +1,44 @@ +import type { AppleLitPanel } from '@agent-device/contracts/interactor-types'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; + +import { appleLitPanel, queryAppleDisplayInventory } from '../core/display-inventory.ts'; +import { isAppleFoldableSimulator } from './orientation.ts'; + +/** + * The panel a runner request names as lit. + * + * A runner cannot tell which panel of a multi-panel device is powered: it has no hinge to read, and + * geometry decides nothing, because the dark panel's windows have perfectly good frames. So the host + * reads the panel table for the request and sends the answer with it, and the runner books its surface + * window against that fact rather than against the application's first window (ADR 0025). + * + * Which panel is lit is read fresh per request: that is exactly the fact a fold changes, and a cached + * answer would send every later request to the panel that has gone dark. Whether the device has more + * than one panel at all is hardware, so `isAppleFoldableSimulator` answers that from its own per-device + * cache and a single-panel iPhone pays one table read for its whole session. + * + * Only a simulator the `fold` fact admits gets here. A physical device is folded by hand (ADR 0025), + * no physical foldable leaf is in the supported matrix, and its runner keeps the historic first-window + * route the same request always sent. + */ +export async function resolveRunnerRequestLitPanel( + device: DeviceInfo, + options: { signal?: AbortSignal } = {}, +): Promise { + if (!(await isAppleFoldableSimulator(device, options.signal))) return undefined; + const inventory = await queryAppleDisplayInventory(device, { signal: options.signal }); + const panel = appleLitPanel(inventory); + if (!panel && !inventory.unresolved) { + emitDiagnostic({ + level: 'debug', + phase: 'apple_runner_lit_panel_unresolved', + data: { + deviceId: device.id, + litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length, + hint: 'The runner request carries no panel and keeps its first-window surface route; runner.log names the window frames it resolved.', + }, + }); + } + return panel; +} diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 87b3e40013..bf9ad96c8f 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -154,7 +154,10 @@ export function createAppleInteractor( ); }, setOrientation: async (orientation) => { - if (await isAppleFoldableSimulator(device, runnerOpts.signal)) { + // A rotation asks the panel table itself rather than answering from the window a surface probe + // may have opened: the two routes differ in correctness, and the historic one reports a + // rotation a foldable never performed. + if (await isAppleFoldableSimulator(device, runnerOpts.signal, { askPanelTableAgain: true })) { return await setAppleFoldableOrientation(device, orientation, { signal: runnerOpts.signal, }); From d4f80daef34896bba77ea101c989a618531f9e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 31 Oct 2026 20:10:36 +0100 Subject: [PATCH 02/29] refactor(ios): resolve one lit-panel window per request Reduce the foldable surface fix to one rule. The host reads the panel table once per surface command (the command traits name which ones) and sends `litPanel`. The runner walks at most eight windows, books the first full-panel window on the lit display, and uses it for dispatch, the coordinate fallback, the viewport, and capture. A regular snapshot keeps only the windows whose element snapshot names the lit display. Requests without `litPanel` keep main's first-window routes unchanged. The graft-and-prune tree pass, the panel-rule module, the APP_SCREEN_OFF_LIT_PANEL code, and the rotation retry option are gone. Co-Authored-By: Claude Opus 5.5 --- .../RunnerAppScreenCapture.swift | 49 +- .../RunnerPanelSurface.h | 96 --- .../RunnerPanelSurface.m | 96 --- .../RunnerPanelSurfaceTree.swift | 158 ----- .../RunnerRequestPanel.swift | 55 +- .../RunnerTests+CommandDispatch.swift | 6 +- .../RunnerTests+Interaction.swift | 71 ++- .../RunnerTests+Keyboard.swift | 14 +- .../RunnerTests+Lifecycle.swift | 9 +- .../RunnerTests+Models.swift | 25 +- .../RunnerTests+Snapshot.swift | 247 ++------ .../RunnerTests+SnapshotAcquisition.swift | 5 +- .../RunnerTests+SnapshotCapturePlan.swift | 28 +- .../RunnerTests.swift | 6 +- .../RunnerXCTestEventBridge.h | 101 +-- .../RunnerXCTestEventBridge.m | 326 +++++----- .../RunnerPanelSurfaceTreeTests.swift | 237 ------- .../UnitTests/RunnerRequestPanelTests.swift | 36 +- .../RunnerTests+AppScreenCaptureTests.swift | 35 +- .../UnitTests/RunnerTests+SnapshotTests.swift | 4 +- .../RunnerTests+XCTestEventBridgeTests.swift | 589 +++++------------- .../SnapshotModels.swift | 6 +- contracts/fixtures/dispatch-disclosure.json | 6 - docs/adr/0025-foldable-apple-panels.md | 33 +- docs/agents/device-foldable.md | 43 +- .../index-dispatch-disclosure.test.ts | 1 - .../runner-error-classification.test.ts | 1 - packages/apple-runner/src/index.ts | 1 + .../apple-runner/src/runner-command-traits.ts | 49 ++ packages/apple-runner/src/runner-contract.ts | 18 +- packages/contracts/src/interactor-types.ts | 18 - .../src/core/display-inventory.ts | 25 +- .../platform-apple/src/core/runner-client.ts | 89 +-- .../src/foldable/orientation.test.ts | 40 +- .../src/foldable/orientation.ts | 70 +-- .../src/foldable/runner-panel.test.ts | 41 +- .../src/foldable/runner-panel.ts | 45 +- packages/platform-apple/src/interactor.ts | 5 +- 38 files changed, 675 insertions(+), 2009 deletions(-) delete mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h delete mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m delete mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift delete mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index bee77183b8..55761e8a57 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -21,16 +21,11 @@ enum RunnerAppScreenCaptureFailure: String, Error { case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED" case unresolvedScreen = "APP_SCREEN_UNRESOLVED" case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE" - /// The request named a lit panel and every window it found is on the other one. Guessing the bigger - /// window would capture a screen nobody is looking at, so the capture refuses with this instead. - case offLitPanel = "APP_SCREEN_OFF_LIT_PANEL" init(_ failure: RunnerApplicationScreenFailure) { switch failure { case .unresolvedWindow: self = .unresolvedWindow - case .offLitPanel: - self = .offLitPanel default: self = .unresolvedScreen } @@ -39,13 +34,11 @@ enum RunnerAppScreenCaptureFailure: String, Error { var message: String { switch self { case .unresolvedWindow: - return "Neither the target app nor the system surface resolved a window, so no display could be named." + return "Neither the target app nor the system surface resolved a window on the lit display, so no display could be named." case .unresolvedScreen: return "The resolved app window did not report the display it is shown on." case .unrenderableImage: return "The resolved display handed back an image that could not be drawn upright." - case .offLitPanel: - return "Every window the app and the system surface reported is on the panel the host reports as dark." } } @@ -57,26 +50,19 @@ enum RunnerAppScreenCaptureFailure: String, Error { return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved." case .unrenderableImage: return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question." - case .offLitPanel: - return "Read the device's fold pose and panel table: the host named a lit panel no window occupies. runner.log names the panel and the window frames it refused." } } } extension RunnerTests { - /// The target rule, kept apart from the two queries so the rule itself is testable: a process that - /// names no window, or none on the lit panel, asks the system surface, and nothing else does. - /// - /// A panel refusal moves on for the same reason an unresolved window does: the home screen is - /// SpringBoard's window, so an app with nothing on the lit panel says nothing about where the lit - /// panel's content is. The system surface is asked, and if neither names a surface on it, the - /// panel refusal is the answer the host needs. + /// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved + /// window asks the system surface, and nothing else does. func selectObservedScreenCapture( resolving: () -> Result, fallingBack: () -> Result ) -> Result { let outcome = resolving() - if case .failure(let failure) = outcome, failure == .unresolvedWindow || failure == .offLitPanel { + if case .failure(.unresolvedWindow) = outcome { return fallingBack() } return outcome @@ -97,13 +83,10 @@ extension RunnerTests { /// Never activates `app`: an observation must not change which app is foregrounded just to get a /// screen, so the caller passes the app context it already intends to observe. /// - /// The panel arrives as an argument rather than being read here: an operation books its whole screen - /// on the panel its request named, and a snapshot of that fact cannot be replaced under it by the - /// command that follows. Callers that live for one command read `requestSurfacePanel` on the command - /// queue and pass what they got, so the panel survives any hop to main. + /// `litPanel` is the panel the request named; nil keeps the app's first window. func captureResolvedAppScreen( app: XCUIApplication, - litPanel: RunnerLitPanel + litPanel: RunnerLitPanel? ) -> Result { var outcome: Result = .failure( .unresolvedScreen @@ -126,13 +109,10 @@ extension RunnerTests { /// keeps that capture honest — it names a display a window actually occupies instead of reaching /// for `XCUIScreen.main`, which on a foldable is one fixed panel whether or not it is dark (#2728). /// - /// Only an unresolved window or a panel refusal moves on to the system surface. A window that - /// resolved on the right panel and then refused to name its display is the failure the host has to - /// see, and no second question about a different process should bury it. - func captureObservedScreen( - app: XCUIApplication, - litPanel: RunnerLitPanel - ) -> Result< + /// Only an unresolved *window* moves on to the system surface. A window that resolved and then + /// refused to name its display is the failure the host has to see, and no second question about a + /// different process should bury it. + func captureObservedScreen(app: XCUIApplication, litPanel: RunnerLitPanel?) -> Result< CapturedAppScreen, RunnerAppScreenCaptureFailure > { selectObservedScreenCapture( @@ -143,12 +123,17 @@ extension RunnerTests { private static func resolveCapturedAppScreen( app: XCUIApplication, - litPanel: RunnerLitPanel + litPanel: RunnerLitPanel? ) -> Result { var screen: AnyObject? var displayID: UInt = 0 var resolution: RunnerApplicationScreenFailure = .unresolvedScreen - guard RunnerResolveApplicationScreen(app, litPanel, &screen, &displayID, &resolution), + let resolved = if let litPanel { + RunnerResolveLitPanelScreen(app, litPanel, &screen, &displayID, &resolution) + } else { + RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution) + } + guard resolved, let captureScreen = screen as? XCUIScreen else { return .failure(RunnerAppScreenCaptureFailure(resolution)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h deleted file mode 100644 index 770dd1d5c1..0000000000 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.h +++ /dev/null @@ -1,96 +0,0 @@ -// RunnerPanelSurface.h — the one rule that names the window a command books its surface against. -// -// Four runner paths need "the window the screen is": the interaction anchor and reference frame, the -// snapshot viewport, the display a capture names, and the display a synthesized event record routes -// to. Each used to take the application's first window. On a foldable home screen that is -// SpringBoard's status-bar window — a 223x317 box on a panel that may be dark — so taps, the -// snapshot viewport, and captures all landed on a screen nobody was looking at. They resolve through -// this rule instead. -// -// Only this file declares which window wins. The reads that feed it stay with their owners: the -// XCTest bridge reads an application's windows and their display identity, and hands the facts here, -// so a device the rule cannot settle is refused the same way by every caller. - -#import -#import - -NS_ASSUME_NONNULL_BEGIN - -// The panel the host reports as lit, in the units the panel table reports it: which display is -// powered, and that panel's own native box in points. The runner cannot read a hinge, and geometry -// cannot say which of two panels is powered, so only the host can name this (ADR 0025): it reads the -// panel table for the request and sends the answer with it. A request that carried no panel leaves -// `displayID` 0. -// -// The size is the panel's *native* box, not the box an app shows on it. The system presents the inner -// panel turned, so its interaction space swaps the side lengths (ADR 0025), which is why the rule -// compares a box either way up. -typedef struct { - NSUInteger displayID; - CGFloat widthPt; - CGFloat heightPt; -} RunnerLitPanel; - -// Whether a request named a panel at all. Only a real display identity counts: a panel with no -// display cannot be compared against a window, so it reads as no panel. -static inline BOOL RunnerLitPanelIsNamed(RunnerLitPanel panel) { - return panel.displayID != 0; -} - -// One window the runner measured: the display it is shown on, and the box it occupies in the app's -// coordinate space. -typedef struct { - NSUInteger displayID; - CGRect frame; -} RunnerPanelWindowFact; - -// What the rule made of an application's windows. Every outcome but `NoWindow` names the window it -// chose or the first one it looked at, through `selected`, so a refusal can report the frame it -// refused instead of returning a silent nil. -typedef NS_ENUM(NSUInteger, RunnerPanelSurfaceSelection) { - // A window on the lit display whose box is that panel's box, either way up: the screen to book. - RunnerPanelSurfaceSelectionLitPanelWindow, - // Windows exist on the lit display but none carries the panel's box, so the app occupies part of - // it (Split View, a stage, a sheet). The largest is the surface to book: being on the lit panel is - // the fact a first-window walk could never establish. - RunnerPanelSurfaceSelectionLitDisplayWindow, - // Every window the application reported is on a display the host reports as dark. - RunnerPanelSurfaceSelectionOffLitPanel, - // No window answered a usable frame, so there is no surface to book right now. - RunnerPanelSurfaceSelectionNoWindow, -}; - -// How many windows one walk reads. A device presenting more qualifying windows than this is not a -// surface this rule can settle, and reading past the cap would only widen the risk of reading a -// window that has already vanished (#2995). -#define AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS 8 - -// Half a point: the rounding tolerance the full-screen proof already uses. Sub-pixel layout is not a -// different panel. -#define AGENT_DEVICE_PANEL_SURFACE_TOLERANCE 0.5 - -// Whether `frame` is the lit panel's surface: anchored at the origin, and the panel's size either way -// up. A frame that is not a measurement at all never qualifies. -FOUNDATION_EXPORT BOOL RunnerPanelFrameIsLitPanelSurface( - CGRect frame, - RunnerLitPanel litPanel -); - -// Decides which of `windows` a command books against, from the facts alone: no queries, no device -// state, no clock. `litPanel` must name a display: a caller whose request carried no panel has -// nothing to compare a window against and keeps its historic first-window route instead of calling -// this. `count` is capped by the caller at `AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS`. -// -// `selected` reports the window the decision was made from and `selectedIndex` its slot in `windows`, -// so a caller keeps the window element it measured instead of reading it again. On a refusal -// `selected` is the first window it looked at, or a null frame when there were none, so the caller's -// refusal names what it saw. -FOUNDATION_EXPORT RunnerPanelSurfaceSelection RunnerPanelSurfaceSelectWindow( - const RunnerPanelWindowFact *windows, - NSUInteger count, - RunnerLitPanel litPanel, - RunnerPanelWindowFact *selected, - NSUInteger *selectedIndex -); - -NS_ASSUME_NONNULL_END diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m deleted file mode 100644 index f8d5560718..0000000000 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurface.m +++ /dev/null @@ -1,96 +0,0 @@ -#import "RunnerPanelSurface.h" -#import "ForegroundApplicationGeometry.h" -#import - -BOOL RunnerPanelFrameIsLitPanelSurface(CGRect frame, RunnerLitPanel litPanel) { - if (!AgentDeviceForegroundFrameIsMeasurable(frame)) { - return NO; - } - if (fabs(frame.origin.x) > AGENT_DEVICE_PANEL_SURFACE_TOLERANCE || - fabs(frame.origin.y) > AGENT_DEVICE_PANEL_SURFACE_TOLERANCE) { - return NO; - } - if (!isfinite(litPanel.widthPt) || !isfinite(litPanel.heightPt)) { - return NO; - } - BOOL upright = - fabs(frame.size.width - litPanel.widthPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE && - fabs(frame.size.height - litPanel.heightPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE; - // A panel the system presents turned reports its window in that panel's own native space, so the - // iPhone Duo inner panel's 951x669 interaction box is the transpose of the 669x951 the panel table - // names for it (ADR 0025). - BOOL turned = - fabs(frame.size.width - litPanel.heightPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE && - fabs(frame.size.height - litPanel.widthPt) <= AGENT_DEVICE_PANEL_SURFACE_TOLERANCE; - return upright || turned; -} - -static CGFloat RunnerPanelWindowFrameArea(CGRect frame) { - CGFloat width = frame.size.width > 1 ? frame.size.width : 1; - CGFloat height = frame.size.height > 1 ? frame.size.height : 1; - return width * height; -} - -RunnerPanelSurfaceSelection RunnerPanelSurfaceSelectWindow( - const RunnerPanelWindowFact *windows, - NSUInteger count, - RunnerLitPanel litPanel, - RunnerPanelWindowFact *selected, - NSUInteger *selectedIndex -) { - if (selected != NULL) { - *selected = (RunnerPanelWindowFact){.displayID = 0, .frame = CGRectNull}; - } - if (selectedIndex != NULL) { - *selectedIndex = NSUIntegerMax; - } - if (windows == NULL || count == 0) { - return RunnerPanelSurfaceSelectionNoWindow; - } - if (selected != NULL) { - *selected = windows[0]; - } - if (selectedIndex != NULL) { - *selectedIndex = 0; - } - - RunnerPanelWindowFact largestPlaced = windows[0]; - NSUInteger largestPlacedIndex = 0; - BOOL sawPlaced = NO; - CGFloat largestPlacedArea = 0; - for (NSUInteger index = 0; index < count; index += 1) { - RunnerPanelWindowFact candidate = windows[index]; - if (candidate.displayID != litPanel.displayID) { - continue; - } - if (RunnerPanelFrameIsLitPanelSurface(candidate.frame, litPanel)) { - if (selected != NULL) { - *selected = candidate; - } - if (selectedIndex != NULL) { - *selectedIndex = index; - } - return RunnerPanelSurfaceSelectionLitPanelWindow; - } - CGFloat area = RunnerPanelWindowFrameArea(candidate.frame); - if (!sawPlaced || area > largestPlacedArea) { - sawPlaced = YES; - largestPlacedArea = area; - largestPlaced = candidate; - largestPlacedIndex = index; - } - } - if (sawPlaced) { - if (selected != NULL) { - *selected = largestPlaced; - } - if (selectedIndex != NULL) { - *selectedIndex = largestPlacedIndex; - } - return RunnerPanelSurfaceSelectionLitDisplayWindow; - } - // The lit display named no window, so every window the app has is on the dark panel. Taking the - // biggest of them by geometry would book a gesture or a capture on a screen nobody is looking at, - // so this refuses and leaves the caller to report the panel it looked at. - return RunnerPanelSurfaceSelectionOffLitPanel; -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift deleted file mode 100644 index 7e048cd987..0000000000 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerPanelSurfaceTree.swift +++ /dev/null @@ -1,158 +0,0 @@ -import Foundation -import AgentDeviceSnapshotPresentation - -/// The tree a multi-panel device publishes. -/// -/// A capture walks the application element, and on a foldable that hands back whichever window the -/// platform serializes. On an iPhone Duo home screen, the application's subtree reported SpringBoard's -/// outer-panel window and the menu that had already closed on it, while the icons on the lit panel were -/// absent. Booking the viewport against the lit panel, as the surface rule does, would then make every -/// published node a claim about a screen nobody is looking at. -/// -/// Three answers come from the panel the request named, and this type is the only place that applies -/// them (ADR 0025): -/// -/// 1. The root node carries the panel's box, because the host reads that node as the box the payload is -/// measured in. On a multi-panel device the application element's own frame names a panel, not the -/// panel in use. -/// 2. Surfaces that cannot be seen on that panel are dropped, because their refs would name points on a -/// dark screen and the fold can only clip them into a shape nobody can act on. -/// 3. When the tree carries no window for the panel at all, the caller reads the resolved surface window -/// once and grafts it, which is what makes the lit panel's icons addressable. -/// -/// A request that names no panel reaches none of this: on a single-panel device the application element's -/// frame is the surface box and every window is on the one panel, so the historic tree is published as it -/// was read. -enum RunnerPanelSurfaceTree { - /// One point of drift between two reported window boxes is layout, not a different panel. This is - /// looser than the surface rule's half point (`AGENT_DEVICE_PANEL_SURFACE_TOLERANCE`) on purpose: - /// that rule compares an app's box against the panel fact the host divided out of pixels, while this - /// type compares two boxes the runtime reported itself. - static let tolerance: CGFloat = 1 - - /// The wire types a whole surface arrives as, in the payload's own vocabulary. `Other` is - /// deliberately absent: XCTest reports it for most containers, and treating it as a surface would - /// drop content the fold can clip. `Sheet` and `Drawer` are what XCTest would name them; today its - /// tree reports them as `Element()`, so in practice only `Window` is dropped, which is the - /// conservative direction. The graft asks the platform's element types instead - /// (`RunnerTests.surfaceElementTypes`), because it reads live elements rather than a payload. - static let surfaceTypeNames: Set = ["Application", "Window", "Sheet", "Drawer"] - - /// Whether a reported box is that panel's box, either way up: the system presents the inner panel - /// turned, so its window arrives in the panel's own native space (ADR 0025). - static func matchesSurfaceBox(_ frame: CGRect, _ surface: CGRect) -> Bool { - guard isMeasurable(frame), isMeasurable(surface) else { return false } - guard abs(frame.minX) <= tolerance, abs(frame.minY) <= tolerance else { return false } - return (near(frame.width, surface.width) && near(frame.height, surface.height)) - || (near(frame.width, surface.height) && near(frame.height, surface.width)) - } - - /// Whether a reported box can be seen on the named panel. - static func liesOnSurface(_ frame: CGRect, _ surface: CGRect) -> Bool { - guard isMeasurable(frame), isMeasurable(surface) else { return false } - return frame.minX >= surface.minX - tolerance - && frame.minY >= surface.minY - tolerance - && frame.maxX <= surface.maxX + tolerance - && frame.maxY <= surface.maxY + tolerance - } - - /// Whether one already-read layer of a capture carries a window on the named panel. That is the - /// capture's own evidence that it already shows the lit panel, so no second read is needed. - static func carriesSurfaceWindow( - _ layer: [Layer], - surface: CGRect, - isSurface: (Layer) -> Bool, - frame: (Layer) -> CGRect - ) -> Bool { - layer.contains { isSurface($0) && matchesSurfaceBox(frame($0), surface) } - } - - /// What narrowing a payload to one panel changed, so the caller can report it and a test can read it. - struct Published { - let nodes: [RawAXNode] - /// Whether the root moved onto the panel's box. - let republishedRoot: Bool - /// How many surfaces were removed with their subtrees. - let droppedSurfaces: Int - } - - /// The tree as published on the named panel: the root carrying the panel's box and the off-panel - /// surfaces removed with their subtrees, re-indexed for the payload. - /// - /// `nil` means the tree already said exactly this, which is the common case and costs no copy. Only a - /// payload rooted at the application is rewritten: that root is the node whose children are the app's - /// whole window list, so it is the one shape where a child naming a surface names a screen. A payload - /// already rooted at a window was read from one surface and cannot hold another one's content. - /// - /// Only a root's own children are dropped: a surface is a whole screen, so its subtree stands or falls - /// with it, while a node deeper in the tree that reaches past the panel is content the fold clips. - static func publishing(nodes: [RawAXNode], surface: CGRect) -> Published? { - guard let root = nodes.first, root.parentIndex == nil, root.type == "Application", - nodes.dropFirst().contains(where: { $0.parentIndex == nil }) == false - else { - return nil - } - let offPanel = nodes.filter { node in - node.parentIndex == root.index - && node.index != root.index - && surfaceTypeNames.contains(node.type) - && !liesOnSurface(node.rect.cgRect, surface) - } - let panelBox = SnapshotRect(surface) - let republishRoot = root.rect != panelBox - guard republishRoot || !offPanel.isEmpty else { return nil } - - let dropped = subtreeIndexes(of: offPanel.map(\.index), in: nodes) - var kept: [RawAXNode] = [] - kept.reserveCapacity(nodes.count - dropped.count) - var reindex: [Int: Int] = [:] - reindex.reserveCapacity(nodes.count - dropped.count) - for node in nodes where !dropped.contains(node.index) { - reindex[node.index] = kept.count - kept.append(node) - } - let published = kept.map { node in - let republished = node.parentIndex == nil - ? node.replacing(rect: panelBox, hittable: node.hittable) - : node - return republished.reindexed( - index: reindex[node.index] ?? node.index, - depth: node.depth, - parentIndex: node.parentIndex.flatMap { reindex[$0] } - ) - } - return Published( - nodes: published, - republishedRoot: republishRoot, - droppedSurfaces: offPanel.count - ) - } - - /// Depth counts ancestors and only whole sibling subtrees are removed, so a kept node keeps its depth. - /// Indexes shift because the payload's wire array is positional. - private static func subtreeIndexes(of roots: [Int], in nodes: [RawAXNode]) -> Set { - var childrenByParent: [Int: [Int]] = [:] - for node in nodes { - if let parent = node.parentIndex { - childrenByParent[parent, default: []].append(node.index) - } - } - var dropped = Set(roots) - var frontier = roots - while let index = frontier.popLast() { - for child in childrenByParent[index] ?? [] where dropped.insert(child).inserted { - frontier.append(child) - } - } - return dropped - } - - private static func isMeasurable(_ frame: CGRect) -> Bool { - frame.minX.isFinite && frame.minY.isFinite && frame.size.width.isFinite && frame.size.height.isFinite - && frame.size.width > 0 && frame.size.height > 0 - } - - private static func near(_ left: CGFloat, _ right: CGFloat) -> Bool { - abs(left - right) <= tolerance - } -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 1814701a9f..53326897b9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -1,51 +1,26 @@ import XCTest -// MARK: - Request-scoped lit panel (#2728, ADR 0025) -// -// A foldable lights one panel at a time, and only the host can say which: the runner has no hinge to -// read, and geometry cannot settle it, because the dark panel's windows have perfectly good frames. -// So the host reads the panel table for the request and names the lit panel in it (`Command.litPanel`), -// and every surface resolver books its window against that fact rather than against the application's -// first window — which on a foldable home screen is SpringBoard's status-bar window. -// -// The resolvers that need the fact sit on the gesture, snapshot, capture and synthesized-dispatch -// paths. Each of them runs the command that carries the fact, and none of them has the `Command` in -// scope: the viewport read is a hop inside a capture plan, the capture answers through a result type, -// and a gesture path is handed a context rather than a request. So the runner holds the panel of the -// command in flight in one place, which is also the only place that answers the question. -// -// Commands are served one at a time on `commandExecutionQueue`, which is where the slot is written. -// Readers run on main, so the slot is lock-guarded rather than plain main-owned state, the same split -// `SnapshotProbePenaltyIdentity` makes. Main-queue work abandoned by the execution watchdog can -// therefore read the panel of the command that replaced it; the resolution marker the bridge logs -// names the panel a window was booked against, so such a resolution is visible, not silent. -// -// An operation that keeps sampling after its request is served does not read the slot per sample: it -// reads it once on the command queue and carries that panel as an argument for its whole life. A clip -// records on one panel and a settle loop compares samples of one screen; either one reading the slot -// again would be answering a question about a command it is not part of. - +// The lit panel of the command in flight (ADR 0025). Commands run one at a time on +// `commandExecutionQueue`, where the slot is written, and the surface resolvers read it on main. +// Work that outlives its command, such as a recording pump or a settle loop, reads it once and +// carries that panel instead. extension RunnerTests { - /// Records the panel the command about to be served names as lit. A request that names none clears - /// the slot, so a command for a single-panel device can never inherit a foldable's panel. func setRequestLitPanel(from command: Command) { + var panel: RunnerLitPanel? + if let fact = command.litPanel, fact.displayId != 0, fact.widthPt.isFinite, fact.heightPt.isFinite, + fact.widthPt > 0, fact.heightPt > 0 + { + panel = RunnerLitPanel(displayID: fact.displayId, widthPt: fact.widthPt, heightPt: fact.heightPt) + } requestPanelLock.lock() - requestLitPanel = command.litPanel + requestLitPanel = panel requestPanelLock.unlock() } - /// The lit panel in the units the surface rule compares a window against. An unnamed or unusable - /// panel answers `displayID` 0, which every resolver reads as "keep the historic first-window - /// route": a panel that cannot be named cannot be compared with a window. - var requestSurfacePanel: RunnerLitPanel { + /// The lit panel the current request named, or nil for a single-panel request. + var requestSurfacePanel: RunnerLitPanel? { requestPanelLock.lock() - let panel = requestLitPanel - requestPanelLock.unlock() - guard let panel, panel.displayId != 0, panel.widthPt.isFinite, panel.heightPt.isFinite, - panel.widthPt > 0, panel.heightPt > 0 - else { - return RunnerLitPanel(displayID: 0, widthPt: 0, heightPt: 0) - } - return panel.surfacePanel + defer { requestPanelLock.unlock() } + return requestLitPanel } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index 88031d1076..8b77e09315 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -7,8 +7,6 @@ extension RunnerTests { func executeAccepted(command: Command) throws -> Response { commandJournal.start(command: command) pendingTargetActivation = nil - // Before anything the command resolves: every surface resolver this command reaches reads the - // panel it named, and one that named none must not inherit the previous command's. setRequestLitPanel(from: command) do { let response = try executeDispatched(command: command) @@ -435,9 +433,7 @@ extension RunnerTests { outputPath: resolvedOutPath, fps: command.fps.map { Int32($0) } ) - // One panel for the whole clip, taken while this request still owns the slot: the writer is - // sized from the bootstrap frame, so a pump that asked again later could book the remaining - // frames on the panel of whatever command happened to follow `recordStart`. + // The pump outlives this request, so the clip keeps the panel its `recordStart` named. let clipLitPanel = requestSurfacePanel try startRecording(recorder) { [weak self] in guard let self else { return .failure(.unresolvedScreen) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 109572747c..b26e240b51 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -243,16 +243,17 @@ extension RunnerTests { // transformed subtrees, so a closed drawer at negative x inflates it and // out-of-window coordinates still pass containment. Falls back to app.frame // when no window frame is readable. - /// The app window interactions and captures are booked against, and no window at all when none - /// qualifies. The viewport, the interaction anchor, and the synthesized reference frame all resolve - /// through here, so a fold, a sheet, or a rotation moves every consumer to the same window in the - /// same pass. Synthesized gestures route their display ID through the returned window so the record's + /// The app window interactions and captures are booked against: the first window the app + /// reports with a non-empty frame, and no window at all when none qualifies. The viewport, + /// the interaction anchor, and the synthesized reference frame all resolve through here, so a + /// fold, a sheet, or a rotation moves every consumer to the same window in the same pass. + /// Synthesized gestures route their display ID through the returned window so the record's /// display can never name a different window than the one the reference frame was measured on. /// - /// Which window qualifies is `RunnerResolveSurfaceWindow`'s decision, and this only asks it: on a - /// single-panel device that is the first window with a non-empty frame, and on a foldable it is a - /// window on the panel the request named as lit, which is how a home-screen tap stops landing on - /// SpringBoard's status-bar window on the dark panel. + /// Stops at the first qualifying window. Reading a window that will be discarded is not free: a + /// transient candidate that disappears between its `exists` answer and its `frame` fetch makes + /// XCTest record an issue, and an issue `record(_:)` does not mute ends the runner session + /// (ADR 0004, ADR 0005, #2995). /// /// The read that remains is not atomic: a window vanishing while it is the one being resolved still /// records, and the chosen window is read again on its way to a display ID @@ -264,27 +265,35 @@ extension RunnerTests { return (resolved.window, resolved.frame) } - /// The app's surface window, or nil. Unlike `resolveRunnerWindow` it never falls back to - /// `app.frame`: when the windows query fails because XCTest cannot serialize the app's tree, that - /// read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). - func resolveRunnerWindowElement( + /// The app's first window with a usable frame, or nil. Unlike `resolveRunnerWindow` it never falls + /// back to `app.frame`: when the windows query fails because XCTest cannot serialize the app's + /// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). + func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { + if let litPanel = requestSurfacePanel { + return resolveLitPanelWindow(app: app, litPanel: litPanel) + } + for window in app.windows.allElementsBoundByIndex where window.exists { + let frame = window.frame + if !frame.isEmpty { + return (window, frame) + } + } + return nil + } + + /// A foldable's surface: the window on the panel the request named as lit, never the first window. + func resolveLitPanelWindow( app: XCUIApplication, - litPanel: RunnerLitPanel? = nil + litPanel: RunnerLitPanel ) -> (window: XCUIElement, frame: CGRect)? { - var resolvedWindow: AnyObject? - var windowFrame = CGRect.null - var observedFrame = CGRect.null - let failure = RunnerResolveSurfaceWindow( - app, - litPanel ?? requestSurfacePanel, - &resolvedWindow, - &windowFrame, - &observedFrame - ) - guard failure == .none, let window = resolvedWindow as? XCUIElement else { + var window: AnyObject? + var frame = CGRect.null + guard RunnerResolveLitPanelWindow(app, litPanel, &window, &frame) == .none, + let element = window as? XCUIElement + else { return nil } - return (window, windowFrame) + return (element, frame) } func onScreenWindowFrame(app: XCUIApplication) -> CGRect { @@ -633,17 +642,19 @@ extension RunnerTests { // window is not always the one under the finger — a SpringBoard `alert accept` can live in an // alert window while the wallpaper or status-bar window qualifies first, so a window-relative // anchor misses the button. Reference frames and the synthesized display ID still resolve - // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. - let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) - return origin.withOffset(CGVector(dx: x, dy: y)) -#else + // through `resolveRunnerWindow`. A foldable anchors at the lit panel's window instead, because + // the app origin names the outer panel whichever one is lit. + if requestSurfacePanel == nil { + let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) + return origin.withOffset(CGVector(dx: x, dy: y)) + } +#endif let resolved = resolveRunnerWindow(app: app) let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) let offsetX = x - Double(resolved.frame.origin.x) let offsetY = y - Double(resolved.frame.origin.y) return origin.withOffset(CGVector(dx: offsetX, dy: offsetY)) -#endif } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift index 9ace71ff93..05f3b5b00d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift @@ -241,13 +241,10 @@ extension RunnerTests { requiredConsecutiveMatches: Int ) { let deadline = Date().addingTimeInterval(timeout) - // One panel for the whole settle loop: samples taken against two panels could never match, and a - // loop that outlives its request must not follow the command that replaced it. - let litPanel = requestSurfacePanel - var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app, litPanel: litPanel)] + var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app)] while Date() < deadline { sleepFor(sampleInterval) - samples.append(screenshotFingerprintForStabilityCheck(app: app, litPanel: litPanel)) + samples.append(screenshotFingerprintForStabilityCheck(app: app)) if runnerScreenshotStabilitySettled(samples, requiredConsecutiveMatches: requiredConsecutiveMatches) { logScreenshotStability(samples: samples, settled: true) return @@ -268,11 +265,8 @@ extension RunnerTests { ) } - private func screenshotFingerprintForStabilityCheck( - app: XCUIApplication, - litPanel: RunnerLitPanel - ) -> Data? { - guard let image = captureRunnerFrame(app: app, litPanel: litPanel) else { return nil } + private func screenshotFingerprintForStabilityCheck(app: XCUIApplication) -> Data? { + guard let image = captureRunnerFrame(app: app) else { return nil } return runnerPngData(for: image) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 63657867c6..2597b27669 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -51,8 +51,8 @@ extension RunnerTests { /// identical black frames would then read as a settled screen (#2728). An observation with no /// session window falls to the system surface's window, which is what the home screen is. macOS /// keeps the host display it always recorded. - func captureRunnerFrame(app: XCUIApplication, litPanel: RunnerLitPanel) -> RunnerImage? { - switch captureRunnerFrameResult(app: app, litPanel: litPanel) { + func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? { + switch captureRunnerFrameResult(app: app, litPanel: requestSurfacePanel) { case .success(let captured): return captured.image case .failure: @@ -64,12 +64,9 @@ extension RunnerTests { /// frame — a recording's bootstrap, which sizes the whole writer from it — fails closed with a /// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal /// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728). - /// The panel is a parameter and not a read here for the same reason it is one for the capture: a - /// recording is one observation of one panel, and its pump runs long after the request that started - /// it was served. A caller that lives for one command may read `requestSurfacePanel` itself. func captureRunnerFrameResult( app: XCUIApplication, - litPanel: RunnerLitPanel + litPanel: RunnerLitPanel? ) -> Result { #if os(iOS) return captureObservedScreen(app: app, litPanel: litPanel) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 1a9d2dee6b..9cb03978e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -230,33 +230,13 @@ struct CoordinateTapTextInputEvidence: Codable { let textInputRects: [RunnerGestureViewport] } -/// The panel the host reports as lit, in the units the host's panel table reports it: which display is -/// powered and that panel's own native box in points. -/// -/// Only the host can name this. The runner has no hinge to read, and geometry cannot say which of two -/// panels is powered — the dark panel's window has a perfectly good frame — so the host reads the -/// panel table for the request and sends the answer with it (ADR 0025). A request that carries no -/// panel is a single-panel device's request, and keeps the runner's historic first-window route. -/// -/// `widthPt`/`heightPt` are the panel's *native* box, not the box an app shows on it: the system -/// presents the inner panel turned, so its interaction space swaps the side lengths. -/// -/// The field names are the wire spelling and the request side's own convention (`commandId`, -/// `appBundleId`), so the id keeps its lowercase `d` here even though the surface rule's own type -/// spells the runtime's `XCUIScreen.displayID` property the runtime's way. +/// The wire spelling of the lit panel the host names for a multi-panel device (ADR 0025). struct RunnerLitPanelFact: Codable { let displayId: UInt let widthPt: Double let heightPt: Double } -extension RunnerLitPanelFact { - /// The panel in the units the surface rule compares a window against. - var surfacePanel: RunnerLitPanel { - RunnerLitPanel(displayID: displayId, widthPt: CGFloat(widthPt), heightPt: CGFloat(heightPt)) - } -} - struct Command: Codable { let command: CommandType let commandId: String? @@ -296,9 +276,6 @@ struct Command: Codable { let inlineScreenshot: Bool? let synthesized: Bool? let coordinateTapTextInputEvidence: CoordinateTapTextInputEvidence? - /// The lit panel, sent by the host only for a device whose panel table reports more than one panel. - /// Absent means "single-panel device": every surface resolver then takes its historic first-window - /// route and reads no display identity at all. let litPanel: RunnerLitPanelFact? let steps: [SequenceStep]? } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index e36bf2f2ce..f0ba0cc8d3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -185,7 +185,6 @@ extension RunnerTests { @MainActor func recursiveTreeSnapshotAcquisition( - app: XCUIApplication, context: SnapshotTraversalContext, hint: CaptureHint, processingDeadline: Date @@ -205,98 +204,39 @@ extension RunnerTests { // Acquisition serializes reported frames only; no coordinate space or fold decision is carried // down the walk (#2661). Its sole bounds are raw traversal depth (nil for a regular capture) and // the node cap; `SnapshotPresentation` folds and cuts the normalized array. - var nodes: [RawAXNode] = [] - nodes.append( - makeSnapshotNode( - snapshot: context.rootSnapshot, - evaluation: evaluateSnapshot(context.rootSnapshot), - depth: 0, - index: 0, - parentIndex: nil - ) - ) - var seen = Set() - try walkSnapshotChildren( - of: context.rootSnapshot, - parentIndex: 0, + var root = makeSnapshotNode( + snapshot: context.rootSnapshot, + evaluation: evaluateSnapshot(context.rootSnapshot), depth: 0, - into: &nodes, - seen: &seen, - hint: hint, - collapsedTabStep: { [self] nodes, container, depth, parentIndex in - self.appendCollapsedTabFallbackNodes( - to: &nodes, - containerSnapshot: container, - resolveCandidates: collapsedTabCandidatesOnce, - depth: depth, - parentIndex: parentIndex - ) - } + index: 0, + parentIndex: nil ) - - // A foldable's application subtree can hold only another panel's window: on a Duo home screen it - // reported the outer panel's window and the lit panel's icons were absent. The window the surface - // rule resolved is then the only remaining way to read the screen the request named, and one read - // of it is what makes its content addressable (ADR 0025). Only a request that named a panel asks - // this question: a single-panel app legitimately has no window child for its own frame, and a - // second read there would be a cost the historic route never paid. - let surface = context.viewport.rect - // One panel read for the tier: the decision below and the graft it triggers must name the same - // panel the request did, wherever the graft's own hop to main runs. - let tierLitPanel = requestSurfacePanel - if let surface, tierLitPanel.displayID != 0, - !RunnerPanelSurfaceTree.carriesSurfaceWindow( - context.rootSnapshot.children, - surface: surface, - isSurface: { Self.surfaceElementTypes.contains($0.elementType) }, - frame: { $0.frame } - ) - { - outcome = graftSurfaceWindowNodes( - app: app, - surface: surface, - litPanel: tierLitPanel, - into: &nodes, - seen: seen, - hint: hint, - outcome: outcome, - processingDeadline: processingDeadline + // A multi-panel device lists windows on both panels under one application, and the application's + // frame names one panel. The screen is the lit panel's native box and the windows that name its + // display (ADR 0025). + let litPanel = requestSurfacePanel + var rootChildren = context.rootSnapshot.children + if let litPanel { + root.rect = SnapshotRect(CGRect(x: 0, y: 0, width: litPanel.widthPt, height: litPanel.heightPt)) + rootChildren = rootChildren.filter { Self.snapshotDisplayID($0).map { $0 == litPanel.displayID } ?? true } + } + var nodes: [RawAXNode] = [root] + if Self.canDescendAtRawDepth(0, hint: hint) { + appendCollapsedTabFallbackNodes( + to: &nodes, + containerSnapshot: context.rootSnapshot, + resolveCandidates: collapsedTabCandidatesOnce, + depth: 1, + parentIndex: 0 ) } - return ( - SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: outcome == .deadlineExhausted, - effectiveDepth: nil, - viewport: context.viewport - ), - outcome - ) - } - - /// The element types a whole surface arrives as, in the platform's own vocabulary. The tree reports - /// `Other` for most containers, so it is deliberately absent: a container is not a screen. - static let surfaceElementTypes: Set = [.window, .sheet, .drawer] - - /// One subtree of the walk, applying the walk's own bounds: raw traversal depth, the node cap, and - /// identity de-duplication. Shared by the application walk and by the panel-surface graft so the two - /// cannot disagree about what a payload's nodes mean (#2661). - @MainActor - private func walkSnapshotChildren( - of snapshot: XCUIElementSnapshot, - parentIndex: Int, - depth: Int, - into nodes: inout [RawAXNode], - seen: inout Set, - hint: CaptureHint, - collapsedTabStep: ((inout [RawAXNode], XCUIElementSnapshot, Int, Int) -> Void)? - ) throws { - guard Self.canDescendAtRawDepth(depth, hint: hint) else { return } - collapsedTabStep?(&nodes, snapshot, depth + 1, parentIndex) - var stack: [SnapshotTraversalEntry] = snapshot.children.map { - SnapshotTraversalEntry(snapshot: $0, depth: depth + 1, parentIndex: parentIndex) + var seen = Set() + var stack: [SnapshotTraversalEntry] = [] + if Self.canDescendAtRawDepth(0, hint: hint) { + stack = rootChildren.map { + SnapshotTraversalEntry(snapshot: $0, depth: 1, parentIndex: 0) + } } while let entry = stack.popLast() { @@ -339,119 +279,36 @@ extension RunnerTests { throw regularSnapshotTooLargeFailure(nodeCount: nodes.count) } if Self.canDescendAtRawDepth(depth, hint: hint) { - collapsedTabStep?(&nodes, snapshot, depth + 1, node.index) + appendCollapsedTabFallbackNodes( + to: &nodes, + containerSnapshot: snapshot, + resolveCandidates: collapsedTabCandidatesOnce, + depth: depth + 1, + parentIndex: node.index + ) } } - } - /// The window the surface rule resolved, read once and grafted onto the application root as a sibling - /// of the windows the application subtree already carried. - /// - /// The read is bounded by what the tier has left: a snapshot that cannot start inside the remaining - /// slice is not started, because XCTest cannot cancel one and an abandoned read stalls main for the - /// whole plan. A graft that cannot finish leaves the tree exactly as it was walked, with the nodes and - /// dedup keys its abandoned read produced discarded, so the payload stays one the host can validate: - /// the tier that already read the screen is never failed or marked truncated by a graft on top of it. - @MainActor - private func graftSurfaceWindowNodes( - app: XCUIApplication, - surface: CGRect, - litPanel: RunnerLitPanel, - into nodes: inout [RawAXNode], - seen: Set, - hint: CaptureHint, - outcome: SnapshotTierOutcome, - processingDeadline: Date - ) -> SnapshotTierOutcome { - guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: Self.surfaceWindowReadFloor) else { - // A graft that cannot start leaves the tier's own outcome alone: the tree already walked is - // complete, and calling it `deadlineExhausted` would discard it and penalize the backend. - NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=budget_exhausted") - return outcome - } - guard let resolved = resolveRunnerWindowElement(app: app, litPanel: litPanel) else { - NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=window-unresolved") - return outcome - } - guard resolved.frame.width.isFinite, resolved.frame.height.isFinite, - RunnerPanelSurfaceTree.liesOnSurface(resolved.frame, surface) - else { - NSLog( - "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=window-off-panel frame=(%g,%g %gx%g)", - resolved.frame.origin.x, - resolved.frame.origin.y, - resolved.frame.size.width, - resolved.frame.size.height - ) - return outcome - } - // The window walk above is itself an unbounded XCTest read, so the slice is tested again here: - // what the tier has left after it, and not before, is what the snapshot may still use. - let sliceLeft = processingDeadline.timeIntervalSinceNow - guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: Self.surfaceWindowReadFloor) else { - NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=budget_exhausted") - return outcome - } - // `seen` arrives as a copy on purpose: the walk's dedup keys are kept only when its nodes are, and - // keys for nodes nobody published would hide content that arrives later by the app route. - let graftStart = nodes.count - // The grafted root is the one node this tier appends outside the walk's own cap check, so it is - // asked for here: a tree already at the cap gets no graft, and the read that could not be - // published is not paid for. - guard graftStart < Self.regularSnapshotMaxNodes else { - NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=node-cap nodes=\(graftStart)") - return outcome - } - var seenByGraft = seen - do { - guard let windowSnapshot = try captureSnapshotRootBounded( - resolved.window, - sliceSeconds: sliceLeft - ) else { - NSLog("AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=read-failed") - return outcome - } - let windowNode = makeSnapshotNode( - snapshot: windowSnapshot, - evaluation: evaluateSnapshot(windowSnapshot), - depth: 1, - index: nodes.count, - parentIndex: 0 - ) - nodes.append(windowNode) - try walkSnapshotChildren( - of: windowSnapshot, - parentIndex: windowNode.index, - depth: 1, - into: &nodes, - seen: &seenByGraft, + return ( + SnapshotAcquisition( hint: hint, - collapsedTabStep: nil - ) - NSLog( - "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_GRAFTED frame=(%g,%g %gx%g) nodes=%ld", - resolved.frame.origin.x, - resolved.frame.origin.y, - resolved.frame.size.width, - resolved.frame.size.height, - nodes.count - ) - } catch { - // The tree walked before the graft was publishable as it stands, so a graft that cannot be - // finished never fails the tier: the payload goes out without it, and without the dedup keys its - // abandoned read had added, since those live only in this scope's copy of `seen`. - nodes.removeLast(nodes.count - graftStart) - let failure = error as? SnapshotCaptureFailure - NSLog( - "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE_SKIPPED reason=graft-failed code=\(failure?.code ?? "unknown") nodes=\(graftStart)" - ) - } - return outcome + nodes: nodes, + truncated: outcome == .deadlineExhausted, + effectiveDepth: nil, + viewport: context.viewport + ), + outcome + ) } - /// The least a second snapshot can usefully be given. Below it the read would be abandoned on its way - /// in, which costs the plan more than the tree it would have added. - static let surfaceWindowReadFloor: TimeInterval = 0.25 + /// The display a window snapshot reports, or nil when the runtime names none. + static func snapshotDisplayID(_ snapshot: XCUIElementSnapshot) -> UInt? { + var displayID: UInt? + _ = RunnerObjCExceptionCatcher.catchException({ + displayID = ((snapshot as AnyObject).value(forKey: "displayID") as? NSNumber)?.uintValue + }) + return displayID.flatMap { $0 == 0 ? nil : $0 } + } // See `snapshotFast` above: the single production entry point, no unit-test overload. func snapshotRaw(target: SnapshotCaptureTarget, options: PresentationOptions) throws -> DataPayload { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 594717adf8..6f31243dea 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -76,7 +76,7 @@ extension RunnerTests { /// penalizes the tree backend for this bundle (unless the fresh-process warmup exemption /// applies), and the plan moves to the platform's independent recovery tier when one exists /// (#1105/#1122). - func captureSnapshotRootBounded( + private func captureSnapshotRootBounded( _ element: XCUIElement, sliceSeconds: TimeInterval ) throws -> XCUIElementSnapshot? { @@ -259,9 +259,6 @@ extension RunnerTests { private func snapshotAppFrame(app: XCUIApplication) -> CGRect { #if os(iOS) - // The same resolved surface a tap books against: on a foldable the first window is SpringBoard's - // status-bar window on a panel that may be dark, and a viewport that small makes the presenter - // refuse a tree whose nodes fall outside it. return onScreenWindowFrame(app: app) #else return app.frame diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 4363e69969..c47b77842a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -360,7 +360,6 @@ extension RunnerTests { hint.isRaw ? (try self.rawTreeSnapshotAcquisition(context: context, hint: hint), .completed) : try self.recursiveTreeSnapshotAcquisition( - app: app, context: context, hint: hint, processingDeadline: processingDeadline @@ -418,29 +417,12 @@ extension RunnerTests { // The one coordinate-space pass (#2661): reported frames become the app's orientation space // before presentation reads them. A backend with unknown orientation (the windowless sweep) // turns nothing. - let normalizedNodes = SnapshotGeometrySpace.normalized( - nodes: acquisition.nodes, - viewport: acquisition.viewport - ) - // On a multi-panel device the app element's frame names a panel, not the panel in use, and the host - // reads that root node as the box the whole payload is measured in. Publishing the resolved surface - // box, and dropping what cannot be seen on it, is what the request's panel fact says about this tree. - // Raw is exempt by its own contract: it retains reported frames, and publishes no viewport box the - // host would have to agree with, so a caller asking for the platform's report keeps the whole report. - var publishedNodes = normalizedNodes - if requestSurfacePanel.displayID != 0, !hint.isRaw, let surface = acquisition.viewport.rect, - let published = RunnerPanelSurfaceTree.publishing(nodes: publishedNodes, surface: surface) - { - NSLog( - "AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE republishedRoot=%@ droppedSurfaces=%ld nodes=%ld->%ld", - published.republishedRoot ? "yes" : "no", - published.droppedSurfaces, - normalizedNodes.count, - published.nodes.count + let normalizedAcquisition = acquisition.replacingNodes( + SnapshotGeometrySpace.normalized( + nodes: acquisition.nodes, + viewport: acquisition.viewport ) - publishedNodes = published.nodes - } - let normalizedAcquisition = acquisition.replacingNodes(publishedNodes) + ) let presented: SnapshotBackendCapture do { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index c6be23f12b..b0a552ac94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -92,11 +92,9 @@ final class RunnerTests: XCTestCase { // Set while serving a command that had to re-activate the bound app, and stamped onto that // command's response before it leaves the execution queue (#2682). var pendingTargetActivation: TargetActivationFactPayload? - // The lit panel the command being served names, and the lock that guards it. A foldable's surface - // resolvers read it through `requestSurfacePanel`; see RunnerRequestPanel.swift for why it is not - // main-owned state and what an abandoned command's read can cost. + // Written on the command queue and read on main; see RunnerRequestPanel.swift. let requestPanelLock = NSLock() - var requestLitPanel: RunnerLitPanelFact? + var requestLitPanel: RunnerLitPanel? // iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when // hardware-keyboard input hides the software keyboard. A successful tap on a concrete text // input is a scoped witness for the immediately-following bare type; lifecycle and non-text diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 80139d51d6..7c8b97954b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -1,7 +1,5 @@ #import #import -// The `RunnerLitPanel` the screen resolvers take, and the panel rule their walk applies. -#import "RunnerPanelSurface.h" NS_ASSUME_NONNULL_BEGIN @@ -31,11 +29,30 @@ typedef NS_ENUM(NSUInteger, RunnerApplicationScreenFailure) { RunnerApplicationScreenFailureUnresolvedWindow, // A resolved window whose screen or display identity the XCTest runtime would not answer. RunnerApplicationScreenFailureUnresolvedScreen, - // The request named a lit panel and every window the app reported is on the other one, so no - // surface on the screen the user is looking at exists to book against. - RunnerApplicationScreenFailureOffLitPanel, }; +// The panel the host reports as lit (ADR 0025): its display and its native box in points. The +// system presents a panel turned, so a window on it may report the box with its sides swapped. +typedef struct { + NSUInteger displayID; + CGFloat widthPt; + CGFloat heightPt; +} RunnerLitPanel; + +// The walk reads at most this many windows: each read of a window that has vanished records an +// XCTest issue (#2995), and SpringBoard lists about forty. +#define AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS 8 + +// Resolves the window on `litPanel`: the first window on that display whose frame is the panel's +// box either way up, else the largest window on that display. No window on that display is +// `UnresolvedWindow`; a list the walk could not fully place is `UnresolvedScreen`. +FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( + id application, + RunnerLitPanel litPanel, + id _Nullable *window, + CGRect *windowFrame +); + // Resolves the shared core against the live XCTest runtime. Returns nil on success, or a // "private XCTest synthesis unavailable: ..." message when a class or selector the // private API is expected to expose is missing. `surface` names the calling feature. @@ -45,61 +62,21 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveXCTestEventBridge( RunnerXCTestEventBridge *bridge ); -// Resolves the application's surface window and returns it, so every consumer of the resolved window -// — the capture's display, the interaction anchor and reference frame, the snapshot viewport, and the -// synthesized record's display — books against one window that one rule chose. -// -// `litPanel` is the panel the request named, or a `displayID` of 0 when it named none. With no panel -// this is the historic read: the first window the application reports with a usable frame, and no -// display identity is read at all, so a single-panel device pays nothing it did not pay before. With a -// panel, the walk reads each candidate's frame and display and keeps the window -// `RunnerPanelSurfaceSelectWindow` names, which is how a foldable home screen stops resolving to -// SpringBoard's status-bar window on whichever panel the runtime happens to list first. -// -// `*window` is the window element the runtime handed out, ready for `RunnerReadResolvedWindowScreen` -// or `RunnerResolveWindowDisplayID`; `*windowFrame` is the box the rule booked it for. `*observedFrame` -// is the last frame the walk looked at, so a refusal names what it refused. Private KVC raises rather -// than returns nil, so every read is guarded here and any failure surfaces as the typed `failure`, -// never an exception. The walk is bounded: reading a window that has just vanished makes XCTest record -// an issue that outlives the call (#2995). -FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveSurfaceWindow( - id application, - RunnerLitPanel litPanel, - id _Nullable *window, - CGRect *windowFrame, - CGRect *observedFrame -); - -// Reads the `XCUIScreen` and `displayID` of a window the caller has already resolved and measured. -// The panel walk holds the frame; asking a window it just measured for the same measurement again -// would only add a read of a window that may already be gone. `screen` may be NULL for a caller that -// routes by display identity alone. -FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( - id window, - CGRect observedFrame, - id _Nullable * _Nullable screen, - NSUInteger *displayID -); - -// The application's window elements, at most `maxCount` of them, read through the query the runtime -// exposes rather than a typed property, so a renamed key answers nothing instead of raising. Returns -// nil when the query itself cannot be answered, which a caller reads as an application with no -// windows. `reportedCount`, when asked for, is how many the query named before the cap, so a device -// the cap would silently shorten stays visible. Only the panel walk reads from this list, and it -// reads each element's attributes at most once, so a candidate the rule discards is never asked -// anything again (#2995). -FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyApplicationWindowElements( +// Resolves the `XCUIScreen` hosting the application's window and its `displayID`, the only display +// identity that follows an app to the panel it lights. Reading the window frame must precede +// reading its screen: an unresolved element still reports the main screen, and neither read is +// ever selected by number or position. Private KVC raises rather than returns nil, so both reads +// are guarded here and any failure surfaces as the typed `failure`, never an exception. +FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( id application, - NSUInteger maxCount, - NSUInteger * _Nullable reportedCount + id _Nullable *screen, + NSUInteger *displayID, + RunnerApplicationScreenFailure *failure ); -// Resolves the display hosting the application's surface window: the surface window, then the -// `XCUIScreen` that window reports and its `displayID` — the only display identity that follows an -// app to the panel it lights. Reading the window frame must precede reading its screen, and neither -// read is ever selected by number or position. A request that names no panel reads the historic first -// window and still asks that window for its screen, so single-panel captures are unchanged. -FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( +// `RunnerResolveApplicationScreen` for a multi-panel device: the window comes from +// `RunnerResolveLitPanelWindow` instead of the application's first window. +FOUNDATION_EXPORT BOOL RunnerResolveLitPanelScreen( id application, RunnerLitPanel litPanel, id _Nullable *screen, @@ -107,16 +84,6 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( RunnerApplicationScreenFailure *failure ); -// One window read: does it exist, and what box does it occupy. Both answers are required — an -// application with no window answers `frame` with `(0,0 0x0)` without raising, so only `exists` says -// whether the window is there. The frame is reported even when refused. Returns whether the window is -// there with a measurable box. -FOUNDATION_EXPORT BOOL RunnerReadWindowExistsAndFrame( - id window, - CGRect *observedFrame -); - - // Reads the display ID off an already-resolved window, keeping a synthesized record's display and // the caller's booked reference frame on the same window. The frame is still read before the // screen; a window that answers empty or raises fails closed with a refusal message naming the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 4d07c8122e..0289b60402 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -3,8 +3,6 @@ #import "RunnerAXSnapshotBridge.h" // The shared full-screen rule, from the snapshot bridge; found via HEADER_SEARCH_PATHS. #import "ForegroundApplicationGeometry.h" -// The panel rule this file's window walk applies. -#import "RunnerPanelSurface.h" #import #import #import @@ -111,69 +109,51 @@ return @"no resolved application window"; case RunnerApplicationScreenFailureUnresolvedScreen: return @"no resolved window display ID"; - case RunnerApplicationScreenFailureOffLitPanel: - return @"every application window is off the lit panel"; case RunnerApplicationScreenFailureNone: return @"application screen resolved"; } } -BOOL RunnerReadWindowExistsAndFrame(id window, CGRect *observedFrame) { +// One window read. Both answers are required: an app with no window answers `frame` with +// `(0,0 0x0)` without raising, so only `exists` says whether the window is there. The window +// object is handed back because resolving this instance is what its `screen` will answer from. +// The frame is reported even when refused, so a refusal names what it looked at. +static RunnerApplicationScreenFailure RunnerResolveUsableWindow( + id application, + id _Nullable *window, + CGRect *observedFrame +) { + __block id readWindow = nil; __block CGRect frame = CGRectNull; __block BOOL resolved = NO; NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ - NSNumber *present = [window valueForKey:@"exists"]; + id candidate = [[application valueForKey:@"windows"] valueForKey:@"firstMatch"]; + NSNumber *present = [candidate valueForKey:@"exists"]; resolved = [present isKindOfClass:NSNumber.class] && present.boolValue; - // A window that says it is not there is not asked for its frame. XCTest records an issue for a - // frame read on a vanished element, an unmuted issue ends the runner session, and the walk asks - // several candidates (#2995). - if (!resolved) { - return; - } - NSValue *frameValue = [window valueForKey:@"frame"]; + NSValue *frameValue = [candidate valueForKey:@"frame"]; [frameValue getValue:&frame size:sizeof(frame)]; + readWindow = candidate; }]; *observedFrame = frame; - if (exception != nil) { - NSLog(@"AGENT_DEVICE_RUNNER_APP_WINDOW_FRAME_RAISED frame raised=%@", exception); - return NO; - } - return resolved && AgentDeviceForegroundFrameIsMeasurable(frame); -} - -NSArray *RunnerCopyApplicationWindowElements( - id application, - NSUInteger maxCount, - NSUInteger *reportedCount -) { - if (reportedCount != NULL) { - *reportedCount = 0; - } - __block NSArray *windows = nil; - NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ - id elements = [[application valueForKey:@"windows"] valueForKey:@"allElementsBoundByIndex"]; - if ([elements isKindOfClass:NSArray.class]) { - windows = elements; - } - }]; - if (exception != nil) { - NSLog(@"AGENT_DEVICE_RUNNER_APP_WINDOWS_UNRESOLVED raised=%@", exception); - return nil; - } - if (windows == nil) { - return nil; - } - // The count is the list the query named, before the cap trims it, so a device whose window list - // outgrows the walk is visible to whoever reads the marker. - if (reportedCount != NULL) { - *reportedCount = windows.count; + if (exception != nil || !resolved || !AgentDeviceForegroundFrameIsMeasurable(frame)) { + NSLog( + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) raised=%@ resolved=%@", + frame.origin.x, + frame.origin.y, + frame.size.width, + frame.size.height, + exception != nil ? @"yes" : @"no", + resolved ? @"yes" : @"no" + ); + return RunnerApplicationScreenFailureUnresolvedWindow; } - return windows.count <= maxCount ? windows : [windows subarrayWithRange:NSMakeRange(0, maxCount)]; + *window = readWindow; + return RunnerApplicationScreenFailureNone; } // The screen read that must follow window resolution on that same window instance: before the frame // is asked for, `screen` can still name main, so this never merges into the read ahead of it. -RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( +static RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( id window, CGRect windowFrame, id _Nullable *screen, @@ -201,170 +181,157 @@ RunnerApplicationScreenFailure RunnerReadResolvedWindowScreen( ); return RunnerApplicationScreenFailureUnresolvedScreen; } - if (screen != NULL) { - *screen = readScreen; - } + *screen = readScreen; *displayID = readDisplayID; return RunnerApplicationScreenFailureNone; } -static RunnerApplicationScreenFailure RunnerSurfaceFailureFromPanelSelection( - RunnerPanelSurfaceSelection selection +static BOOL RunnerFrameFillsLitPanel(CGRect frame, RunnerLitPanel panel) { + const CGFloat tolerance = 0.5; + if (fabs(frame.origin.x) > tolerance || fabs(frame.origin.y) > tolerance) { + return NO; + } + BOOL upright = fabs(frame.size.width - panel.widthPt) <= tolerance && + fabs(frame.size.height - panel.heightPt) <= tolerance; + BOOL turned = fabs(frame.size.width - panel.heightPt) <= tolerance && + fabs(frame.size.height - panel.widthPt) <= tolerance; + return upright || turned; +} + +// One window: whether it is there, its frame, and the display showing it. `frame` is not asked of a +// window that says it is gone, because that read records an XCTest issue (#2995). +static RunnerApplicationScreenFailure RunnerReadLitPanelCandidate( + id candidate, + CGRect *frame, + id _Nullable *screen, + NSUInteger *displayID ) { - switch (selection) { - case RunnerPanelSurfaceSelectionLitPanelWindow: - case RunnerPanelSurfaceSelectionLitDisplayWindow: - return RunnerApplicationScreenFailureNone; - case RunnerPanelSurfaceSelectionOffLitPanel: - return RunnerApplicationScreenFailureOffLitPanel; - case RunnerPanelSurfaceSelectionNoWindow: - return RunnerApplicationScreenFailureUnresolvedWindow; + __block CGRect readFrame = CGRectNull; + __block BOOL present = NO; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + NSNumber *exists = [candidate valueForKey:@"exists"]; + present = [exists isKindOfClass:NSNumber.class] && exists.boolValue; + if (present) { + [[candidate valueForKey:@"frame"] getValue:&readFrame size:sizeof(readFrame)]; + } + }]; + *frame = readFrame; + if (exception != nil || !present || !AgentDeviceForegroundFrameIsMeasurable(readFrame)) { + return RunnerApplicationScreenFailureUnresolvedWindow; } + return RunnerReadResolvedWindowScreen(candidate, readFrame, screen, displayID); } -RunnerApplicationScreenFailure RunnerResolveSurfaceWindow( +static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( id application, RunnerLitPanel litPanel, id _Nullable *window, CGRect *windowFrame, - CGRect *observedFrame + id _Nullable *screen ) { *window = nil; *windowFrame = CGRectNull; - *observedFrame = CGRectNull; - - if (!RunnerLitPanelIsNamed(litPanel)) { - // The historic read, and the only one a single-panel device ever takes: the first window the - // query names that exists with a measurable box, and no display identity is asked for at all. - // The list is uncapped here for the same reason the historic read was uncapped: an app may own - // more windows than the panel walk's budget, and stopping early would refuse a screen the - // historic route resolved. Each candidate is asked twice at most, and the walk stops at the - // answer, so a transient window that vanishes mid-read is the only thing that can widen it - // (#2995). - __block CGRect refusedFrame = CGRectNull; - for (id candidate in RunnerCopyApplicationWindowElements( - application, NSUIntegerMax, NULL)) { - CGRect frame = CGRectNull; - if (!RunnerReadWindowExistsAndFrame(candidate, &frame)) { - refusedFrame = frame; - continue; - } - *window = candidate; - *windowFrame = frame; - *observedFrame = frame; - return RunnerApplicationScreenFailureNone; - } - NSLog( - @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=window frame=(%g,%g %gx%g) resolved=no", - refusedFrame.origin.x, - refusedFrame.origin.y, - refusedFrame.size.width, - refusedFrame.size.height - ); - *observedFrame = refusedFrame; + *screen = nil; + __block NSArray *windows = nil; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id elements = [[application valueForKey:@"windows"] valueForKey:@"allElementsBoundByIndex"]; + windows = [elements isKindOfClass:NSArray.class] ? elements : nil; + }]; + if (exception != nil || windows == nil) { return RunnerApplicationScreenFailureUnresolvedWindow; } - - NSUInteger reported = 0; - NSArray *candidates = RunnerCopyApplicationWindowElements( - application, - AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS, - &reported - ); - // The walk reads at most the cap's worth of windows, so it may not have seen the whole list. That - // matters only for a refusal: a refusal that claims the lit panel holds nothing is honest only - // when nothing had to be left unread. - BOOL listTruncated = candidates.count < reported; - NSMutableArray *placed = [NSMutableArray array]; - RunnerPanelWindowFact facts[AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS]; - NSUInteger factCount = 0; - BOOL sawUnplaceableWindow = NO; - for (id candidate in candidates) { - if (factCount == AGENT_DEVICE_MAX_PANEL_SURFACE_WINDOWS) { - break; - } + // A window the walk did not place could be the panel's surface, so only a fully placed list may + // book its largest window or claim the lit panel holds none. + BOOL incomplete = windows.count > AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS; + id chosen = nil; + id chosenScreen = nil; + CGRect chosenFrame = CGRectNull; + RunnerApplicationScreenFailure outcome = RunnerApplicationScreenFailureUnresolvedWindow; + NSUInteger count = MIN(windows.count, (NSUInteger)AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS); + for (NSUInteger index = 0; index < count; index += 1) { + id candidate = windows[index]; CGRect frame = CGRectNull; - if (!RunnerReadWindowExistsAndFrame(candidate, &frame)) { + id candidateScreen = nil; + NSUInteger displayID = 0; + RunnerApplicationScreenFailure read = + RunnerReadLitPanelCandidate(candidate, &frame, &candidateScreen, &displayID); + if (read == RunnerApplicationScreenFailureUnresolvedWindow) { continue; } - NSUInteger displayID = 0; - if (RunnerReadResolvedWindowScreen(candidate, frame, NULL, &displayID) != - RunnerApplicationScreenFailureNone) { - // The window is there but the runtime will not say which display shows it, so the panel rule - // cannot place it. The rest of the walk still gets its chance. - sawUnplaceableWindow = YES; - *observedFrame = frame; + if (read != RunnerApplicationScreenFailureNone) { + incomplete = YES; + continue; + } + if (displayID != litPanel.displayID) { continue; } - facts[factCount] = (RunnerPanelWindowFact){.displayID = displayID, .frame = frame}; - factCount += 1; - [placed addObject:candidate]; + if (RunnerFrameFillsLitPanel(frame, litPanel)) { + chosen = candidate; + chosenScreen = candidateScreen; + chosenFrame = frame; + incomplete = NO; + break; + } + if (chosen == nil || + frame.size.width * frame.size.height > chosenFrame.size.width * chosenFrame.size.height) { + chosen = candidate; + chosenScreen = candidateScreen; + chosenFrame = frame; + } } - - RunnerPanelWindowFact chosen = {.displayID = 0, .frame = CGRectNull}; - NSUInteger chosenIndex = NSUIntegerMax; - RunnerPanelSurfaceSelection selection = RunnerPanelSurfaceSelectionNoWindow; - if (factCount > 0) { - selection = RunnerPanelSurfaceSelectWindow( - facts, - factCount, - litPanel, - &chosen, - &chosenIndex - ); + if (incomplete) { + outcome = RunnerApplicationScreenFailureUnresolvedScreen; + } else if (chosen != nil) { + outcome = RunnerApplicationScreenFailureNone; + *window = chosen; + *windowFrame = chosenFrame; + *screen = chosenScreen; } NSLog( - @"AGENT_DEVICE_RUNNER_PANEL_SURFACE lit=%lu(%.1fx%.1f) reported=%lu placed=%lu unplaceable=%@ selection=%lu frame=(%g,%g %gx%g)", + @"AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=%lu(%gx%g) windows=%lu outcome=%@ frame=(%g,%g %gx%g)", (unsigned long)litPanel.displayID, litPanel.widthPt, litPanel.heightPt, - (unsigned long)reported, - (unsigned long)factCount, - sawUnplaceableWindow ? @"yes" : @"no", - (unsigned long)selection, - chosen.frame.origin.x, - chosen.frame.origin.y, - chosen.frame.size.width, - chosen.frame.size.height + (unsigned long)windows.count, + RunnerApplicationScreenFailureDescription(outcome), + chosenFrame.origin.x, + chosenFrame.origin.y, + chosenFrame.size.width, + chosenFrame.size.height ); - // A refusal names what the walk looked at: the frame the rule held for the list it was given. - if (factCount > 0) { - *observedFrame = chosen.frame; - } - // A claim about the whole window list is honest only when the walk read the whole list. An - // unplaceable window, or a list the cap cut short, leaves part of it unread, and the unread part - // could hold the lit panel's surface. Both of the rule's answers that depend on the rest of the - // list refuse here: `offLitPanel`, which says the lit panel holds nothing, and `litDisplayWindow`, - // which books the largest window the lit display names — with an unread tail, the panel's own - // full-screen window may be the one that went unread and the largest survivor a strip in a corner. - // A gesture booked to the right panel at the wrong box is the failure this fix exists to remove. - BOOL listUnread = sawUnplaceableWindow || listTruncated; - if (listUnread && - (selection == RunnerPanelSurfaceSelectionOffLitPanel || - selection == RunnerPanelSurfaceSelectionLitDisplayWindow)) { - return RunnerApplicationScreenFailureUnresolvedScreen; - } - RunnerApplicationScreenFailure failure = RunnerSurfaceFailureFromPanelSelection(selection); - if (failure != RunnerApplicationScreenFailureNone) { - if (factCount == 0 && sawUnplaceableWindow) { - return RunnerApplicationScreenFailureUnresolvedScreen; - } - return failure; - } - *observedFrame = chosen.frame; - // The rule indexes the windows it was handed, so this cannot trip; it bounds the subscript that - // follows rather than trusting the index across the rule boundary. - if (chosenIndex >= placed.count) { - return RunnerApplicationScreenFailureUnresolvedWindow; + return outcome; +} + +RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( + id application, + RunnerLitPanel litPanel, + id _Nullable *window, + CGRect *windowFrame +) { + id screen = nil; + return RunnerResolveLitPanelWindowScreen(application, litPanel, window, windowFrame, &screen); +} + +BOOL RunnerResolveLitPanelScreen( + id application, + RunnerLitPanel litPanel, + id _Nullable *screen, + NSUInteger *displayID, + RunnerApplicationScreenFailure *failure +) { + *displayID = 0; + id window = nil; + CGRect windowFrame = CGRectNull; + *failure = RunnerResolveLitPanelWindowScreen(application, litPanel, &window, &windowFrame, screen); + if (*failure != RunnerApplicationScreenFailureNone) { + return NO; } - *window = placed[chosenIndex]; - *windowFrame = chosen.frame; - return RunnerApplicationScreenFailureNone; + *displayID = litPanel.displayID; + return YES; } BOOL RunnerResolveApplicationScreen( id application, - RunnerLitPanel litPanel, id _Nullable *screen, NSUInteger *displayID, RunnerApplicationScreenFailure *failure @@ -373,8 +340,7 @@ BOOL RunnerResolveApplicationScreen( *displayID = 0; id window = nil; CGRect windowFrame = CGRectNull; - CGRect observedFrame = CGRectNull; - *failure = RunnerResolveSurfaceWindow(application, litPanel, &window, &windowFrame, &observedFrame); + *failure = RunnerResolveUsableWindow(application, &window, &windowFrame); if (*failure != RunnerApplicationScreenFailureNone) { return NO; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift deleted file mode 100644 index 6e73272bec..0000000000 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerPanelSurfaceTreeTests.swift +++ /dev/null @@ -1,237 +0,0 @@ -import XCTest -import AgentDeviceSnapshotPresentation - -#if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - What one panel's fact says about a captured tree (ADR 0025) - -extension RunnerTests { - /// The Duo's lit inner panel as the interaction space reports it: the panel table names 669x951 and - /// the system presents that panel turned, so its surface box swaps the side lengths. - private static let litSurface = CGRect(x: 0, y: 0, width: 951, height: 669) - /// SpringBoard's outer-panel window, which the platform handed back as the app's only window on the - /// run that found this bug. The device was unfolded, so nothing in it can be seen. - private static let offPanelWindow = CGRect(x: 0, y: 0, width: 466, height: 678) - - private static func panelNode( - _ index: Int, - type: String, - rect: CGRect, - parentIndex: Int? - ) -> RawAXNode { - RawAXNode( - index: index, - type: type, - label: nil, - identifier: nil, - value: nil, - rect: SnapshotRect(rect), - enabled: true, - focused: nil, - selected: nil, - hittable: parentIndex == nil ? false : true, - depth: parentIndex == nil ? 0 : 1, - parentIndex: parentIndex, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - } - - /// The tree the bug produced: an app root naming the dark panel, one window naming the dark panel, and - /// content under it. - private static func darkPanelOnlyTree() -> [RawAXNode] { - [ - panelNode(0, type: "Application", rect: Self.offPanelWindow, parentIndex: nil), - panelNode(1, type: "Window", rect: Self.offPanelWindow, parentIndex: 0), - panelNode(2, type: "Button", rect: CGRect(x: 20, y: 120, width: 250, height: 42), parentIndex: 1), - ] - } - - func testPanelBoxMatchesItsOwnBoxAndThatBoxTurned() { - XCTAssertTrue( - RunnerPanelSurfaceTree.matchesSurfaceBox(Self.litSurface, Self.litSurface), - "the panel's own box is its surface" - ) - XCTAssertTrue( - RunnerPanelSurfaceTree.matchesSurfaceBox( - CGRect(x: 0, y: 0, width: 669, height: 951), - Self.litSurface - ), - "the panel table reports native points and the system presents the panel turned" - ) - XCTAssertFalse( - RunnerPanelSurfaceTree.matchesSurfaceBox(Self.offPanelWindow, Self.litSurface), - "another panel's box is not this panel's surface" - ) - XCTAssertFalse( - RunnerPanelSurfaceTree.matchesSurfaceBox( - CGRect(x: 0, y: 0, width: 951, height: 600), - Self.litSurface - ), - "a window that stops short of the panel is an app occupying part of it, not the surface" - ) - } - - func testASurfaceLiesOnThePanelItCanBeSeenOn() { - XCTAssertTrue( - RunnerPanelSurfaceTree.liesOnSurface(CGRect(x: 0, y: 0, width: 951, height: 669), Self.litSurface) - ) - XCTAssertTrue( - RunnerPanelSurfaceTree.liesOnSurface( - CGRect(x: 100, y: 50, width: 300, height: 200), - Self.litSurface - ), - "a sheet inside the panel's box is on the panel" - ) - XCTAssertFalse( - RunnerPanelSurfaceTree.liesOnSurface(Self.offPanelWindow, Self.litSurface), - "a box reaching past the panel's bottom edge is on another display" - ) - XCTAssertFalse( - RunnerPanelSurfaceTree.liesOnSurface(CGRect(x: 0, y: 0, width: 0, height: 669), Self.litSurface), - "a box with no width names no surface at all" - ) - } - - /// The root node is the box the host reads as the payload's viewport, so an app root that reports a - /// panel's box is republished with the surface box the capture resolved. - func testPanelTreePublishesTheSurfaceBoxOnItsRoot() throws { - let published = try XCTUnwrap( - RunnerPanelSurfaceTree.publishing(nodes: Self.darkPanelOnlyTree(), surface: Self.litSurface) - ) - XCTAssertTrue(published.republishedRoot) - XCTAssertEqual(published.droppedSurfaces, 1) - XCTAssertEqual(published.nodes.count, 1, "the dark panel's window and its content are both gone") - XCTAssertEqual(published.nodes[0].rect.cgRect, Self.litSurface) - XCTAssertNil(published.nodes[0].parentIndex) - } - - /// Dropping a surface drops its subtree: a ref for a control on a dark panel would name a point no - /// gesture can land on, and the host's own containment rule refuses the node anyway. - func testPanelTreeDropsAWindowsWholeSubtree() throws { - var nodes = Self.darkPanelOnlyTree() - nodes.append( - Self.panelNode( - 3, - type: "Cell", - rect: CGRect(x: 20, y: 170, width: 250, height: 42), - parentIndex: 1 - ) - ) - nodes.append( - Self.panelNode( - 4, - type: "Window", - rect: Self.litSurface, - parentIndex: 0 - ) - ) - nodes.append( - Self.panelNode( - 5, - type: "Button", - rect: CGRect(x: 470, y: 54, width: 60, height: 60), - parentIndex: 4 - ) - ) - let published = try XCTUnwrap(RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface)) - XCTAssertEqual(published.droppedSurfaces, 1) - XCTAssertEqual( - published.nodes.map(\.type), - ["Application", "Window", "Button"], - "the lit panel's window and its control survive" - ) - XCTAssertEqual(published.nodes.map(\.index), Array(0..<3), "the wire array stays positional") - XCTAssertEqual(published.nodes[1].parentIndex, 0, "the kept window is still the app's child") - XCTAssertEqual(published.nodes[2].parentIndex, 1, "the kept control is still its window's child") - XCTAssertEqual( - published.nodes[2].depth, - nodes[5].depth, - "whole siblings were removed, so no kept node changed depth" - ) - } - - /// The common case must cost nothing: an app whose window fills the lit panel already publishes the - /// right root box, and there is nothing to drop. - func testPanelTreeLeavesATreeOfTheLitPanelAlone() { - let nodes = [ - Self.panelNode(0, type: "Application", rect: Self.litSurface, parentIndex: nil), - Self.panelNode(1, type: "Window", rect: Self.litSurface, parentIndex: 0), - ] - XCTAssertNil(RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface)) - } - - /// A keyboard band, an alert, or a hosted surface that reaches past the panel's edge is content the - /// fold clips, so only a root's own children are ever dropped. - func testPanelTreeKeepsContentThatReachesPastThePanel() { - let nodes = [ - Self.panelNode(0, type: "Application", rect: Self.litSurface, parentIndex: nil), - Self.panelNode(1, type: "Window", rect: Self.litSurface, parentIndex: 0), - Self.panelNode( - 2, - type: "ScrollView", - rect: CGRect(x: -57, y: -40, width: 1065, height: 749), - parentIndex: 1 - ), - ] - XCTAssertNil(RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface)) - } - - /// The private-AX tier's tree and a scoped capture are rooted at what they read rather than at an - /// application element. Such a payload was read from one surface, so its nodes are left as they were - /// read: narrowing them would be a guess about which screen they came from. - func testPanelTreeLeavesATreeNotRootedAtAnApplicationAlone() { - for nodes in [ - [Self.panelNode(0, type: "Window", rect: Self.offPanelWindow, parentIndex: nil)], - [ - Self.panelNode(0, type: "Other", rect: Self.offPanelWindow, parentIndex: nil), - Self.panelNode(1, type: "Window", rect: Self.offPanelWindow, parentIndex: 0), - ], - ] { - XCTAssertNil( - RunnerPanelSurfaceTree.publishing(nodes: nodes, surface: Self.litSurface), - "no application root, so no window list to narrow" - ) - } - } - - /// The capture's own evidence that its tree already shows the lit panel, which is what decides whether - /// a second read of the resolved window is needed at all. - func testCarriesSurfaceWindowReadsOneLayerOnly() { - struct Layer { - let type: String - let rect: CGRect - } - let layers = [ - Layer(type: "Other", rect: Self.litSurface), - Layer(type: "Window", rect: Self.offPanelWindow), - ] - XCTAssertFalse( - RunnerPanelSurfaceTree.carriesSurfaceWindow( - layers, - surface: Self.litSurface, - isSurface: { $0.type == "Window" }, - frame: { $0.rect } - ), - "the only window is on another panel" - ) - XCTAssertTrue( - RunnerPanelSurfaceTree.carriesSurfaceWindow( - layers + [Layer(type: "Window", rect: CGRect(x: 0, y: 0, width: 669, height: 951))], - surface: Self.litSurface, - isSurface: { $0.type == "Window" }, - frame: { $0.rect } - ), - "a turned window on the panel counts" - ) - XCTAssertFalse( - RunnerPanelSurfaceTree.carriesSurfaceWindow( - [Layer(type: "Other", rect: Self.litSurface)], - surface: Self.litSurface, - isSurface: { $0.type == "Window" }, - frame: { $0.rect } - ), - "a container is not a surface" - ) - } -} -#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 57f55802f3..1d604a8c20 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -1,49 +1,31 @@ import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS -// MARK: - The request-scoped lit panel (#2728, ADR 0025) - extension RunnerTests { - /// The panel a command names is the panel that command's resolvers see, and a request that names - /// none clears whatever the previous command left behind. One runner process can serve a foldable's - /// `tap` and then a single-panel device's, and the second must not resolve against the first's panel. + /// One runner process can serve a foldable's `tap` and then a single-panel request; the second + /// must not resolve against the first's panel. func testRequestPanelFollowsTheCommandAndAnUnnamedPanelClearsIt() throws { setRequestLitPanel(from: try command(namingLitPanel: #"{"displayId":3,"widthPt":669,"heightPt":951}"#)) - XCTAssertEqual(requestSurfacePanel.displayID, 3) - XCTAssertEqual(requestSurfacePanel.widthPt, 669) - XCTAssertEqual(requestSurfacePanel.heightPt, 951) + XCTAssertEqual(requestSurfacePanel?.displayID, 3) + XCTAssertEqual(requestSurfacePanel?.widthPt, 669) + XCTAssertEqual(requestSurfacePanel?.heightPt, 951) setRequestLitPanel(from: try command(namingLitPanel: nil)) - XCTAssertEqual( - requestSurfacePanel.displayID, - 0, - "a request that names no panel must not inherit the previous command's" - ) + XCTAssertNil(requestSurfacePanel) } - /// A panel that cannot describe a box cannot be compared with a window, so it answers as no panel - /// and the resolver keeps the historic route rather than refusing every window it reads. The rows - /// are the ways a panel table can report a panel without naming one: no display, or a display whose - /// box has no size. func testRequestPanelTreatsAPanelNamingNoUsableBoxAsNoPanel() throws { for panel in [ #"{"displayId":0,"widthPt":669,"heightPt":951}"#, #"{"displayId":3,"widthPt":0,"heightPt":951}"#, - #"{"displayId":3,"widthPt":669,"heightPt":0}"#, - #"{"displayId":3,"widthPt":-1,"heightPt":-951}"#, + #"{"displayId":3,"widthPt":669,"heightPt":-951}"#, ] { - setRequestLitPanel(from: try command(namingLitPanel: nil)) setRequestLitPanel(from: try command(namingLitPanel: panel)) - XCTAssertEqual( - requestSurfacePanel.displayID, - 0, - "display 0 is what every resolver reads as the historic first-window route (\(panel))" - ) + XCTAssertNil(requestSurfacePanel, panel) } } - /// Built from JSON, because the slot's input is the wire shape the host sends; a Swift literal would - /// keep compiling if the field were renamed. + /// Decoded from JSON, so a renamed wire field fails here. private func command(namingLitPanel panel: String?) throws -> Command { let fields = ["\"command\":\"tap\"", panel.map { "\"litPanel\":\($0)" }].compactMap { $0 } return try JSONDecoder().decode( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift index 08ab26edd5..74b65986a2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift @@ -49,12 +49,6 @@ extension RunnerTests { RunnerAppScreenCaptureFailure.unrenderableImage.rawValue, "APP_SCREEN_CAPTURE_UNRENDERABLE" ) - // A panel refusal has a system surface left to ask, so it is its own code too: "the app has - // nothing on the lit panel" and "no window anywhere" send an operator to different places. - XCTAssertEqual( - RunnerAppScreenCaptureFailure(.offLitPanel).rawValue, - "APP_SCREEN_OFF_LIT_PANEL" - ) // A success-shaped reason cannot become a success: the only way here is a caller that forgot to // branch, and the capture it describes did not happen. XCTAssertEqual( @@ -124,30 +118,6 @@ extension RunnerTests { ) } - /// Every window the session app has is on the dark panel: that says nothing about where the lit - /// panel's content is, because the home screen is SpringBoard's window. So the panel refusal moves - /// the question on exactly like an unresolved window does. - func testObservedScreenCaptureAsksTheSystemSurfaceWhenEveryAppWindowIsOffTheLitPanel() { - var askedSystemSurface = false - let outcome = selectObservedScreenCapture( - resolving: { .failure(.offLitPanel) }, - fallingBack: { - askedSystemSurface = true - return .success( - CapturedAppScreen( - image: UIImage(), - displayID: 3, - pixelWidth: 2852, - pixelHeight: 2006, - pixelsPerPoint: 3 - ) - ) - } - ) - XCTAssertTrue(askedSystemSurface, "the lit panel's content belongs to another process") - XCTAssertEqual(try? outcome.get().displayID, 3) - } - /// A window that resolved and then refused to name its display is the failure the host has to /// see; asking a second process would replace a real refusal with an unrelated capture. func testObservedScreenCaptureReportsADisplayRefusalWithoutAskingTheSystemSurface() { @@ -159,10 +129,7 @@ extension RunnerTests { return .failure(.unrenderableImage) } ) - XCTAssertFalse( - askedSystemSurface, - "only an unresolved window or a panel refusal may reach the system surface" - ) + XCTAssertFalse(askedSystemSurface, "only an unresolved window may reach the system surface") guard case .failure(let refusal) = outcome else { return XCTFail("a display that refused must not become a capture") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift index 6b5dcb0ae0..e71ab44527 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -392,7 +392,6 @@ extension RunnerTests { let context = try XCTUnwrap(makeSnapshotTraversalContext(app: app, hint: hint)) let startedAt = Date() let capture = try recursiveTreeSnapshotAcquisition( - app: app, context: context, hint: hint, processingDeadline: startedAt.addingTimeInterval(treeCaptureSliceBudget) @@ -431,11 +430,10 @@ extension RunnerTests { let context = try XCTUnwrap(makeSnapshotTraversalContext(app: app, hint: hint)) let tabBar = try XCTUnwrap(Self.snapshotDescendants(of: context.rootSnapshot).first { $0.elementType == .tabBar }) XCTAssertTrue(tabBar.children.isEmpty, "the fixture must exercise collapsed-tab recovery") - let unstarted = try recursiveTreeSnapshotAcquisition(app: app, context: context, hint: hint, processingDeadline: Date()) + let unstarted = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: Date()) XCTAssertEqual(unstarted.outcome, .deadlineExhausted) XCTAssertTrue(unstarted.acquisition.truncated) let capture = try recursiveTreeSnapshotAcquisition( - app: app, context: context, hint: hint, processingDeadline: Date().addingTimeInterval(treeCaptureSliceBudget) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index b46575da06..9fb3383b79 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -32,15 +32,12 @@ private final class DisplayWindowFixture: NSObject { private final class DisplayWindowsFixture: NSObject { @objc let firstMatch: NSObject + init(_ window: NSObject) { firstMatch = window } +} + +private final class WindowListFixture: NSObject { @objc let allElementsBoundByIndex: [NSObject] - init(_ window: NSObject) { - firstMatch = window - allElementsBoundByIndex = [window] - } - init(windows: [NSObject]) { - firstMatch = windows.first ?? NSObject() - allElementsBoundByIndex = windows - } + init(_ windows: [NSObject]) { allElementsBoundByIndex = windows } } private final class DisplayApplicationFixture: NSObject { @@ -68,101 +65,6 @@ private final class FramelessWindowFixture: NSObject { } extension RunnerTests { - /// The no-panel request a single-panel device sends. Every existing case below reads this: a - /// request that names no panel keeps the historic first-window route. - static var noPanelFixture: RunnerLitPanel { RunnerLitPanel(displayID: 0, widthPt: 0, heightPt: 0) } - - /// The Duo's lit inner panel as a request names it: the panel table reports 669x951 native points. - static var duoInnerPanel: RunnerLitPanel { RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) } - - /// The inner panel's home-screen window as the runtime reports it: presented turned, so the - /// side lengths swap and it reads 951x669. - static let duoHomeScreenFrame = CGRect(x: 0, y: 0, width: 951, height: 669) - - fileprivate static func duoHomeScreenWindow() -> DisplayWindowFixture { - DisplayWindowFixture(frame: duoHomeScreenFrame, displayID: 3) - } - - /// SpringBoard's first reported window: the 226x155.3 status-bar window, on the outer panel. - fileprivate static func springBoardStatusBarWindow() -> DisplayWindowFixture { - DisplayWindowFixture(frame: CGRect(x: 452, y: 0, width: 226, height: 155.3), displayID: 1) - } - - /// `count` full-screen windows on the outer panel, which is the dark one while the inner panel is - /// lit. SpringBoard really names dozens, so a case that needs the walk's budget spent uses this. - fileprivate static func outerPanelWindows(_ count: Int) -> [NSObject] { - (0.. DisplayWindowFixture { - DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 226, height: 155.3), displayID: 3) - } - - /// A folded Duo's home screen seen while the inner panel is lit: every window it names is on the - /// outer panel, so nothing here may become the surface. - fileprivate static func darkPanelOnlyWindows() -> [NSObject] { - [springBoardStatusBarWindow()] + outerPanelWindows(1) - } - - fileprivate static func application(_ windows: [NSObject]) -> DisplayApplicationFixture { - DisplayApplicationFixture(windows: DisplayWindowsFixture(windows: windows)) - } - - /// One screen read through the full route, holding the three in-out boxes every caller declares. - /// `screenSeed` lets a refusal case prove the resolver clears a screen object it was handed. - fileprivate func resolveScreen( - _ application: NSObject, - litPanel: RunnerLitPanel, - screenSeed: AnyObject? = nil - ) -> (succeeded: Bool, screen: AnyObject?, displayID: UInt, failure: RunnerApplicationScreenFailure) { - var screen = screenSeed - var displayID: UInt = 0 - var failure = RunnerApplicationScreenFailure.none - let succeeded = RunnerResolveApplicationScreen( - application, litPanel, &screen, &displayID, &failure - ) - return (succeeded, screen, displayID, failure) - } - - /// One screen read that must refuse, and the three facts every refusal holds: the typed failure, - /// no screen object, and no display named. - fileprivate func assertScreenReadRefuses( - _ application: NSObject, - expected: RunnerApplicationScreenFailure, - litPanel: RunnerLitPanel, - screenMessage: String? = nil - ) { - let read = resolveScreen(application, litPanel: litPanel, screenSeed: NSObject()) - XCTAssertFalse(read.succeeded) - XCTAssertEqual(read.failure, expected) - if let screenMessage { - XCTAssertNil(read.screen, screenMessage) - } else { - XCTAssertNil(read.screen) - } - XCTAssertEqual(read.displayID, 0, "a refusal names no display") - } - - /// One surface walk, holding the same in-out boxes, so a case states only the shape it feeds and - /// the window it expects back. - fileprivate func resolveSurfaceWindow( - _ application: NSObject, - litPanel: RunnerLitPanel - ) -> (failure: RunnerApplicationScreenFailure, window: AnyObject?, windowFrame: CGRect, observedFrame: CGRect) { - var window: AnyObject? - var windowFrame = CGRect.null - var observedFrame = CGRect.null - let failure = RunnerResolveSurfaceWindow( - application, litPanel, &window, &windowFrame, &observedFrame - ) - return (failure, window, windowFrame, observedFrame) - } - - func testSynthesizedDisplayRoutesByTheResolvedWindow() { let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 3) var displayID: UInt = 0 @@ -214,357 +116,98 @@ extension RunnerTests { } func testResolvedScreenHandsBackTheScreenTheResolvedWindowReported() { - let window = Self.duoHomeScreenWindow() - let read = resolveScreen(DisplayApplicationFixture(window), litPanel: Self.noPanelFixture) - XCTAssertTrue(read.succeeded) - XCTAssertEqual(read.displayID, 3) + let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 951, height: 669), displayID: 3) + var screen: AnyObject? + var displayID: UInt = 0 + var failure = RunnerApplicationScreenFailure.unresolvedScreen + XCTAssertTrue( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) + ) + XCTAssertEqual(displayID, 3) XCTAssertTrue(window.resolved, "the window's screen only answers once its query resolved") XCTAssertTrue( - read.screen === window.screen, + screen === window.screen, "the capture must go to the screen object this window reported, not a re-queried one" ) } func testResolvedScreenRefusesAWindowLookupThatRaises() { - assertScreenReadRefuses( - DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), - expected: .unresolvedWindow, - litPanel: Self.noPanelFixture + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), + &screen, + &displayID, + &failure + ) ) + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) } func testResolvedScreenRefusesAResolvedWindowWithNoAnswerableScreen() { - assertScreenReadRefuses( - DisplayApplicationFixture(ScreenlessWindowFixture()), - expected: .unresolvedScreen, - litPanel: Self.noPanelFixture + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen( + DisplayApplicationFixture(ScreenlessWindowFixture()), + &screen, + &displayID, + &failure + ) ) + XCTAssertEqual(failure, .unresolvedScreen) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) } func testResolvedScreenRefusesAWindowReportingNoDisplayID() { - let outerPanel = DisplayWindowFixture( - frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 0) - assertScreenReadRefuses( - DisplayApplicationFixture(outerPanel), expected: .unresolvedScreen, litPanel: Self.noPanelFixture + let window = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 0) + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) ) + XCTAssertEqual(failure, .unresolvedScreen) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) } func testResolvedScreenRefusesAnEmptyWindowFrameWithoutNamingAnyDisplay() { - assertScreenReadRefuses( - DisplayApplicationFixture(DisplayWindowFixture(frame: .zero, displayID: 3)), - expected: .unresolvedWindow, - litPanel: Self.noPanelFixture, - screenMessage: "a dark panel must never become the capture target by default" + let window = DisplayWindowFixture(frame: .zero, displayID: 3) + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none + XCTAssertFalse( + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) ) + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertNil(screen, "a dark panel must never become the capture target by default") } /// A cold query answers `frame` with a usable rect on some runtimes while its `exists` says the /// window is not there. Geometry alone must not be trusted into capturing a panel nobody resolved. func testResolvedScreenRefusesAWindowTheQuerySaysIsNotThere() { - let absent = DisplayWindowFixture( - frame: Self.duoHomeScreenFrame, displayID: 3, present: false) - assertScreenReadRefuses( - DisplayApplicationFixture(absent), expected: .unresolvedWindow, litPanel: Self.noPanelFixture - ) - } - - /// #2995: a `frame` read on an element the query says is not there can make XCTest record an issue, - /// and an unmuted issue ends the runner session. The walk reads `exists` and stops there, on both - /// routes, so a candidate that vanished between listing and reading costs one read, not two. - func testSurfaceWalkNeverAsksAnAbsentWindowForItsFrame() { - for panel in [Self.noPanelFixture, Self.duoInnerPanel] { - let absent = DisplayWindowFixture( - frame: Self.duoHomeScreenFrame, displayID: 3, present: false) - let walk = resolveSurfaceWindow(Self.application([absent]), litPanel: panel) - XCTAssertEqual(walk.failure, .unresolvedWindow) - XCTAssertFalse( - absent.resolved, - "an absent window must not be asked for its frame (panel=\(panel.displayID))" - ) - } - } - - /// The panel rule's own decision, from facts alone: no queries, no device, no clock. The rows are - /// the shapes a foldable home screen actually presents — SpringBoard's status-bar window is a - /// 226x155.3 box, and the inner panel reports 951x669 for the 669x951 the panel table names. - func testPanelSurfaceRuleSelectsTheWindowOnTheLitPanel() { - let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) - let statusBar = RunnerPanelWindowFact( - displayID: 1, frame: CGRect(x: 452, y: 0, width: 226, height: 155.3)) - let home = RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 0, width: 951, height: 669)) - var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) - var index: UInt = UInt.max - - XCTAssertEqual( - RunnerPanelSurfaceSelectWindow([statusBar, home], 2, lit, &selected, &index), - .litPanelWindow - ) - XCTAssertEqual(index, 1, "the window on the lit panel wins over the one listed first") - XCTAssertEqual(selected.displayID, 3) - XCTAssertEqual(selected.frame, home.frame) - } - - func testPanelSurfaceRuleAcceptsAPanelPresentedTurned() { - // The panel table names the inner panel 669x951; the system presents it turned, so its window - // reports 951x669. Both ways up are the same panel. - let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) - let upright = RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 0, width: 669, height: 951)) - let turned = RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 0, width: 951, height: 669)) - for candidate in [upright, turned] { - let facts = [candidate] - var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) - var index: UInt = UInt.max - XCTAssertEqual( - RunnerPanelSurfaceSelectWindow(facts, 1, lit, &selected, &index), - .litPanelWindow, - "expected \(candidate.frame) to qualify as the lit panel's surface" - ) - XCTAssertEqual(selected.frame, candidate.frame) - XCTAssertEqual(index, 0) - } - } - - func testPanelSurfaceRuleRefusesWindowsThatAreAllOffTheLitPanel() { - // A full-screen window on the *dark* panel is the trap: geometry alone would happily book a - // gesture on a screen nobody is looking at. - let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) - let facts = [ - RunnerPanelWindowFact(displayID: 1, frame: CGRect(x: 452, y: 0, width: 226, height: 155.3)), - RunnerPanelWindowFact(displayID: 1, frame: CGRect(x: 0, y: 0, width: 466, height: 678)), - ] - var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) - var index: UInt = UInt.max - XCTAssertEqual( - RunnerPanelSurfaceSelectWindow(facts, 2, lit, &selected, &index), - .offLitPanel - ) - XCTAssertEqual(selected.displayID, 1, "a refusal reports the first window it looked at") - XCTAssertEqual(index, 0) - } - - func testPanelSurfaceRuleBooksTheLargestWindowPlacedOnTheLitPanel() { - // An app that occupies part of the lit panel (Split View, a stage, a sheet) has no box equal to - // the panel's, and being on the lit panel is still the fact a first-window walk could not - // establish. The status-bar window is on the same display and loses on size. - let lit = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) - let facts = [ - RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 452, y: 0, width: 226, height: 155.3)), - RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 0, y: 90, width: 300, height: 400)), - RunnerPanelWindowFact(displayID: 3, frame: CGRect(x: 20, y: 20, width: 120, height: 900)), - ] - var selected = RunnerPanelWindowFact(displayID: 0, frame: .null) - var index: UInt = UInt.max - XCTAssertEqual( - RunnerPanelSurfaceSelectWindow(facts, 3, lit, &selected, &index), - .litDisplayWindow - ) - XCTAssertEqual(index, 1) - XCTAssertEqual(selected.frame, CGRect(x: 0, y: 90, width: 300, height: 400)) - } - - func testPanelSurfaceRuleRefusesNoWindows() { - let facts: [RunnerPanelWindowFact] = [] - var selected = RunnerPanelWindowFact(displayID: 7, frame: CGRect(x: 1, y: 2, width: 3, height: 4)) - var index: UInt = UInt.max - XCTAssertEqual( - RunnerPanelSurfaceSelectWindow( - facts, 0, RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951), &selected, &index), - .noWindow - ) - XCTAssertTrue(CGRectIsNull(selected.frame), "a refusal with nothing to report names no frame") - XCTAssertEqual(index, UInt.max) - } - - func testPanelSurfaceRuleRequiresTheOriginAndRejectsUnmeasurableFrames() { - let lit = RunnerLitPanel(displayID: 1, widthPt: 466, heightPt: 678) - XCTAssertTrue( - RunnerPanelFrameIsLitPanelSurface(CGRect(x: 0, y: 0, width: 466, height: 678), lit) - ) - // Half a point is rounding; a whole point is a different box. - XCTAssertTrue( - RunnerPanelFrameIsLitPanelSurface(CGRect(x: 0.4, y: -0.4, width: 466, height: 678), lit) - ) - XCTAssertFalse( - RunnerPanelFrameIsLitPanelSurface(CGRect(x: 1, y: 0, width: 466, height: 678), lit) - ) - // The same size in the wrong place, and a panel's size on the wrong panel's shape. - XCTAssertFalse( - RunnerPanelFrameIsLitPanelSurface(CGRect(x: 452, y: 0, width: 466, height: 678), lit) + let window = DisplayWindowFixture( + frame: CGRect(x: 0, y: 0, width: 951, height: 669), + displayID: 3, + present: false ) + var screen: AnyObject? = NSObject() + var displayID: UInt = 99 + var failure = RunnerApplicationScreenFailure.none XCTAssertFalse( - RunnerPanelFrameIsLitPanelSurface(CGRect(x: 0, y: 0, width: 223, height: 317), lit) - ) - for frame in [CGRect.zero, .null, .infinite] { - XCTAssertFalse(RunnerPanelFrameIsLitPanelSurface(frame, lit), "\(frame) is not a measurement") - } - } - - /// The walk the four surface paths run: with a panel, the app's first window is not the answer. - /// The foldable home screen below is the shape that made a long press open nothing — the runtime - /// lists SpringBoard's status-bar window first, on the dark panel. - func testSurfaceWindowWalkWithAPanelSkipsTheFirstWindowOnTheDarkPanel() { - let statusBar = Self.springBoardStatusBarWindow() - let home = Self.duoHomeScreenWindow() - let walk = resolveSurfaceWindow(Self.application([statusBar, home]), litPanel: Self.duoInnerPanel) - XCTAssertEqual(walk.failure, .none) - XCTAssertTrue(walk.window === home, "the surface is the window on the lit panel") - XCTAssertEqual(walk.windowFrame, Self.duoHomeScreenFrame) - } - - /// A request that names no panel is a single-panel device's request: the walk stops at the first - /// window the query names and never asks a window which display it is on. - func testSurfaceWindowWalkWithoutAPanelKeepsTheFirstWindowAndReadsNoDisplay() { - let first = DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 390, height: 844), displayID: 1) - let walk = resolveSurfaceWindow(Self.application([first]), litPanel: Self.noPanelFixture) - XCTAssertEqual(walk.failure, .none) - XCTAssertTrue(walk.window === first) - XCTAssertEqual(walk.windowFrame, CGRect(x: 0, y: 0, width: 390, height: 844)) - } - - func testSurfaceWindowWalkRefusesAWindowOnTheDarkPanel() { - let walk = resolveSurfaceWindow( - Self.application(Self.darkPanelOnlyWindows()), litPanel: Self.duoInnerPanel) - XCTAssertEqual(walk.failure, .offLitPanel, "no window is on the lit panel, so nothing may be booked") - XCTAssertNil(walk.window, "a dark panel must never become the surface by default") - XCTAssertEqual(walk.windowFrame, .null) - } - - /// A window whose display the runtime will not name cannot be placed on a panel. It is skipped, - /// and a window that does answer still resolves. - func testSurfaceWindowWalkSkipsAWindowThatWillNotNameItsDisplay() { let home = Self.duoHomeScreenWindow() - let walk = resolveSurfaceWindow( - Self.application([ScreenlessWindowFixture(), home]), litPanel: Self.duoInnerPanel) - XCTAssertEqual(walk.failure, .none) - XCTAssertTrue(walk.window === home) - } - - /// When every window that has a frame refuses to name its display, the walk has nothing to place, - /// and the host must see the display refusal rather than a window it can retry. - func testSurfaceWindowWalkReportsADisplayRefusalWhenNoWindowCanBePlaced() { - let walk = resolveSurfaceWindow( - Self.application([ScreenlessWindowFixture()]), litPanel: Self.duoInnerPanel) - XCTAssertEqual(walk.failure, .unresolvedScreen) - XCTAssertNil(walk.window) - } - - /// `offLitPanel` says the lit panel holds nothing, which is a claim about the whole window list. A - /// window the runtime would not place leaves part of that list unread, so the walk reports the - /// display refusal it can support instead of a panel it only partly saw. - func testSurfaceWindowWalkReportsADisplayRefusalWhenAnUnreadableWindowRemains() { - let walk = resolveSurfaceWindow( - Self.application(Self.darkPanelOnlyWindows() + [ScreenlessWindowFixture()]), - litPanel: Self.duoInnerPanel - ) - // An unreadable window could be the lit panel's surface, so nothing may be claimed about it. - XCTAssertEqual(walk.failure, .unresolvedScreen) - XCTAssertNil(walk.window) - } - - /// `litDisplayWindow` books the largest window the lit display names. When the query named more - /// windows than the walk may read, the panel's own full-screen window could be the one left unread, - /// and the largest survivor could be a strip in a corner. That is the right panel at the wrong box, - /// so an unread list refuses instead of booking there. - func testSurfaceWindowWalkRefusesAWindowOnTheLitPanelWhenTheListWasCutShort() { - // Seven full-screen windows on the dark panel, then a strip the lit panel does name. The cap - // reads all eight and never reaches the ninth and tenth, where the lit panel's own full-screen - // window would sit. - let unreadHome = DisplayWindowFixture( - frame: Self.duoHomeScreenFrame, displayID: Self.duoInnerPanel.displayID) - let walk = resolveSurfaceWindow( - Self.application( - Self.outerPanelWindows(7) + [Self.innerPanelStripWindow(), unreadHome, unreadHome]), - litPanel: Self.duoInnerPanel - ) - XCTAssertEqual( - walk.failure, .unresolvedScreen, - "the lit panel's surface could be the window the cap left unread" - ) - XCTAssertNil(walk.window) - } - - /// The same list the case above refuses, short enough that the walk read all of it: the strip is - /// then the honest answer, which is what makes the refusal above about the unread windows and - /// nothing else. - func testSurfaceWindowWalkBooksTheLargestWindowOnTheLitPanelWhenTheListWasReadFully() { - let walk = resolveSurfaceWindow( - Self.application(Self.outerPanelWindows(6) + [Self.innerPanelStripWindow()]), - litPanel: Self.duoInnerPanel - ) - XCTAssertEqual(walk.failure, .none) - XCTAssertEqual(walk.windowFrame, CGRect(x: 0, y: 0, width: 226, height: 155.3)) - } - - /// A request naming no panel keeps the historic uncapped read, so an app whose usable window sits - /// behind the panel walk's budget still resolves the window the historic route resolved. - func testSurfaceWindowWalkWithoutAPanelReadsPastThePanelWalkBudget() { - let unusable = (0..<9).map { _ in - DisplayWindowFixture(frame: .zero, displayID: 1) - } - let usable = DisplayWindowFixture( - frame: CGRect(x: 0, y: 0, width: 390, height: 844), displayID: 1) - let walk = resolveSurfaceWindow( - Self.application(unusable + [usable]), litPanel: Self.noPanelFixture) - XCTAssertEqual(walk.failure, .none) - XCTAssertTrue( - walk.window === usable, - "a no-panel request must not refuse a screen the historic uncapped walk resolved" - ) - } - - func testSurfaceWindowWalkRefusesAnApplicationThatAnswersNoWindows() { - for panel in [Self.noPanelFixture, Self.duoInnerPanel] { - let walk = resolveSurfaceWindow( - DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), litPanel: panel) - XCTAssertEqual(walk.failure, .unresolvedWindow) - XCTAssertNil(walk.window) - } - } - - /// The full route a capture takes: the panel decides which window's `XCUIScreen` the capture uses, - /// and it is the screen object that window reported. - func testResolvedScreenFollowsTheLitPanelNotTheFirstWindow() { - let statusBar = Self.springBoardStatusBarWindow() - let home = Self.duoHomeScreenWindow() - let read = resolveScreen( - Self.application([statusBar, home]), litPanel: Self.duoInnerPanel) - XCTAssertTrue(read.succeeded) - XCTAssertEqual(read.failure, .none) - XCTAssertEqual(read.displayID, 3, "the capture must name the lit panel, not the display listed first") - XCTAssertTrue(read.screen === home.screen) - } - - /// A capture's screen read is the one consumer that needs the `XCUIScreen` object; a caller that - /// routes by display identity alone must be able to ask without one. - func testResolvedWindowScreenAnswersDisplayIDWithoutARequestedScreen() { - let window = Self.duoHomeScreenWindow() - var displayID: UInt = 0 - XCTAssertEqual( - RunnerReadResolvedWindowScreen(window, window.frame, nil, &displayID), - .none - ) - XCTAssertEqual(displayID, 3) - } - - /// The walk is bounded: reading more windows than the cap would widen the risk of asking a window - /// that has already vanished for anything (#2995). - func testWindowElementCopyIsCappedAtThePanelWalkBudget() { - let windows: [NSObject] = (0..<12).map { - DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 100, height: 100), displayID: UInt($0 + 1)) - } - let application = DisplayApplicationFixture(windows: DisplayWindowsFixture(windows: windows)) - var reported: UInt = 0 - XCTAssertEqual(RunnerCopyApplicationWindowElements(application, 8, &reported)?.count, 8) - XCTAssertEqual( - reported, 12, "the marker reports the list the query named, not the cap that trimmed it" - ) - XCTAssertNotNil(RunnerCopyApplicationWindowElements(application, 20, &reported)) - XCTAssertNil( - RunnerCopyApplicationWindowElements( - DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), 8, nil), - "a windows query that raises answers no windows, and does not raise here" + RunnerResolveApplicationScreen(DisplayApplicationFixture(window), &screen, &displayID, &failure) ) + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertNil(screen) + XCTAssertEqual(displayID, 0) } - /// The sole-screen rule's decision, at the seam the production probe feeds: the values the /// AXRuntime probe reads in, never a device list. The windowed/unknown rows below fail without /// the fills-screen proof, on any family and on the macOS host lane, and each keeps its exact @@ -609,5 +252,101 @@ extension RunnerTests { ) } } + // MARK: - The lit panel's window (ADR 0025) + + /// The inner panel as the host names it: 669x951 native points, presented turned as 951x669. + private static let innerPanel = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + private static let innerHome = CGRect(x: 0, y: 0, width: 951, height: 669) + /// SpringBoard's first window: the status bar on the outer panel, whichever panel is lit. + private static let outerStatusBar = CGRect(x: 452, y: 0, width: 226, height: 155.3) + + private func resolveLitPanelWindow( + _ windows: [NSObject], + panel: RunnerLitPanel = innerPanel + ) -> (failure: RunnerApplicationScreenFailure, window: AnyObject?, frame: CGRect) { + var window: AnyObject? + var frame = CGRect.null + let failure = RunnerResolveLitPanelWindow( + DisplayApplicationFixture(windows: WindowListFixture(windows)), panel, &window, &frame) + return (failure, window, frame) + } + + func testLitPanelWindowSkipsTheFirstWindowOnTheDarkPanel() { + let statusBar = DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1) + let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let later = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let walk = resolveLitPanelWindow([statusBar, home, later]) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === home) + XCTAssertEqual(walk.frame, Self.innerHome) + XCTAssertFalse(later.resolved, "the walk stops at the first full-panel window") + } + + func testLitPanelWindowAcceptsThePanelBoxEitherWayUp() { + for frame in [CGRect(x: 0, y: 0, width: 669, height: 951), Self.innerHome] { + let window = DisplayWindowFixture(frame: frame, displayID: 3) + let walk = resolveLitPanelWindow([window]) + XCTAssertEqual(walk.failure, .none, "\(frame)") + XCTAssertTrue(walk.window === window) + } + } + + /// A full-size window on the dark panel is the trap: geometry alone would book a screen nobody + /// is looking at. + func testLitPanelWindowRefusesWindowsThatAreAllOnTheDarkPanel() { + let walk = resolveLitPanelWindow([ + DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1), + DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1), + ]) + XCTAssertEqual(walk.failure, .unresolvedWindow) + XCTAssertNil(walk.window) + } + + /// An app that occupies part of the lit panel has no full-panel box; being on the lit display is + /// still the fact the first-window route could not establish. + func testLitPanelWindowBooksTheLargestWindowOnTheLitDisplay() { + let strip = DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 3) + let sheet = DisplayWindowFixture(frame: CGRect(x: 0, y: 90, width: 300, height: 400), displayID: 3) + let walk = resolveLitPanelWindow([ + DisplayWindowFixture(frame: Self.innerHome, displayID: 1), strip, sheet, + ]) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.window === sheet) + } + + /// A window the walk could not place, or one past its cap, could be the panel's surface, so a + /// lesser window is not booked in its place. + func testLitPanelWindowRefusesALesserWindowWhenTheListWasNotFullyPlaced() { + let sheet = DisplayWindowFixture(frame: CGRect(x: 0, y: 90, width: 300, height: 400), displayID: 3) + XCTAssertEqual( + resolveLitPanelWindow([ScreenlessWindowFixture(), sheet]).failure, .unresolvedScreen) + let beyondCap = (0.. RawAXNode { + func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode { var updated = self updated.rect = rect updated.hittable = hittable diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 446bd0283a..66616ff5ba 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -247,12 +247,6 @@ "trigger": "structured runner reply with code APP_SCREEN_CAPTURE_UNRENDERABLE: a refusal that ran nothing", "dispatched": "no" }, - { - "id": "ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL", - "producer": "ios-runner", - "trigger": "structured runner reply with code APP_SCREEN_OFF_LIT_PANEL: a refusal that ran nothing", - "dispatched": "no" - }, { "id": "ios-runner.reply.ELEMENT_NOT_FOUND", "producer": "ios-runner", diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 2b8efd2134..81f3d3fc24 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -8,6 +8,7 @@ The touch-overlay export diagnosis was corrected on 2026-09-22 under #2707. The Android emulator pose amendment is dated 2026-10-10. The no-panel refusal correction (#3761) is dated 2026-10-18. The guest readback amendment is dated 2026-10-26. +The runner surface amendment is dated 2026-10-31. This ADR covers iPhone Duo (iOS 27.1, `iPhone19,4`) and Apple devices with more than one integrated CoreDevice display. The amendment below also covers foldable Android emulators. @@ -364,7 +365,7 @@ Rejected alternatives: `devicectl device orientation set` reports success and has no effect on a simulator that `simctl boot` started. `orientation` therefore rotates a multi-panel simulator through the guest device-state control: the HID event that Device Hub's orientation menu sends (`orientation-picker-control`, type `enum`). The control has no upside-down value, so `portrait-upside-down` is refused with `unsupported-foldable-orientation`. -Routing uses the `fold` admission and the panel table. A resolved panel count is a hardware fact, so the daemon probes each simulator once. A rotation asks that table itself and never answers from the short window a surface probe opens after a table that would not reply: the two routes differ in correctness, because on a multi-panel simulator the historic runner route has no effect and its readback repeats the request, so answering from that window would report a rotation the device never performed. +Routing uses the `fold` admission and the panel table. A resolved panel count is a hardware fact, so the daemon probes each simulator once. The verdict comes from `devicectl device orientation get`, a process that never sent the HID event. CoreDevice answers each query with the value that it sampled at the previous query, so the first read after a dispatch is discarded. @@ -709,31 +710,23 @@ Keyframes and panel geometry remain Apple-only. The emulator console supports only fixed postures. Android reports the app window on the next snapshot instead of a panel. -## Amendment: the lit panel chooses the window, not the window list (2026-10-31) +## Amendment: the lit panel chooses the runner's window (2026-10-31) -Resolving a screen had two authorities. Capture asked CoreDevice which panel is lit; the runner took the application's first window. The first changes when the device folds and the second does not, so on a foldable the two disagreed. +SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -SpringBoard keeps a window per panel. On a Duo home screen, its first reported window is the 226x155.3 status-bar window on the outer panel, whichever panel is lit. Four paths booked against that window: the interaction viewport, the synthesized reference frame, the snapshot viewport, and the capture's display read. Every home-screen gesture therefore entered a panel that might be dark, and the snapshot viewport was 223x317 on a 951x669 screen. One resolver owns all four paths: `RunnerResolveSurfaceWindow` (`RunnerXCTestEventBridge.m`). It reads each window's `exists`, frame, and `screen.displayID` once, and applies the rule in `RunnerPanelSurface.m`. +Decision: the host names the lit panel in each runner request that resolves a screen. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. A request without the field keeps the historic first-window route unchanged. -Which panel is lit is a host fact. The runner has no hinge to read, and geometry cannot settle it, because the dark panel's window has a full-size frame at the origin. So the request carries it. The host reads the panel table per request (`resolveRunnerRequestLitPanel`) and names the panel in `litPanel` (`displayId`, `widthPt`, `heightPt`) only for a device whose table reports more than one integrated panel, and only on a command that resolves a screen. Selection requires a named display. A frame equal to the panel's box, or that box transposed, is the panel's surface; otherwise the largest window on that display is the surface. Both sizes are the panel's native points, never rotated, because the system presents the inner panel turned and its interaction space swaps the side lengths. - -A request naming no panel takes the historic read: the first window with a usable frame, and no display identity is asked for at all. Two kinds of device send that request, and only one of them is single-panel. A single-panel device is the common case. A multi-panel device also sends it whenever the panel table could not be read, or decided nothing, because a surface fact has no pressure to name some panel and guess: a capture must name a display or refuse, a surface booking can fall back. So an absent `litPanel` is never evidence that a device has one panel. This is why the fact travels per request rather than as runner state. An iPhone pays nothing it did not pay before, and a fold cannot be answered from a stale panel, because the next request asks again. +With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A walk that could not place every window refuses with `APP_SCREEN_UNRESOLVED`. A list with no window on the lit display is `APP_SCREEN_WINDOW_UNRESOLVED`, and capture then asks the system surface. Dispatch, the coordinate fallback, the viewport, and capture use that one window. A regular snapshot keeps only the application's windows whose element snapshot reports the lit display. Its root takes the panel's native box. Rejected alternatives: -- **The largest window: rejected.** It guesses power from size. It picks the 951x669 inner window while the device is closed, the same class of error as the implicit capture default. -- **Panel points derived in the runner: rejected.** The host already measures panel power, and the runner's own screen read can name the dark panel. -- **Threading one optional panel through every Swift signature: rejected.** Forty call sites, four of which have no `Command` in scope, for one answer per command. The runner holds the panel of the command in flight in one lock-guarded slot (`RunnerRequestPanel.swift`), which is the only place that answers the question. An operation that outlives its request takes the panel as an argument instead of reading the slot per sample: a recording books every frame on the panel its `recordStart` named, because the writer is sized from the bootstrap frame, and a keyboard settle loop compares samples taken on one panel. Reading the slot again from such a loop would follow whatever command replaced the request. -- **Choosing a panel when the table decides nothing: rejected.** Capture may fall back to the `primary` panel because it must name some display or refuse; a surface fact has no such pressure, and naming the wrong panel would refuse the screen the user is looking at. Such a request names no panel. -- **A `--screen outer|inner` flag for gestures: rejected again, for the same reason as for capture.** The panel is measured. It is not asked for. +- **Largest window overall: rejected.** It guesses panel power from size. +- **Pruning the application tree by geometry: rejected.** A dark 466x678 window fits inside the inner panel's box, and a turned inner window does not. +- **Rooting the snapshot at the booked window: rejected.** The first full-panel SpringBoard window has no icons, and an app's keyboard lives in another window. +- **A device-wide panel cached in the runner: rejected.** It is stale after a fold. Consequences: -- A multi-panel device whose every window is on the dark panel refuses instead of booking there, with the typed code `APP_SCREEN_OFF_LIT_PANEL`. The system surface is asked first, as for an unresolved window, so only a device with nothing on the lit panel refuses. -- `runner.log` names the decision: `AGENT_DEVICE_RUNNER_PANEL_SURFACE lit=(x) reported= placed= unplaceable= selection= frame=(…)` reports the panel a window was booked against and the frame that won. `reported` is what the window query named before the cap, `placed` is how many of those the runtime would say a display for. `selection` is the rule's answer: 0 `litPanelWindow`, 1 `litDisplayWindow`, 2 `offLitPanel`, 3 `noWindow`. A single-panel device takes the historic walk and prints none of this. -- The two synthesized-input markers say what the booked surface produced. `AGENT_DEVICE_RUNNER_SYNTHESIZED_DISPATCH kind= point=(x,y) reference=(, x) orientation=` names no display: `reference` is the booked surface's own box, so on a foldable it must be the lit panel's presented box and `reference=(452,0 226x155.3)` is the status-bar window this fix removes. Compare that box with `PANEL_SURFACE frame=`. `AGENT_DEVICE_RUNNER_SYNTHESIZED_RECORD name= displayID= interfaceOrientation=` names the track's display: the display ID of the window the gesture was booked on, so it must equal `PANEL_SURFACE lit=`. -- A Duo's SpringBoard names 40-43 windows while the walk books at most eight, so the marker reads `reported=40 placed=8`. The lit panel's full-screen window sits inside the first eight in every pose observed. A refusal that claims something about the whole list is refused instead when the walk could not read the whole list: `offLitPanel` becomes "could not decide" when a window was unplaceable or the cap cut the list short, and so does `litDisplayWindow`, whose largest survivor could be a corner strip while the panel's real surface went unread. `selection=3` means no window the walk read could be placed on a display, which is always `placed=0`: it says nothing about windows the cap left unread. -- A regular capture of a multi-panel device reports what the panel fact said about its tree: `AGENT_DEVICE_RUNNER_PANEL_SURFACE_TREE republishedRoot= droppedSurfaces= nodes=->`, and `…_TREE_GRAFTED frame=(…) nodes=` when the application's own subtree carried no window for the lit panel. `…_TREE_SKIPPED reason=` names why a graft did not happen, and `graft-failed` carries the capture code that refused it. A skipped graft leaves the tree as it was walked, with the nodes and dedup keys its abandoned read produced discarded; it never fails the tier and never marks it exhausted. -- Main-queue work abandoned by the execution watchdog can read the panel of the command that replaced it. The log line makes such a resolution visible. A single-panel device reads no panel, so it has nothing to repair. -- Recording, keyboard settling, and a read by point name their panel too: `recordStart`, `keyboardDismiss`, and `readText` resolve a screen (each samples it or places a point in it), so they carry the fact. Without it a clip of a foldable's dark panel is a black video whose bootstrap succeeded. -- The wire field is a two-declaration contract like the capture metadata: `AppleLitPanel` in `@agent-device/contracts/interactor-types` and `RunnerLitPanelFact` in `RunnerTests+Models.swift`. +- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame, and `SYNTHESIZED_RECORD displayID=` must equal `lit=`. +- A recording keeps the panel that its `recordStart` named. +- Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. diff --git a/docs/agents/device-foldable.md b/docs/agents/device-foldable.md index 5bf57f56a6..0719bb2e99 100644 --- a/docs/agents/device-foldable.md +++ b/docs/agents/device-foldable.md @@ -1,25 +1,30 @@ # Foldable Apple verification Before you change capture behavior on a multi-panel device, read ADR 0025. -- Panels: `xcrun devicectl device info displays --device ` shows each integrated panel with - `backlightState`; only the lit panel permits capture. In a scoped set, `simctl --set spawn - ~/.agent-device/fold-helper//fold-helper panels` reads it. -- Input routing: after each pose change, verify a fresh control. A successful synthesis - acknowledgement does not prove a hit. Examine the runner and simulator `testmanagerd`/BackBoard - logs for display identity and delivery. -- Surface identity: verify the window the runner booked, not only the panel the host named; it owns - gesture coordinates and its target screen. Its `runner.log` markers are listed in ADR 0025 and - each must name the lit panel, except `SYNTHESIZED_DISPATCH reference=(…)`, which names no display: - compare it with the booked surface. A 226x155.3 reference frame is the status-bar window: the bug. -- Pose: `agent-device fold closed|half-open|open` sends private HID inside the simulator; the same - helper reads the hinge back in `hinge` mode. After a fold, take another snapshot: folds invalidate - refs and coordinates. Verify locally, Device Hub stopped and the simulator booted through `simctl - boot`, in the default set and with `--ios-simulator-device-set`: all three poses, capture of the - active panel, and an app interaction. Duo coverage stays local until GHA supports the runtime. - After a fresh boot a closed hinge can keep the inner panel lit for over 30 seconds, so lit-panel - state never settles a pose. -- Touch overlays export at the captured track size. A failed check keeps the raw file and `record - stop` reports `overlayWarning`; `--hide-touches` is the raw-capture fallback. +To list the panels, run `xcrun devicectl device info displays --device `. +In a scoped set, run `xcrun simctl --set spawn ~/.agent-device/fold-helper//fold-helper panels`. + +- Panels: the list shows each integrated panel with `backlightState`. Only the lit panel permits capture. + A capture of the dark panel exits 0 and writes an all-black PNG. +- Input routing: after each pose change, verify a fresh control. + A successful synthesis acknowledgement does not prove a hit. + Examine the runner and simulator `testmanagerd`/BackBoard logs for display identity and delivery. + The resolved app window owns gesture coordinates and its target screen. +- Pose: `agent-device fold closed|half-open|open` sends private HID inside the simulator. + The same helper reads the hinge back in `hinge` mode. + After a fold, take another snapshot. Folds invalidate refs and coordinates. + Verify locally with Device Hub stopped and the simulator booted through `simctl boot`. + Test these items in the default set and with `--ios-simulator-device-set`: + - All three poses. + - Capture of the active panel. + - An app interaction. + Until GHA supports the runtime, Duo coverage remains local. + After a fresh boot, a closed hinge can keep the inner panel lit for more than 30 seconds. + The lit panel is evidence only. It cannot separate `half-open` from `open`. +- Touch overlays export at the captured track size. + The compositor checks its output size and non-black content against the raw capture. + If that check fails, `record stop` keeps the raw file. It reports `overlayWarning`. + `record start --hide-touches` is only for the fastest raw capture. - To launch on iOS 27.0/27.1, iOS 27 SDK apps need UIScene lifecycle. - The iOS 27.1 runtime (24A94425) accepts only the iPhone Duo device type. For a non-foldable control, use another runtime. diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index 60a123b8db..3fcb985970 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -178,7 +178,6 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_CAPTURE_UNRENDERABLE': () => replyFailure('APP_SCREEN_CAPTURE_UNRENDERABLE'), - 'ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL': () => replyFailure('APP_SCREEN_OFF_LIT_PANEL'), 'ios-runner.reply.ELEMENT_NOT_FOUND': () => replyFailure('ELEMENT_NOT_FOUND'), 'ios-runner.reply.ELEMENT_OFFSCREEN': () => replyFailure('ELEMENT_OFFSCREEN'), 'ios-runner.reply.AMBIGUOUS_MATCH': () => replyFailure('AMBIGUOUS_MATCH'), diff --git a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts index 65e4fd1853..7c881cc936 100644 --- a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts +++ b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts @@ -426,7 +426,6 @@ test('runner_endpoint_probe_exhausted is decided by the typed reason, not the me test('a refused screen capture keeps the runner reason and stays off the wire code', () => { assert.deepEqual([...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].sort(), [ 'APP_SCREEN_CAPTURE_UNRENDERABLE', - 'APP_SCREEN_OFF_LIT_PANEL', 'APP_SCREEN_UNRESOLVED', 'APP_SCREEN_WINDOW_UNRESOLVED', ]); diff --git a/packages/apple-runner/src/index.ts b/packages/apple-runner/src/index.ts index c77a021966..d860b6e345 100644 --- a/packages/apple-runner/src/index.ts +++ b/packages/apple-runner/src/index.ts @@ -29,6 +29,7 @@ export { resolveRunnerRequestSignal, type RunnerCommand, } from './runner-contract.ts'; +export { runnerCommandResolvesSurface } from './runner-command-traits.ts'; export { isRunnerMainThreadTimeoutError } from './runner-error-classification.ts'; export { withAppleRunnerProvider, diff --git a/packages/apple-runner/src/runner-command-traits.ts b/packages/apple-runner/src/runner-command-traits.ts index b419962847..d2672344f8 100644 --- a/packages/apple-runner/src/runner-command-traits.ts +++ b/packages/apple-runner/src/runner-command-traits.ts @@ -92,6 +92,55 @@ export const RUNNER_COMMAND_TRAITS = { shutdown: DEFAULT_TRAITS, } as const satisfies Record; +/** + * Whether the runner resolves a screen for the command: a window to book a gesture on, a viewport, + * or a display to sample. Such a request for a multi-panel device names the lit panel (ADR 0025). + */ +const RUNNER_COMMAND_RESOLVES_SURFACE = { + tap: true, + mouseClick: false, + longPress: true, + drag: true, + remotePress: false, + type: true, + swipe: true, + scroll: true, + desktopScroll: false, + findText: false, + querySelector: false, + readText: true, + snapshot: true, + screenshot: true, + backInApp: true, + backSystem: true, + home: false, + rotate: false, + gesture: true, + gestureViewport: true, + appSwitcher: true, + actionButton: false, + volumeUp: false, + volumeDown: false, + keyboardDismiss: true, + keyboardReturn: false, + alert: false, + sequence: true, + recordStart: true, + recordStop: false, + status: false, + uptime: false, + appState: false, + pasteboardWrite: false, + activate: false, + terminate: false, + targetReset: false, + shutdown: false, +} as const satisfies Record; + +export function runnerCommandResolvesSurface(command: RunnerCommand): boolean { + return RUNNER_COMMAND_RESOLVES_SURFACE[command.command]; +} + export function isReadOnlyRunnerCommand(command: RunnerCommand): boolean { return readRunnerCommandTraits(command).readOnly; } diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index cf7dafe38f..86d84f907d 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -15,7 +15,6 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot'; import type { ClickButton } from '@agent-device/contracts/click-button'; import type { - AppleLitPanel, CoordinateTapTextInputEvidence, ElementSelectorKey, } from '@agent-device/contracts/interactor-types'; @@ -165,12 +164,10 @@ export type RunnerCommand = { inlineScreenshot?: boolean; synthesized?: boolean; /** - * The panel the host reports as lit, sent only for a device whose panel table reports more than one - * integrated panel and only on a command that resolves a screen. Absent means "single-panel device" - * or "the panel table decided nothing": the runner then keeps its historic first-window surface - * route rather than guessing a panel (ADR 0025). + * The lit panel's display and native box in points, sent only for a multi-panel device on a + * command that resolves a screen (ADR 0025). */ - litPanel?: AppleLitPanel; + litPanel?: { displayId: number; widthPt: number; heightPt: number }; steps?: RunnerSequenceStep[]; }; @@ -278,19 +275,12 @@ export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCC const RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_WINDOW_UNRESOLVED'; const RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_UNRESOLVED'; const RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE = 'APP_SCREEN_CAPTURE_UNRENDERABLE'; -const RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE = 'APP_SCREEN_OFF_LIT_PANEL'; -/** - * Every runner code meaning "this capture did not happen", covering the display, the image, and the - * panel. The last one is a foldable's own refusal: the request named a lit panel and neither the - * target app nor the system surface had a window on it. The capture does not fall back to the dark - * panel, which would answer with a valid all-black PNG, so this code is what the host sees. - */ +/** Every runner code meaning "this capture did not happen", covering the display and the image. */ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = new Set([ RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE, - RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE, ]); /** diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 475486f370..0de826f22b 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -107,24 +107,6 @@ export type PressPointOptions = Readonly<{ coordinateTapTextInputEvidence?: CoordinateTapTextInputEvidence; }>; -/** - * The panel the host reports as lit, in the units the panel table reports it: which display is - * powered, and that panel's own native box in points (`nativeSize` divided by `pointScale`). - * - * A runner on a multi-panel device cannot decide which panel is powered — it has no hinge to read, - * and geometry decides nothing because the dark panel's windows have perfectly good frames — so a - * request that resolves a screen names the panel here and the runner books its surface against it - * (ADR 0025). A request that carries no panel is a single-panel device's request. - * - * The size is the panel's native box, deliberately not the box an app is showing on it: the system - * presents the iPhone Duo inner panel turned, so its interaction space swaps the side lengths. - */ -export type AppleLitPanel = Readonly<{ - displayId: number; - widthPt: number; - heightPt: number; -}>; - /** * Legacy success text retained for compatibility when the XCTest runner used * the Maestro non-hittable coordinate fallback. Usage itself is carried by diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index eee9d33fd8..476cd6b774 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,5 +1,5 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; -import type { AppleLitPanel } from '@agent-device/contracts/interactor-types'; +import type { RunnerCommand } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; @@ -411,28 +411,15 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi } /** - * The panel to name in a runner request, from a panel table read. - * - * Absent unless panel power itself decided one panel. A capture may still name the `primary` panel - * when panel power decides nothing (ADR 0025), because a capture has to name some display or refuse; - * a surface fact has no such pressure, and naming a panel the table did not settle would have the - * runner refuse the panel the user is actually looking at. The `primary` flag plays no part here: a - * payload can decide which panel is lit and still omit that flag. - * - * A panel the runtime names display 0 cannot be sent: 0 is the wire's "no panel named" identity, so - * such a display would look panel-carrying on the request and behave single-panel in the runner. + * The panel a runner request names, only when panel power decided exactly one panel. Unlike a + * capture, a surface request does not fall back to the `primary` panel: it can keep the runner's + * historic route instead of guessing. Display 0 is the runtime's "no display" identity. */ -export function appleLitPanel(inventory: AppleDisplayInventory): AppleLitPanel | undefined { - // `activeDisplay` is set only by a table that named a panel to capture, and an incomplete table - // names none: a panel the parser dropped may be the lit one, and naming the survivor would send - // every surface request to a display that can be dark. - if (!inventory.multiScreen || inventory.unresolved || !inventory.activeDisplay) { - return undefined; - } +export function appleLitPanel(inventory: AppleDisplayInventory): RunnerCommand['litPanel'] { + if (!inventory.multiScreen || inventory.unresolved || !inventory.activeDisplay) return undefined; const litPanels = inventory.displays.filter((display) => display.power === 'lit'); const panel = litPanels.length === 1 ? litPanels[0]! : undefined; if (!panel || panel.displayId <= 0) return undefined; - if (!panel.pointScale || !Number.isFinite(panel.pointScale)) return undefined; return { displayId: panel.displayId, widthPt: panel.widthPx / panel.pointScale, diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 9afbfacf2d..a3929acd75 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -1,105 +1,42 @@ import { bindAppleRunnerHost, resolveRunnerRequestSignal, + runnerCommandResolvesSurface, runAppleRunnerCommand as runHostBoundAppleRunnerCommand, type AppleRunnerCommandOptions, type RunnerCommand, } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { AppleLitPanel } from '@agent-device/contracts/interactor-types'; import { appleRunnerHost } from './runner-host.ts'; /** - * Composition root for the Apple runner operations: the only module that binds the real host - * capabilities from `runner-host.ts`, and the module daemon, platform, and CLI consumers import the - * host-bound operations from, so every such consumer evaluates after the binding. Runner modules - * must not reach the host while they evaluate: they load before this module binds it. Types and + * Composition root for the Apple runner operations: the only module that binds + * the real host capabilities from `runner-host.ts`, and the module daemon, + * platform, and CLI consumers import the host-bound operations from, so every + * such consumer evaluates after the binding. Runner modules must not reach the + * host while they evaluate: they load before this module binds it. Types and * host-free helpers come from the package façade directly. - * - * Because every Apple runner request leaves the platform through here, this is also where a request - * that resolves a screen gains the one fact a runner cannot read for itself: which panel is lit. */ bindAppleRunnerHost(appleRunnerHost); /** - * Runner commands whose execution resolves a screen — a window to book a gesture against, a viewport - * to place a capture or a snapshot in, or the display to capture. Each of them used the application's - * first window, which on a foldable home screen is SpringBoard's status-bar window on a panel that - * may be dark, so each carries the panel the host read as lit for the request. - * - * A command outside this set resolves no surface and carries no panel: readiness, lifecycle, - * clipboard and element- or selector-targeted traffic stays byte-for-byte the request it was. - */ -const RUNNER_SURFACE_RESOLVING_COMMANDS: ReadonlySet = new Set([ - 'tap', - 'longPress', - 'drag', - 'type', - 'swipe', - 'scroll', - 'gesture', - 'sequence', - 'backInApp', - 'backSystem', - 'appSwitcher', - 'gestureViewport', - 'snapshot', - 'screenshot', - // Both sample the screen from the runner: `recordStart` sizes and pumps its frames, and - // `keyboardDismiss` fingerprints screenshots to settle. Without the panel they would read the - // dark panel and report success — an all-black clip, or a keyboard that "settled" on a screen - // that never changed. `readText` resolves the surface window to place the point it was handed. - 'recordStart', - 'keyboardDismiss', - 'readText', -]); - -/** - * The request as the wire carries it: a surface-resolving command for a multi-panel device names the - * lit panel, and anything else is the command it was. A command that already names a panel is left - * alone, so a caller that resolved one for its whole sequence is not overruled mid-flight. - */ -function withRunnerRequestLitPanel( - command: RunnerCommand, - litPanel: AppleLitPanel | undefined, -): RunnerCommand { - if (!litPanel || command.litPanel || !RUNNER_SURFACE_RESOLVING_COMMANDS.has(command.command)) { - return command; - } - return { ...command, litPanel }; -} - -/** - * Sends `command` to the runner, naming the lit panel when the device presents more than one panel and - * the command resolves a screen. A single-panel device, an unreadable panel table, and a multi-panel - * device whose table decides nothing all send no panel, and the runner keeps its historic surface - * route. - * - * The panel owner is loaded on first use, not at module load: this module sits in the eager closure - * of the Apple façades, and the panel probe is host tooling only a multi-panel request needs (ADR - * 0019). + * Sends `command` to the runner. A command that resolves a screen names the lit panel for a + * multi-panel device. The panel probe loads on first use, because this module is in the eager closure + * of the Apple façades (ADR 0019). */ export async function runAppleRunnerCommand( device: DeviceInfo, command: RunnerCommand, options: AppleRunnerCommandOptions = {}, ): Promise> { - if (!RUNNER_SURFACE_RESOLVING_COMMANDS.has(command.command)) { - return await runHostBoundAppleRunnerCommand(device, command, options); - } - // A command that already names a panel is not re-probed: a caller that resolved one for its whole - // sequence keeps the answer it resolved, and the probe is not paid twice for one request. - if (command.litPanel) { + if (command.litPanel || !runnerCommandResolvesSurface(command)) { return await runHostBoundAppleRunnerCommand(device, command, options); } const { resolveRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); - // The request's own cancellation covers the probe too: a cancelled surface command must not wait - // out the panel table's timeout before the transport sees the cancellation. - const signal = resolveRunnerRequestSignal(options); - const litPanel = await resolveRunnerRequestLitPanel(device, { signal }); + const litPanel = await resolveRunnerRequestLitPanel(device, resolveRunnerRequestSignal(options)); return await runHostBoundAppleRunnerCommand( device, - withRunnerRequestLitPanel(command, litPanel), + litPanel ? { ...command, litPanel } : command, options, ); } @@ -136,5 +73,3 @@ export { verifyLeaseRunnerPidIdentity, writeRunnerCacheMetadataForArtifacts, } from '@agent-device/apple-runner'; - -export type { AppleRunnerCommandOptions, RunnerCommand }; diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index eaa8991150..b954d445e1 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -15,7 +15,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../core/tool-provider.ts'; import { createAppleInteractor } from '../interactor.ts'; -import { isAppleFoldableSimulator } from './orientation.ts'; +import { readFoldablePanelTable } from './orientation.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; import { IPHONE_17_SINGLE_PANEL, IPHONE_DUO_CLOSED } from '../__tests__/display-fixtures.ts'; @@ -201,44 +201,11 @@ test('a foldable rotation that CoreDevice never reports is refused as unverified expect(sim.helpers).toHaveLength(1); }); -/** - * A cancelled panel-table read is the caller going away, not a toolchain that will not answer, so it - * must not spend the retry window the `unresolved` answer opens: the next request asks again. - */ -test('a cancelled panel-table probe does not keep the device single-panel', async () => { - const device = simulator(); - let reads = 0; - const provider = createLocalAppleToolProvider({ - runCommand: async (_command, args, options) => { - expect(args.slice(0, 4)).toEqual(['devicectl', 'device', 'info', 'displays']); - if (options?.signal?.aborted) throw new Error('devicectl aborted by the caller'); - reads += 1; - fs.writeFileSync(args.at(-1)!, JSON.stringify({ result: { displays: DUO_DISPLAYS } })); - return { stdout: '', stderr: '', exitCode: 0 }; - }, - }); - const ask = (signal?: AbortSignal) => - withAppleToolProvider(provider, () => isAppleFoldableSimulator(device, signal)); - - const cancelled = new AbortController(); - cancelled.abort(); - await expect(ask(cancelled.signal)).resolves.toBe(false); - // The request that follows is free to ask again, and the answer it gets is the device's own. - await expect(ask()).resolves.toBe(true); - // One read: the cancelled probe never took the retry window the `unresolved` answer opens. - expect(reads).toBe(1); -}); - -/** - * A rotation is decided by the panel table, never by the window a surface probe opened: the historic - * runner route has no effect on a foldable and its readback repeats the request, so answering from - * that window would report a rotation the device never performed. - */ +/** A rotation reads the panel table itself: the surface probe's retry window never holds it. */ test('a rotation is not routed by a pending panel-table retry', async () => { const device = simulator(); let tableFailures = 0; const sim = fakeSimulator(device, DUO_DISPLAYS); - // A surface probe finds a panel table that will not answer, which opens the retry window. const failing = createLocalAppleToolProvider({ runCommand: async (_command, args) => { if (args.slice(0, 4).join(' ') === 'devicectl device info displays') { @@ -250,10 +217,9 @@ test('a rotation is not routed by a pending panel-table retry', async () => { throw new Error(`unexpected tool call: ${args.join(' ')}`); }, }); - await withAppleToolProvider(failing, () => isAppleFoldableSimulator(device, undefined)); + await withAppleToolProvider(failing, () => readFoldablePanelTable(device, undefined)); expect(tableFailures).toBe(1); - // The rotation that follows pays its own read, gets the device's real answer, and folds through HID. await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); expect(sim.helpers).toEqual([['rotate', 'landscape-left']]); expect(mockRunner).not.toHaveBeenCalled(); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 5d01d8d251..32aa79ec4c 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -9,7 +9,10 @@ import { IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS, } from '../core/config.ts'; import { readAppleDeviceOrientation } from '../core/device-orientation.ts'; -import { queryAppleDisplayInventory } from '../core/display-inventory.ts'; +import { + queryAppleDisplayInventory, + type AppleDisplayInventory, +} from '../core/display-inventory.ts'; import { simulatorAddressFor } from '../core/simctl.ts'; import { createFoldHelper } from './fold-helper.ts'; import { appleFoldableFacts } from './runtime.ts'; @@ -26,62 +29,47 @@ const ORIENTATION_UNVERIFIED_HINT = /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); -// A panel table the runtime would not answer is retried, but not once per command: every runner -// request that resolves a surface asks this question, and an unbounded retry pays the probe cost on -// each of them. A few seconds absorbs one command burst; it is short enough that the next request -// minutes after a transient CoreDevice failure asks the panel table again. `apple_panel_table_retry_ -// pending` names the window during which a foldable is treated as single-panel. -const PANEL_TABLE_RETRY_MS = 5_000; -const panelTableRetriesAt = createTtlMemo({ - ttlMs: PANEL_TABLE_RETRY_MS, - scheduleExpiry: true, -}); - /** * Whether a rotation must go through guest HID: a simulator the `fold` fact admits that has more * than one integrated panel. XCUIDevice rotation has no effect there, and its readback repeats * the requested value. - * - * `askPanelTableAgain` skips the retry window and reads the table now. A caller that asks once per - * command, such as a rotation, takes that route: a surface probe is paid on every runner request, so - * only it may answer from the window, and holding a rotation there would send a foldable through the - * route whose readback repeats the request and reports a rotation that never happened. */ export async function isAppleFoldableSimulator( device: DeviceInfo, signal: AbortSignal | undefined, - options: { askPanelTableAgain?: boolean } = {}, ): Promise { if (!appleFoldableFacts(device).setFoldPose.available) return false; const known = multiPanelByDevice.get(device.id); if (known !== undefined) return known; - if (!options.askPanelTableAgain && panelTableRetriesAt.get(device.id)) { - // The caller cannot tell this answer from a device it proved single-panel, so the reason has to - // be in the log: a runner request that follows takes its historic first-window surface route. - emitDiagnostic({ - level: 'debug', - phase: 'apple_panel_table_retry_pending', - data: { - deviceId: device.id, - retryInMs: PANEL_TABLE_RETRY_MS, - hint: 'The panel table would not answer, so until it is asked again this device is treated as single-panel: a runner request carries no lit panel.', - }, - }); - return false; - } + const inventory = await queryAppleDisplayInventory(device, { signal }); + if (inventory.unresolved) return false; + multiPanelByDevice.set(device.id, inventory.multiScreen); + return inventory.multiScreen; +} + +// Every surface command reads the panel table, so one that would not answer is not asked again +// for a few seconds: a failing probe can cost its whole timeout. +const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); + +/** + * One panel-table read of a multi-panel simulator, or undefined for a device the `fold` fact refuses, + * one known to have a single panel, or a table that would not answer. It records the panel count it + * reads, so a single-panel simulator pays one read per daemon. + */ +export async function readFoldablePanelTable( + device: DeviceInfo, + signal: AbortSignal | undefined, +): Promise { + if (!appleFoldableFacts(device).setFoldPose.available) return undefined; + if (multiPanelByDevice.get(device.id) === false) return undefined; + if (unansweredPanelTables.get(device.id)) return undefined; const inventory = await queryAppleDisplayInventory(device, { signal }); if (inventory.unresolved) { - // A probe the caller cancelled is not evidence that the panel table will not answer, so it must - // not spend the retry window: the next request asks again instead of inheriting this one's - // single-panel fallback for the whole window. The caller's own cancellation surfaces downstream, - // where the request it belongs to reaches the transport. - if (!signal?.aborted) { - panelTableRetriesAt.set(device.id, true); - } - return false; + if (!signal?.aborted) unansweredPanelTables.set(device.id, true); + return undefined; } multiPanelByDevice.set(device.id, inventory.multiScreen); - return inventory.multiScreen; + return inventory.multiScreen ? inventory : undefined; } /** Rotates a foldable simulator through guest HID and verifies the rotation through CoreDevice. */ diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index b99df12425..fb037ab858 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -49,41 +49,52 @@ test('a multi-panel request names the lit panel in the units the runner compares answering(IPHONE_DUO_CLOSED); // A closed Duo lights its outer panel: 1398x2034 px at pointScale 3, which is the 466x678 the // runner finds as that panel's SpringBoard window. - assert.deepEqual(await resolveRunnerRequestLitPanel(simulator(), {}), { + assert.deepEqual(await resolveRunnerRequestLitPanel(simulator(), undefined), { displayId: 1, widthPt: 466, heightPt: 678, }); }); -test('the panel is read again for the next request, so a fold moves the surface', async () => { +test('each request reads the panel table once, so a fold moves the surface', async () => { const device = simulator(); answering(IPHONE_DUO_CLOSED); - assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); - // The first request also paid for the panel COUNT probe; every later one pays only for the read - // that answers which panel is lit now. - const readsAfterFirst = mockPanelTable.mock.calls.length; + assert.equal((await resolveRunnerRequestLitPanel(device, undefined))?.displayId, 1); + assert.equal(mockPanelTable.mock.calls.length, 1, 'the panel count comes from the same read'); answering(IPHONE_DUO_OPEN); assert.equal( - (await resolveRunnerRequestLitPanel(device, {}))?.displayId, + (await resolveRunnerRequestLitPanel(device, undefined))?.displayId, 3, 'an opened device must be asked again, not answered from the panel that just went dark', ); - assert.equal(mockPanelTable.mock.calls.length - readsAfterFirst, 1); + assert.equal(mockPanelTable.mock.calls.length, 2); }); test('a device with one panel sends no panel and is never asked again', async () => { const device = simulator(); answering(IPHONE_17_SINGLE_PANEL); - assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); const reads = mockPanelTable.mock.calls.length; - assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); assert.equal(mockPanelTable.mock.calls.length, reads, 'one panel is hardware, probed once'); }); -test('a device whose panel table cannot be read sends no panel', async () => { +test('a panel table that would not answer is not asked again by the next request', async () => { + const device = simulator(); mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); + assert.equal(mockPanelTable.mock.calls.length, 1); +}); + +test('a cancelled panel-table read leaves the next request free to ask', async () => { + const device = simulator(); + mockPanelTable.mockRejectedValueOnce(new Error('devicectl aborted by the caller')); + const cancelled = new AbortController(); + cancelled.abort(); + assert.equal(await resolveRunnerRequestLitPanel(device, cancelled.signal), undefined); + answering(IPHONE_DUO_CLOSED); + assert.equal((await resolveRunnerRequestLitPanel(device, undefined))?.displayId, 1); }); test('a multi-panel device whose panel power decides nothing sends no panel', async () => { @@ -100,7 +111,7 @@ test('a multi-panel device whose panel power decides nothing sends no panel', as })), }, }); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), undefined), undefined); }); test('a multi-panel device whose panel table is incomplete sends no panel', async () => { @@ -114,12 +125,12 @@ test('a multi-panel device whose panel table is incomplete sends no panel', asyn displays: [outer, { ...inner, nativeSize: undefined }], }, }); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), undefined), undefined); }); test('a device the fold fact refuses never costs a panel table read', async () => { // A physical iPhone is folded by hand (ADR 0025) and no physical foldable leaf is in the // supported matrix, so its runner keeps the surface route the request always sent. - assert.equal(await resolveRunnerRequestLitPanel(IOS_DEVICE, {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(IOS_DEVICE, undefined), undefined); assert.equal(mockPanelTable.mock.calls.length, 0); }); diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index a902f1b31f..8c7843b637 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -1,44 +1,17 @@ -import type { AppleLitPanel } from '@agent-device/contracts/interactor-types'; +import type { RunnerCommand } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { appleLitPanel, queryAppleDisplayInventory } from '../core/display-inventory.ts'; -import { isAppleFoldableSimulator } from './orientation.ts'; +import { appleLitPanel } from '../core/display-inventory.ts'; +import { readFoldablePanelTable } from './orientation.ts'; /** - * The panel a runner request names as lit. - * - * A runner cannot tell which panel of a multi-panel device is powered: it has no hinge to read, and - * geometry decides nothing, because the dark panel's windows have perfectly good frames. So the host - * reads the panel table for the request and sends the answer with it, and the runner books its surface - * window against that fact rather than against the application's first window (ADR 0025). - * - * Which panel is lit is read fresh per request: that is exactly the fact a fold changes, and a cached - * answer would send every later request to the panel that has gone dark. Whether the device has more - * than one panel at all is hardware, so `isAppleFoldableSimulator` answers that from its own per-device - * cache and a single-panel iPhone pays one table read for its whole session. - * - * Only a simulator the `fold` fact admits gets here. A physical device is folded by hand (ADR 0025), - * no physical foldable leaf is in the supported matrix, and its runner keeps the historic first-window - * route the same request always sent. + * The lit panel a runner request names, read fresh per request because a fold changes it (ADR + * 0025). Undefined keeps the runner's historic first-window route. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo, - options: { signal?: AbortSignal } = {}, -): Promise { - if (!(await isAppleFoldableSimulator(device, options.signal))) return undefined; - const inventory = await queryAppleDisplayInventory(device, { signal: options.signal }); - const panel = appleLitPanel(inventory); - if (!panel && !inventory.unresolved) { - emitDiagnostic({ - level: 'debug', - phase: 'apple_runner_lit_panel_unresolved', - data: { - deviceId: device.id, - litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length, - hint: 'The runner request carries no panel and keeps its first-window surface route; runner.log names the window frames it resolved.', - }, - }); - } - return panel; + signal: AbortSignal | undefined, +): Promise { + const inventory = await readFoldablePanelTable(device, signal); + return inventory ? appleLitPanel(inventory) : undefined; } diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index bf9ad96c8f..87b3e40013 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -154,10 +154,7 @@ export function createAppleInteractor( ); }, setOrientation: async (orientation) => { - // A rotation asks the panel table itself rather than answering from the window a surface probe - // may have opened: the two routes differ in correctness, and the historic one reports a - // rotation a foldable never performed. - if (await isAppleFoldableSimulator(device, runnerOpts.signal, { askPanelTableAgain: true })) { + if (await isAppleFoldableSimulator(device, runnerOpts.signal)) { return await setAppleFoldableOrientation(device, orientation, { signal: runnerOpts.signal, }); From d7bbb85a6da32d4383a27839e0e644213e8ae8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 31 Oct 2026 20:21:39 +0100 Subject: [PATCH 03/29] docs: correct the lit-panel comments a review flagged Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerUITests/RunnerRequestPanel.swift | 3 +-- .../platform-apple/src/foldable/runner-panel.test.ts | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 53326897b9..daa5bbf6dc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -2,8 +2,7 @@ import XCTest // The lit panel of the command in flight (ADR 0025). Commands run one at a time on // `commandExecutionQueue`, where the slot is written, and the surface resolvers read it on main. -// Work that outlives its command, such as a recording pump or a settle loop, reads it once and -// carries that panel instead. +// A recording pump outlives its command, so `recordStart` reads the panel once and carries it. extension RunnerTests { func setRequestLitPanel(from command: Command) { var panel: RunnerLitPanel? diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index fb037ab858..c4bf72957e 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -98,10 +98,7 @@ test('a cancelled panel-table read leaves the next request free to ask', async ( }); test('a multi-panel device whose panel power decides nothing sends no panel', async () => { - // Both panels lit: the panel table settles no single panel. `activeDisplay` still names the - // primary panel for a capture, which has to name some display or refuse, but a surface fact is - // not forced to guess, and naming the wrong one would have the runner refuse the panel the user - // is looking at. + // Both panels lit: panel power settles no single panel, so the runner keeps its historic route. answering({ result: { displays: IPHONE_DUO_CLOSED.result.displays.map((display) => ({ @@ -115,10 +112,7 @@ test('a multi-panel device whose panel power decides nothing sends no panel', as }); test('a multi-panel device whose panel table is incomplete sends no panel', async () => { - // The table named two integrated panels and the parser could read only one of them, so the - // unread one may be the lit one. Its capture still names the parsed panel because a capture must - // name a display or refuse; a surface fact is not allowed to book every gesture on a display that - // can be dark. + // The parser read one of two integrated panels, so the unread one may be the lit one. const [outer, inner] = IPHONE_DUO_CLOSED.result.displays; answering({ result: { From 3b0d2c41ae6b417fb2c41d360e3b1edde016ecae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 31 Oct 2026 21:12:03 +0100 Subject: [PATCH 04/29] fix(ios): fail closed when the lit panel cannot be booked A request that names a lit panel now books that panel's window before it acts. A refusal comes back typed before any gesture or snapshot, and every later read in the request reuses the booked window, so nothing falls back to the application's own coordinates. - The window walk refuses after an unplaced window, even when a full-panel window follows it. - An app whose windows are all on the dark panel is APP_SCREEN_OFF_LIT_PANEL, and a capture no longer swaps it for SpringBoard. - A lit-panel snapshot runs only the tree tier and keeps only windows proven on the lit display. - A resolved panel-table read clears the probe back-off, and the probe never outlasts the request's timeoutMs. Co-Authored-By: Claude Opus 5.5 --- .../RunnerAppScreenCapture.swift | 9 ++- .../RunnerRequestPanel.swift | 41 ++++++++++++ .../RunnerTests+CommandExecution.swift | 7 +++ .../RunnerTests+Interaction.swift | 22 ++----- .../RunnerTests+Snapshot.swift | 21 ++++++- .../RunnerTests+SnapshotCapturePlan.swift | 2 + .../RunnerTests+SnapshotExecution.swift | 13 ++++ .../RunnerTests.swift | 1 + .../RunnerXCTestEventBridge.h | 7 ++- .../RunnerXCTestEventBridge.m | 19 ++++-- .../UnitTests/RunnerRequestPanelTests.swift | 54 ++++++++++++++++ .../RunnerTests+AppScreenCaptureTests.swift | 19 ++++++ .../RunnerTests+XCTestEventBridgeTests.swift | 12 +++- contracts/fixtures/dispatch-disclosure.json | 6 ++ docs/adr/0025-foldable-apple-panels.md | 8 ++- .../index-dispatch-disclosure.test.ts | 1 + .../runner-error-classification.test.ts | 1 + packages/apple-runner/src/runner-contract.ts | 7 ++- .../platform-apple/src/core/runner-client.ts | 5 +- .../src/foldable/orientation.test.ts | 2 +- .../src/foldable/orientation.ts | 23 ++++--- .../src/foldable/runner-panel.test.ts | 63 +++++++++++++++---- .../src/foldable/runner-panel.ts | 14 ++++- 23 files changed, 297 insertions(+), 60 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index 55761e8a57..0150fac0ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -21,11 +21,14 @@ enum RunnerAppScreenCaptureFailure: String, Error { case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED" case unresolvedScreen = "APP_SCREEN_UNRESOLVED" case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE" + case offLitPanel = "APP_SCREEN_OFF_LIT_PANEL" init(_ failure: RunnerApplicationScreenFailure) { switch failure { case .unresolvedWindow: self = .unresolvedWindow + case .offLitPanel: + self = .offLitPanel default: self = .unresolvedScreen } @@ -34,11 +37,13 @@ enum RunnerAppScreenCaptureFailure: String, Error { var message: String { switch self { case .unresolvedWindow: - return "Neither the target app nor the system surface resolved a window on the lit display, so no display could be named." + return "Neither the target app nor the system surface resolved a window, so no display could be named." case .unresolvedScreen: return "The resolved app window did not report the display it is shown on." case .unrenderableImage: return "The resolved display handed back an image that could not be drawn upright." + case .offLitPanel: + return "Every window of the target app is on the panel the host reports as dark." } } @@ -50,6 +55,8 @@ enum RunnerAppScreenCaptureFailure: String, Error { return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved." case .unrenderableImage: return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question." + case .offLitPanel: + return "Unfold or fold the device so the target app is on the lit panel, or bring it forward there; runner.log names the panel in AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW." } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index daa5bbf6dc..f8ea223fc4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -4,6 +4,11 @@ import XCTest // `commandExecutionQueue`, where the slot is written, and the surface resolvers read it on main. // A recording pump outlives its command, so `recordStart` reads the panel once and carries it. extension RunnerTests { + struct LitPanelSurface { + let window: XCUIElement + let frame: CGRect + } + func setRequestLitPanel(from command: Command) { var panel: RunnerLitPanel? if let fact = command.litPanel, fact.displayId != 0, fact.widthPt.isFinite, fact.heightPt.isFinite, @@ -13,6 +18,7 @@ extension RunnerTests { } requestPanelLock.lock() requestLitPanel = panel + requestLitSurface = nil requestPanelLock.unlock() } @@ -22,4 +28,39 @@ extension RunnerTests { defer { requestPanelLock.unlock() } return requestLitPanel } + + /// Books the lit panel's window for the rest of the request, or returns why it cannot. A request + /// that named no panel books nothing. Once booked, every surface read in the request answers with + /// this window, so no later read can fall back to the application's own coordinates. + func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { + guard let litPanel = requestSurfacePanel else { return nil } + switch Self.resolveLitPanelSurface(app: app, litPanel: litPanel) { + case .failure(let refusal): + return refusal + case .success(let surface): + requestPanelLock.lock() + requestLitSurface = surface + requestPanelLock.unlock() + return nil + } + } + + var bookedLitPanelSurface: LitPanelSurface? { + requestPanelLock.lock() + defer { requestPanelLock.unlock() } + return requestLitSurface + } + + static func resolveLitPanelSurface( + app: AnyObject, + litPanel: RunnerLitPanel + ) -> Result { + var window: AnyObject? + var frame = CGRect.null + let failure = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame) + guard failure == .none, let element = window as? XCUIElement else { + return .failure(RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure)) + } + return .success(LitPanelSurface(window: element, frame: frame)) + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index b6754691fc..013a202bc0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -14,6 +14,13 @@ extension RunnerTests { if command.invalidatesRememberedTextEntryTap { clearRememberedTextEntryTap() } + // A screenshot resolves its own surface, the target app and then the system surface. + if command.command != .screenshot, let refusal = bookRequestLitPanelSurface(app: activeApp) { + return Response( + ok: false, + error: ErrorPayload(code: refusal.rawValue, message: refusal.message, hint: refusal.hint) + ) + } switch command.command { case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime, .appState, .pasteboardWrite, .snapshot: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index b26e240b51..9d77369660 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -269,8 +269,11 @@ extension RunnerTests { /// back to `app.frame`: when the windows query fails because XCTest cannot serialize the app's /// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { - if let litPanel = requestSurfacePanel { - return resolveLitPanelWindow(app: app, litPanel: litPanel) + if requestSurfacePanel != nil { + if bookedLitPanelSurface == nil { + _ = bookRequestLitPanelSurface(app: app) + } + return bookedLitPanelSurface.map { ($0.window, $0.frame) } } for window in app.windows.allElementsBoundByIndex where window.exists { let frame = window.frame @@ -281,21 +284,6 @@ extension RunnerTests { return nil } - /// A foldable's surface: the window on the panel the request named as lit, never the first window. - func resolveLitPanelWindow( - app: XCUIApplication, - litPanel: RunnerLitPanel - ) -> (window: XCUIElement, frame: CGRect)? { - var window: AnyObject? - var frame = CGRect.null - guard RunnerResolveLitPanelWindow(app, litPanel, &window, &frame) == .none, - let element = window as? XCUIElement - else { - return nil - } - return (element, frame) - } - func onScreenWindowFrame(app: XCUIApplication) -> CGRect { resolveRunnerWindow(app: app).frame } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index f0ba0cc8d3..d271fac001 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -212,13 +212,18 @@ extension RunnerTests { parentIndex: nil ) // A multi-panel device lists windows on both panels under one application, and the application's - // frame names one panel. The screen is the lit panel's native box and the windows that name its - // display (ADR 0025). + // frame names one panel. The screen is the lit panel's native box and only the windows proven on + // its display (ADR 0025). let litPanel = requestSurfacePanel var rootChildren = context.rootSnapshot.children if let litPanel { root.rect = SnapshotRect(CGRect(x: 0, y: 0, width: litPanel.widthPt, height: litPanel.heightPt)) - rootChildren = rootChildren.filter { Self.snapshotDisplayID($0).map { $0 == litPanel.displayID } ?? true } + rootChildren = Self.windowsOnLitDisplay(rootChildren, litDisplayID: litPanel.displayID) { + Self.snapshotDisplayID($0) + } + if rootChildren.isEmpty { + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } } var nodes: [RawAXNode] = [root] if Self.canDescendAtRawDepth(0, hint: hint) { @@ -301,6 +306,16 @@ extension RunnerTests { ) } + /// The windows a lit-panel snapshot keeps: only those whose reported display is the lit one. A + /// window that reports no display is dropped, because it may be on the dark panel. + static func windowsOnLitDisplay( + _ windows: [Window], + litDisplayID: UInt, + displayID: (Window) -> UInt? + ) -> [Window] { + windows.filter { displayID($0) == litDisplayID } + } + /// The display a window snapshot reports, or nil when the runtime names none. static func snapshotDisplayID(_ snapshot: XCUIElementSnapshot) -> UInt? { var displayID: UInt? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index c47b77842a..f25d1ad2ff 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -162,6 +162,8 @@ extension RunnerTests { preferredBackend: options.preferredBackend ) #endif + // Only the tree tier can prove which panel a window is on, so a multi-panel request runs it alone. + let plan = plan == Self.regularVisiblePlan && requestSurfacePanel != nil ? [.recursiveTree] : plan let effective = Self.effectiveSnapshotCapturePlan( plan, xCTestChannelPenalized: xCTestChannelPenalized, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index b4063d9b0e..7e9b24e8c8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -99,6 +99,14 @@ extension RunnerTests { ) throws -> Response { let options = Self.presentationOptions(from: command) do { + let litPanelRefusal = try runMainThreadWork( + "snapshot_lit_panel", + timeout: Self.mainThreadExecutionTimeout, + timeoutError: Self.mainThreadExecutionTimeoutError + ) { + self.bookRequestLitPanelSurface(app: target.app) + } + if let litPanelRefusal { throw litPanelRefusal } var payload: DataPayload if options.raw { payload = try snapshotRaw(target: target, options: options) @@ -128,6 +136,11 @@ extension RunnerTests { } setNeedsPostSnapshotInteractionDelay() return Response(ok: true, data: payload) + } catch let refusal as RunnerAppScreenCaptureFailure { + return Response( + ok: false, + error: ErrorPayload(code: refusal.rawValue, message: refusal.message, hint: refusal.hint) + ) } catch let failure as SnapshotCaptureFailure { invalidateCachedTargetAfterSnapshotFailure() return Response( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b0a552ac94..ad556e2519 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -95,6 +95,7 @@ final class RunnerTests: XCTestCase { // Written on the command queue and read on main; see RunnerRequestPanel.swift. let requestPanelLock = NSLock() var requestLitPanel: RunnerLitPanel? + var requestLitSurface: LitPanelSurface? // iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when // hardware-keyboard input hides the software keyboard. A successful tap on a concrete text // input is a scoped witness for the immediately-following bare type; lifecycle and non-text diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 7c8b97954b..03330ab3bc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -29,6 +29,8 @@ typedef NS_ENUM(NSUInteger, RunnerApplicationScreenFailure) { RunnerApplicationScreenFailureUnresolvedWindow, // A resolved window whose screen or display identity the XCTest runtime would not answer. RunnerApplicationScreenFailureUnresolvedScreen, + // The application has windows, and every one of them is on a display other than the lit panel. + RunnerApplicationScreenFailureOffLitPanel, }; // The panel the host reports as lit (ADR 0025): its display and its native box in points. The @@ -44,8 +46,9 @@ typedef struct { #define AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS 8 // Resolves the window on `litPanel`: the first window on that display whose frame is the panel's -// box either way up, else the largest window on that display. No window on that display is -// `UnresolvedWindow`; a list the walk could not fully place is `UnresolvedScreen`. +// box either way up, else the largest window on that display. An unplaced window, or a list past +// the cap with no full-panel window in it, is `UnresolvedScreen`. Windows that are all on another +// display are `OffLitPanel`; no window at all is `UnresolvedWindow`. FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( id application, RunnerLitPanel litPanel, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 0289b60402..38d85dd9c2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -109,6 +109,8 @@ return @"no resolved application window"; case RunnerApplicationScreenFailureUnresolvedScreen: return @"no resolved window display ID"; + case RunnerApplicationScreenFailureOffLitPanel: + return @"every application window is off the lit panel"; case RunnerApplicationScreenFailureNone: return @"application screen resolved"; } @@ -240,9 +242,11 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( if (exception != nil || windows == nil) { return RunnerApplicationScreenFailureUnresolvedWindow; } - // A window the walk did not place could be the panel's surface, so only a fully placed list may - // book its largest window or claim the lit panel holds none. - BOOL incomplete = windows.count > AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS; + // A window the walk could not place may be the panel's surface, so it refuses every booking. A + // list past the cap refuses only when no full-panel window was found before it. + BOOL unplaced = NO; + BOOL truncated = windows.count > AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS; + BOOL placedAny = NO; id chosen = nil; id chosenScreen = nil; CGRect chosenFrame = CGRectNull; @@ -259,9 +263,10 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( continue; } if (read != RunnerApplicationScreenFailureNone) { - incomplete = YES; + unplaced = YES; continue; } + placedAny = YES; if (displayID != litPanel.displayID) { continue; } @@ -269,7 +274,7 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( chosen = candidate; chosenScreen = candidateScreen; chosenFrame = frame; - incomplete = NO; + truncated = NO; break; } if (chosen == nil || @@ -279,13 +284,15 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( chosenFrame = frame; } } - if (incomplete) { + if (unplaced || truncated) { outcome = RunnerApplicationScreenFailureUnresolvedScreen; } else if (chosen != nil) { outcome = RunnerApplicationScreenFailureNone; *window = chosen; *windowFrame = chosenFrame; *screen = chosenScreen; + } else if (placedAny) { + outcome = RunnerApplicationScreenFailureOffLitPanel; } NSLog( @"AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=%lu(%gx%g) windows=%lu outcome=%@ frame=(%g,%g %gx%g)", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 1d604a8c20..b667a5ea13 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -35,3 +35,57 @@ extension RunnerTests { } } #endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + /// A window whose display the runtime does not name may be on the dark panel, so it is dropped. + func testLitPanelSnapshotKeepsOnlyWindowsProvenOnTheLitDisplay() { + let windows: [(name: String, displayID: UInt?)] = [("outer", 1), ("unknown", nil), ("inner", 3)] + let kept = Self.windowsOnLitDisplay(windows, litDisplayID: 3) { $0.displayID } + XCTAssertEqual(kept.map(\.name), ["inner"]) + } + +#if os(iOS) + /// Runs `command` against the launched app while the request names display 999, which none of + /// the app's windows is on. + @MainActor + private func responseWithTheAppOffTheLitPanel( + _ json: String, + _ run: (Command) throws -> Response + ) throws -> Response { + app.launch() + mainOwned.app = app + defer { + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture(json) + setRequestLitPanel(from: command) + return try run(command) + } + + @MainActor + func testCoordinateTapRefusesBeforeDispatchWhenTheAppIsOffTheLitPanel() throws { + var fellBack = true + let response = try responseWithTheAppOffTheLitPanel( + #"{"command":"tap","x":40,"y":120,"synthesized":true,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + ) { command in + let response = try executeOnMainPrepared(command: command, activeApp: app) + fellBack = resolveRunnerWindowElement(app: app) != nil + return response + } + XCTAssertEqual(response.error?.code, "APP_SCREEN_OFF_LIT_PANEL") + XCTAssertFalse(fellBack, "no read in the request may fall back to the app") + } + + @MainActor + func testSnapshotRefusesWhenTheAppIsOffTheLitPanel() throws { + let response = try responseWithTheAppOffTheLitPanel( + #"{"command":"snapshot","interactiveOnly":true,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + ) { try executeSnapshotDispatched(command: $0) } + XCTAssertEqual(response.error?.code, "APP_SCREEN_OFF_LIT_PANEL") + } +#endif +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift index 74b65986a2..aa4de3af57 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AppScreenCaptureTests.swift @@ -42,6 +42,7 @@ extension RunnerTests { RunnerAppScreenCaptureFailure(.unresolvedScreen).rawValue, "APP_SCREEN_UNRESOLVED" ) + XCTAssertEqual(RunnerAppScreenCaptureFailure(.offLitPanel).rawValue, "APP_SCREEN_OFF_LIT_PANEL") // The display resolved and the image did not: a distinct code, because an operator chasing a // panel problem is chasing the wrong thing here. This one has no bridge counterpart, so it is // named on the Swift reason rather than mapped from one. @@ -136,6 +137,24 @@ extension RunnerTests { XCTAssertEqual(refusal.rawValue, "APP_SCREEN_UNRESOLVED") } + /// A requested app whose windows are all on the dark panel is refused: the system surface would + /// answer with a capture of a different app. + func testObservedScreenCaptureRefusesAnAppOffTheLitPanelWithoutAskingTheSystemSurface() { + var askedSystemSurface = false + let outcome = selectObservedScreenCapture( + resolving: { .failure(RunnerAppScreenCaptureFailure(.offLitPanel)) }, + fallingBack: { + askedSystemSurface = true + return .failure(.unrenderableImage) + } + ) + XCTAssertFalse(askedSystemSurface) + guard case .failure(let refusal) = outcome else { + return XCTFail("an app off the lit panel must not become a capture") + } + XCTAssertEqual(refusal, .offLitPanel) + } + func testObservedScreenCaptureKeepsTheSessionAppCaptureItResolved() { var askedSystemSurface = false let captured = CapturedAppScreen( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 9fb3383b79..f05252dcf0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -298,7 +298,17 @@ extension RunnerTests { DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1), DisplayWindowFixture(frame: CGRect(x: 0, y: 0, width: 466, height: 678), displayID: 1), ]) - XCTAssertEqual(walk.failure, .unresolvedWindow) + XCTAssertEqual(walk.failure, .offLitPanel) + XCTAssertNil(walk.window) + XCTAssertEqual(resolveLitPanelWindow([]).failure, .unresolvedWindow, "no window at all") + } + + /// A window the walk could not place may itself be the panel's surface, so a full-panel window + /// read after it is not booked. + func testLitPanelWindowRefusesAFullPanelWindowReadAfterAnUnplacedOne() { + let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let walk = resolveLitPanelWindow([ScreenlessWindowFixture(), home]) + XCTAssertEqual(walk.failure, .unresolvedScreen) XCTAssertNil(walk.window) } diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 66616ff5ba..446bd0283a 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -247,6 +247,12 @@ "trigger": "structured runner reply with code APP_SCREEN_CAPTURE_UNRENDERABLE: a refusal that ran nothing", "dispatched": "no" }, + { + "id": "ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL", + "producer": "ios-runner", + "trigger": "structured runner reply with code APP_SCREEN_OFF_LIT_PANEL: a refusal that ran nothing", + "dispatched": "no" + }, { "id": "ios-runner.reply.ELEMENT_NOT_FOUND", "producer": "ios-runner", diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 81f3d3fc24..832b2c931f 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -714,9 +714,13 @@ Android reports the app window on the next snapshot instead of a panel. SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -Decision: the host names the lit panel in each runner request that resolves a screen. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. A request without the field keeps the historic first-window route unchanged. +Decision: the host names the lit panel in each runner request that resolves a screen. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. A request without the field keeps the historic first-window route unchanged. -With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A walk that could not place every window refuses with `APP_SCREEN_UNRESOLVED`. A list with no window on the lit display is `APP_SCREEN_WINDOW_UNRESOLVED`, and capture then asks the system surface. Dispatch, the coordinate fallback, the viewport, and capture use that one window. A regular snapshot keeps only the application's windows whose element snapshot reports the lit display. Its root takes the panel's native box. +With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. + +The request books the window before it acts. A refusal is returned before any gesture or snapshot, and every later read in the request uses the booked window. Dispatch, the coordinate fallback, and the viewport therefore never fall back to the application's own frame. Capture asks the system surface only when the app has no window. An app on the dark panel is refused instead. + +A regular snapshot runs only the tree tier, because only that tier can prove which display a window is on. It keeps only the windows whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the panel's native box. Rejected alternatives: diff --git a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts index 3fcb985970..60a123b8db 100644 --- a/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/apple-runner/src/__tests__/index-dispatch-disclosure.test.ts @@ -178,6 +178,7 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_CAPTURE_UNRENDERABLE': () => replyFailure('APP_SCREEN_CAPTURE_UNRENDERABLE'), + 'ios-runner.reply.APP_SCREEN_OFF_LIT_PANEL': () => replyFailure('APP_SCREEN_OFF_LIT_PANEL'), 'ios-runner.reply.ELEMENT_NOT_FOUND': () => replyFailure('ELEMENT_NOT_FOUND'), 'ios-runner.reply.ELEMENT_OFFSCREEN': () => replyFailure('ELEMENT_OFFSCREEN'), 'ios-runner.reply.AMBIGUOUS_MATCH': () => replyFailure('AMBIGUOUS_MATCH'), diff --git a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts index 7c881cc936..65e4fd1853 100644 --- a/packages/apple-runner/src/__tests__/runner-error-classification.test.ts +++ b/packages/apple-runner/src/__tests__/runner-error-classification.test.ts @@ -426,6 +426,7 @@ test('runner_endpoint_probe_exhausted is decided by the typed reason, not the me test('a refused screen capture keeps the runner reason and stays off the wire code', () => { assert.deepEqual([...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].sort(), [ 'APP_SCREEN_CAPTURE_UNRENDERABLE', + 'APP_SCREEN_OFF_LIT_PANEL', 'APP_SCREEN_UNRESOLVED', 'APP_SCREEN_WINDOW_UNRESOLVED', ]); diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 86d84f907d..21fcddb643 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -275,12 +275,17 @@ export const SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE = 'SCROLL_KEYBOARD_OCC const RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_WINDOW_UNRESOLVED'; const RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE = 'APP_SCREEN_UNRESOLVED'; const RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE = 'APP_SCREEN_CAPTURE_UNRENDERABLE'; +const RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE = 'APP_SCREEN_OFF_LIT_PANEL'; -/** Every runner code meaning "this capture did not happen", covering the display and the image. */ +/** + * Every runner code meaning "this capture did not happen", covering the display, the image, and the + * panel: on a foldable, a target app whose windows are all on the dark panel is refused. + */ export const RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES: ReadonlySet = new Set([ RUNNER_SCREEN_WINDOW_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_UNRESOLVED_RUNNER_CODE, RUNNER_SCREEN_CAPTURE_UNRENDERABLE_RUNNER_CODE, + RUNNER_SCREEN_OFF_LIT_PANEL_RUNNER_CODE, ]); /** diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index a3929acd75..a1ac67f379 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -33,7 +33,10 @@ export async function runAppleRunnerCommand( return await runHostBoundAppleRunnerCommand(device, command, options); } const { resolveRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); - const litPanel = await resolveRunnerRequestLitPanel(device, resolveRunnerRequestSignal(options)); + const litPanel = await resolveRunnerRequestLitPanel(device, { + signal: resolveRunnerRequestSignal(options), + requestTimeoutMs: command.timeoutMs, + }); return await runHostBoundAppleRunnerCommand( device, litPanel ? { ...command, litPanel } : command, diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index b954d445e1..725ef43db5 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -217,7 +217,7 @@ test('a rotation is not routed by a pending panel-table retry', async () => { throw new Error(`unexpected tool call: ${args.join(' ')}`); }, }); - await withAppleToolProvider(failing, () => readFoldablePanelTable(device, undefined)); + await withAppleToolProvider(failing, () => readFoldablePanelTable(device, {})); expect(tableFailures).toBe(1); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 32aa79ec4c..6184e25ebb 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -29,6 +29,15 @@ const ORIENTATION_UNVERIFIED_HINT = /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); +// Every surface command reads the panel table, so one that would not answer is not asked again +// for a few seconds: a failing probe can cost its whole timeout. +const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); + +function recordPanelTable(device: DeviceInfo, inventory: AppleDisplayInventory): void { + multiPanelByDevice.set(device.id, inventory.multiScreen); + unansweredPanelTables.delete(device.id); +} + /** * Whether a rotation must go through guest HID: a simulator the `fold` fact admits that has more * than one integrated panel. XCUIDevice rotation has no effect there, and its readback repeats @@ -43,14 +52,10 @@ export async function isAppleFoldableSimulator( if (known !== undefined) return known; const inventory = await queryAppleDisplayInventory(device, { signal }); if (inventory.unresolved) return false; - multiPanelByDevice.set(device.id, inventory.multiScreen); + recordPanelTable(device, inventory); return inventory.multiScreen; } -// Every surface command reads the panel table, so one that would not answer is not asked again -// for a few seconds: a failing probe can cost its whole timeout. -const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); - /** * One panel-table read of a multi-panel simulator, or undefined for a device the `fold` fact refuses, * one known to have a single panel, or a table that would not answer. It records the panel count it @@ -58,17 +63,17 @@ const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); */ export async function readFoldablePanelTable( device: DeviceInfo, - signal: AbortSignal | undefined, + options: { signal?: AbortSignal; timeoutMs?: number }, ): Promise { if (!appleFoldableFacts(device).setFoldPose.available) return undefined; if (multiPanelByDevice.get(device.id) === false) return undefined; if (unansweredPanelTables.get(device.id)) return undefined; - const inventory = await queryAppleDisplayInventory(device, { signal }); + const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) { - if (!signal?.aborted) unansweredPanelTables.set(device.id, true); + if (!options.signal?.aborted) unansweredPanelTables.set(device.id, true); return undefined; } - multiPanelByDevice.set(device.id, inventory.multiScreen); + recordPanelTable(device, inventory); return inventory.multiScreen ? inventory : undefined; } diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index c4bf72957e..9150fa60b9 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -1,12 +1,14 @@ import assert from 'node:assert/strict'; import { beforeEach, test, vi } from 'vitest'; import { bindTestCommandHost } from '@agent-device/test-kit'; +import { withAppleRunnerProvider } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; vi.mock('../foldable/fold-helper-cache.ts', () => ({ ensureFoldHelperBinary: vi.fn(async () => ({ path: '/cache/fold-helper' })), })); -vi.mock('../core/devicectl.ts', () => ({ +vi.mock('../core/devicectl.ts', async (importOriginal) => ({ + ...(await importOriginal()), runIosDevicectlJsonRequest: vi.fn(async () => ({ ok: true as const, payload: { result: { displays: [] } }, @@ -14,6 +16,12 @@ vi.mock('../core/devicectl.ts', () => ({ })); import { runIosDevicectlJsonRequest } from '../core/devicectl.ts'; +import { runAppleRunnerCommand } from '../core/runner-client.ts'; +import { + recordingRunnerProvider, + type RecordedRunnerCall, +} from '../__tests__/recording-runner-provider.ts'; +import { isAppleFoldableSimulator } from './orientation.ts'; import { resolveRunnerRequestLitPanel } from './runner-panel.ts'; import { IOS_DEVICE, IOS_SIMULATOR } from '../__tests__/device-fixtures.ts'; import { @@ -49,7 +57,7 @@ test('a multi-panel request names the lit panel in the units the runner compares answering(IPHONE_DUO_CLOSED); // A closed Duo lights its outer panel: 1398x2034 px at pointScale 3, which is the 466x678 the // runner finds as that panel's SpringBoard window. - assert.deepEqual(await resolveRunnerRequestLitPanel(simulator(), undefined), { + assert.deepEqual(await resolveRunnerRequestLitPanel(simulator(), {}), { displayId: 1, widthPt: 466, heightPt: 678, @@ -59,11 +67,11 @@ test('a multi-panel request names the lit panel in the units the runner compares test('each request reads the panel table once, so a fold moves the surface', async () => { const device = simulator(); answering(IPHONE_DUO_CLOSED); - assert.equal((await resolveRunnerRequestLitPanel(device, undefined))?.displayId, 1); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); assert.equal(mockPanelTable.mock.calls.length, 1, 'the panel count comes from the same read'); answering(IPHONE_DUO_OPEN); assert.equal( - (await resolveRunnerRequestLitPanel(device, undefined))?.displayId, + (await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3, 'an opened device must be asked again, not answered from the panel that just went dark', ); @@ -73,17 +81,17 @@ test('each request reads the panel table once, so a fold moves the surface', asy test('a device with one panel sends no panel and is never asked again', async () => { const device = simulator(); answering(IPHONE_17_SINGLE_PANEL); - assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); const reads = mockPanelTable.mock.calls.length; - assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); assert.equal(mockPanelTable.mock.calls.length, reads, 'one panel is hardware, probed once'); }); test('a panel table that would not answer is not asked again by the next request', async () => { const device = simulator(); mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); - assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); - assert.equal(await resolveRunnerRequestLitPanel(device, undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); assert.equal(mockPanelTable.mock.calls.length, 1); }); @@ -92,9 +100,9 @@ test('a cancelled panel-table read leaves the next request free to ask', async ( mockPanelTable.mockRejectedValueOnce(new Error('devicectl aborted by the caller')); const cancelled = new AbortController(); cancelled.abort(); - assert.equal(await resolveRunnerRequestLitPanel(device, cancelled.signal), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, { signal: cancelled.signal }), undefined); answering(IPHONE_DUO_CLOSED); - assert.equal((await resolveRunnerRequestLitPanel(device, undefined))?.displayId, 1); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); }); test('a multi-panel device whose panel power decides nothing sends no panel', async () => { @@ -108,7 +116,7 @@ test('a multi-panel device whose panel power decides nothing sends no panel', as })), }, }); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); }); test('a multi-panel device whose panel table is incomplete sends no panel', async () => { @@ -119,12 +127,41 @@ test('a multi-panel device whose panel table is incomplete sends no panel', asyn displays: [outer, { ...inner, nativeSize: undefined }], }, }); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); }); test('a device the fold fact refuses never costs a panel table read', async () => { // A physical iPhone is folded by hand (ADR 0025) and no physical foldable leaf is in the // supported matrix, so its runner keeps the surface route the request always sent. - assert.equal(await resolveRunnerRequestLitPanel(IOS_DEVICE, undefined), undefined); + assert.equal(await resolveRunnerRequestLitPanel(IOS_DEVICE, {}), undefined); assert.equal(mockPanelTable.mock.calls.length, 0); }); + +test('a resolved read lifts the back-off a failed one opened', async () => { + const device = simulator(); + mockPanelTable.mockRejectedValueOnce(new Error('devicectl does not report displays')); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + answering(IPHONE_DUO_CLOSED); + // A rotation reads the table on its own route; its answer clears the back-off for surface reads. + assert.equal(await isAppleFoldableSimulator(device, undefined), true); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); +}); + +test('the probe takes the request timeout when it is the shorter one', async () => { + answering(IPHONE_DUO_CLOSED); + await resolveRunnerRequestLitPanel(simulator(), { requestTimeoutMs: 1_200 }); + assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 1_200); + await resolveRunnerRequestLitPanel(simulator(), { requestTimeoutMs: 60_000 }); + assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 5_000); +}); + +test('a surface command probes the panel table within its own timeout', async () => { + answering(IPHONE_DUO_CLOSED); + const device = simulator({ name: 'iPhone Duo' }); + const calls: RecordedRunnerCall[] = []; + await withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 1_200 }), + ); + assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 1_200); + assert.equal(calls[0]?.command.litPanel?.displayId, 1); +}); diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 8c7843b637..48cf0a8ce8 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -1,17 +1,25 @@ import type { RunnerCommand } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS } from '../core/config.ts'; import { appleLitPanel } from '../core/display-inventory.ts'; import { readFoldablePanelTable } from './orientation.ts'; /** * The lit panel a runner request names, read fresh per request because a fold changes it (ADR - * 0025). Undefined keeps the runner's historic first-window route. + * 0025). The read never outlasts the request's own `timeoutMs`. Undefined keeps the runner's + * historic first-window route. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo, - signal: AbortSignal | undefined, + options: { signal?: AbortSignal; requestTimeoutMs?: number }, ): Promise { - const inventory = await readFoldablePanelTable(device, signal); + const inventory = await readFoldablePanelTable(device, { + signal: options.signal, + timeoutMs: Math.min( + options.requestTimeoutMs ?? IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, + IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, + ), + }); return inventory ? appleLitPanel(inventory) : undefined; } From 3104c1a74299c2ae26608152fad9ad4432501268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 31 Oct 2026 21:54:29 +0100 Subject: [PATCH 05/29] fix(ios): order the panel-table back-off behind newer resolved reads A failed read that finishes after a newer read resolved no longer sets the 5-second back-off: each read takes a sequence number, and a failure yields to any later read that resolved. The ADR now says the snapshot traverses the tree and keeps every window proven on the lit display. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 4 ++-- .../src/foldable/orientation.ts | 20 +++++++++++++++---- .../src/foldable/runner-panel.test.ts | 18 +++++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 832b2c931f..51046996d8 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -718,9 +718,9 @@ Decision: the host names the lit panel in each runner request that resolves a sc With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. -The request books the window before it acts. A refusal is returned before any gesture or snapshot, and every later read in the request uses the booked window. Dispatch, the coordinate fallback, and the viewport therefore never fall back to the application's own frame. Capture asks the system surface only when the app has no window. An app on the dark panel is refused instead. +The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window, so they never fall back to the application's own frame. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. -A regular snapshot runs only the tree tier, because only that tier can prove which display a window is on. It keeps only the windows whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the panel's native box. +A regular snapshot runs only the tree tier, because only that tier can prove which display a window is on. It traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the panel's native box. Rejected alternatives: diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index 6184e25ebb..e3b58bcb25 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -32,10 +32,19 @@ const multiPanelByDevice = new Map(); // Every surface command reads the panel table, so one that would not answer is not asked again // for a few seconds: a failing probe can cost its whole timeout. const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); +// Reads overlap, so a failure may finish after a read that started later and already resolved. +// Each read takes a sequence number, and a failure never overrides a newer resolved read. +let panelTableReads = 0; +const latestResolvedRead = new Map(); -function recordPanelTable(device: DeviceInfo, inventory: AppleDisplayInventory): void { +function recordPanelTable( + device: DeviceInfo, + inventory: AppleDisplayInventory, + read: number, +): void { multiPanelByDevice.set(device.id, inventory.multiScreen); unansweredPanelTables.delete(device.id); + latestResolvedRead.set(device.id, Math.max(read, latestResolvedRead.get(device.id) ?? 0)); } /** @@ -50,9 +59,10 @@ export async function isAppleFoldableSimulator( if (!appleFoldableFacts(device).setFoldPose.available) return false; const known = multiPanelByDevice.get(device.id); if (known !== undefined) return known; + const read = ++panelTableReads; const inventory = await queryAppleDisplayInventory(device, { signal }); if (inventory.unresolved) return false; - recordPanelTable(device, inventory); + recordPanelTable(device, inventory, read); return inventory.multiScreen; } @@ -68,12 +78,14 @@ export async function readFoldablePanelTable( if (!appleFoldableFacts(device).setFoldPose.available) return undefined; if (multiPanelByDevice.get(device.id) === false) return undefined; if (unansweredPanelTables.get(device.id)) return undefined; + const read = ++panelTableReads; const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) { - if (!options.signal?.aborted) unansweredPanelTables.set(device.id, true); + const newerReadResolved = (latestResolvedRead.get(device.id) ?? 0) > read; + if (!options.signal?.aborted && !newerReadResolved) unansweredPanelTables.set(device.id, true); return undefined; } - recordPanelTable(device, inventory); + recordPanelTable(device, inventory, read); return inventory.multiScreen ? inventory : undefined; } diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index 9150fa60b9..580e7a103b 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -165,3 +165,21 @@ test('a surface command probes the panel table within its own timeout', async () assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 1_200); assert.equal(calls[0]?.command.litPanel?.displayId, 1); }); + +test('a slow failed read does not back off over a newer read that resolved', async () => { + const device = simulator(); + let failSlowRead!: (error: Error) => void; + mockPanelTable.mockImplementationOnce( + () => new Promise((_resolve, reject) => (failSlowRead = reject)), + ); + const slow = resolveRunnerRequestLitPanel(device, {}); + answering(IPHONE_DUO_CLOSED); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 1); + failSlowRead(new Error('devicectl does not report displays')); + assert.equal(await slow, undefined); + assert.equal( + (await resolveRunnerRequestLitPanel(device, {}))?.displayId, + 1, + 'the stale failure must not hold the next request on the historic route', + ); +}); From 0f8e83dbb642f09b41eafa9ebf3edfb09224b490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 31 Oct 2026 22:04:27 +0100 Subject: [PATCH 06/29] docs(adr): count the off-lit-panel capture refusal ADR 0025's capture contract listed three fail-closed cases. The lit-panel amendment added a fourth, `offLitPanel` (APP_SCREEN_OFF_LIT_PANEL), and changed when capture asks the system surface on a multi-panel device. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 51046996d8..be67f6d5a3 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -149,12 +149,13 @@ The vitest twin and runner unit-test lane decode this golden table. Contract drift therefore fails CI on both sides. Capture first asks for the target app's window, then the system surface's window. +On a multi-panel device, it asks the system surface only when the app has no window at all. SpringBoard owns the home-screen window. Thus, a session without an app, or with a stopped app, can still resolve a window's display. The result always names a display occupied by a window. It never defaults to `XCUIScreen.main`, which is dark on a Duo when the app uses the inner panel. -A capture fails closed only in three cases. Neither window resolves. A resolved window names no display. A resolved display yields an unrenderable image. -The typed reasons are `unresolvedWindow`, `unresolvedScreen`, and `unrenderableImage` (`RunnerAppScreenCapture.swift`). +A capture fails closed only in four cases. Neither window resolves. A resolved window names no display. A resolved display yields an unrenderable image. On a multi-panel device, every window of the target app is on the dark panel. +The typed reasons are `unresolvedWindow`, `unresolvedScreen`, `unrenderableImage`, and `offLitPanel` (`APP_SCREEN_OFF_LIT_PANEL`) (`RunnerAppScreenCapture.swift`). The query uses both `exists` and frame evidence to refuse an app without a window. Such an app returns `(0,0 0x0)` from `frame` without an exception. Geometry alone cannot prove that no window exists. From 153a101e820459f4450f90b8b6520aec180d0ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 19:15:19 +0100 Subject: [PATCH 07/29] fix(ios): keep every lit-panel read on proven lit windows - Element reads (readText, the text-input lookup, selector queries) search only under the windows that report the lit display. One bridge owner, RunnerCopyLitPanelWindows, provides them. - The lit panel is written onto the main queue behind work already queued there, so work a previous command abandoned keeps its own panel. The booking is keyed to its app. - A snapshot books in command preparation, publishes the booked frame as its root, refuses a pin to a backend that cannot prove the panel, and filters --raw too. - A known multi-panel device whose panel table goes unread is refused with a typed panel-table-unreadable error. The runner gets the budget the panel read left. Co-Authored-By: Claude Opus 5.5 --- .../RunnerAppScreenCapture.swift | 5 + .../RunnerRequestPanel.swift | 97 ++++++++++--------- .../RunnerTests+CommandDispatch.swift | 2 +- .../RunnerTests+CommandExecution.swift | 7 +- .../RunnerTests+Interaction.swift | 29 +++--- .../RunnerTests+Lifecycle.swift | 2 +- .../RunnerTests+Navigation.swift | 5 +- .../RunnerTests+Snapshot.swift | 40 ++++---- .../RunnerTests+SnapshotCapturePlan.swift | 21 +++- .../RunnerTests+SnapshotCaptureTarget.swift | 2 + .../RunnerTests+SnapshotExecution.swift | 32 +++--- .../RunnerTests+TextInputProbe.swift | 3 +- .../RunnerTests.swift | 5 +- .../RunnerXCTestEventBridge.h | 5 + .../RunnerXCTestEventBridge.m | 33 +++++-- .../UnitTests/RunnerRequestPanelTests.swift | 66 ++++++++++++- .../RunnerTests+XCTestEventBridgeTests.swift | 16 +++ docs/adr/0025-foldable-apple-panels.md | 8 +- .../platform-apple/src/core/runner-client.ts | 9 +- .../src/foldable/orientation.test.ts | 2 +- .../src/foldable/orientation.ts | 38 +++++--- .../src/foldable/runner-panel.test.ts | 30 +++++- .../src/foldable/runner-panel.ts | 25 ++++- 23 files changed, 337 insertions(+), 145 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index 0150fac0ba..fa81a85bf4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -34,6 +34,11 @@ enum RunnerAppScreenCaptureFailure: String, Error { } } + /// The refusal as a runner reply. + var response: Response { + Response(ok: false, error: ErrorPayload(code: rawValue, message: message, hint: hint)) + } + var message: String { switch self { case .unresolvedWindow: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index f8ea223fc4..9454401a9b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -1,14 +1,23 @@ import XCTest -// The lit panel of the command in flight (ADR 0025). Commands run one at a time on -// `commandExecutionQueue`, where the slot is written, and the surface resolvers read it on main. -// A recording pump outlives its command, so `recordStart` reads the panel once and carries it. -extension RunnerTests { - struct LitPanelSurface { - let window: XCUIElement - let frame: CGRect - } +// The lit panel of the command in flight (ADR 0025). It is main-owned, and `executeAccepted` writes +// it onto the main queue ahead of the command's own main-thread work. Main work a previous command +// abandoned therefore runs first and still reads that command's panel. A recording pump outlives +// its command, so `recordStart` reads the panel once and carries it. +struct LitPanelSurface { + let app: XCUIApplication + let window: XCUIElement + let frame: CGRect +} + +/// The command's lit panel and what it booked on it. Read and written only on the main thread. +final class RunnerLitPanelState { + var panel: RunnerLitPanel? + var surface: LitPanelSurface? + var windows: (app: XCUIApplication, elements: [XCUIElement])? +} +extension RunnerTests { func setRequestLitPanel(from command: Command) { var panel: RunnerLitPanel? if let fact = command.litPanel, fact.displayId != 0, fact.widthPt.isFinite, fact.heightPt.isFinite, @@ -16,51 +25,49 @@ extension RunnerTests { { panel = RunnerLitPanel(displayID: fact.displayId, widthPt: fact.widthPt, heightPt: fact.heightPt) } - requestPanelLock.lock() - requestLitPanel = panel - requestLitSurface = nil - requestPanelLock.unlock() - } - - /// The lit panel the current request named, or nil for a single-panel request. - var requestSurfacePanel: RunnerLitPanel? { - requestPanelLock.lock() - defer { requestPanelLock.unlock() } - return requestLitPanel - } - - /// Books the lit panel's window for the rest of the request, or returns why it cannot. A request - /// that named no panel books nothing. Once booked, every surface read in the request answers with - /// this window, so no later read can fall back to the application's own coordinates. - func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { - guard let litPanel = requestSurfacePanel else { return nil } - switch Self.resolveLitPanelSurface(app: app, litPanel: litPanel) { - case .failure(let refusal): - return refusal - case .success(let surface): - requestPanelLock.lock() - requestLitSurface = surface - requestPanelLock.unlock() - return nil + let state = litPanelState + let write = { + state.panel = panel + state.surface = nil + state.windows = nil + } + if Thread.isMainThread { + write() + } else { + DispatchQueue.main.async(execute: write) } } - var bookedLitPanelSurface: LitPanelSurface? { - requestPanelLock.lock() - defer { requestPanelLock.unlock() } - return requestLitSurface - } + /// The lit panel the current request named, or nil for a single-panel request. Main only. + var requestLitPanel: RunnerLitPanel? { litPanelState.panel } - static func resolveLitPanelSurface( - app: AnyObject, - litPanel: RunnerLitPanel - ) -> Result { + /// Books `app`'s window on the lit panel for the rest of the request, or returns why it cannot. A + /// request that named no panel books nothing. Later surface reads of `app` answer with this + /// window, so none of them can fall back to the application's own coordinates. + func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { + guard let litPanel = requestLitPanel else { return nil } + if let booked = litPanelState.surface, booked.app === app { return nil } var window: AnyObject? var frame = CGRect.null let failure = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame) guard failure == .none, let element = window as? XCUIElement else { - return .failure(RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure)) + return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - return .success(LitPanelSurface(window: element, frame: frame)) + litPanelState.surface = LitPanelSurface(app: app, window: element, frame: frame) + return nil + } + + /// Where an element read searches: the application, or on a lit-panel request only the windows + /// proven on the lit display, so a dark-panel element can never answer. + func elementSearchRoots(app: XCUIApplication) -> [XCUIElement] { + guard let litPanel = requestLitPanel else { return [app] } + if let cached = litPanelState.windows, cached.app === app { return cached.elements } + let windows = RunnerCopyLitPanelWindows(app, litPanel).compactMap { $0 as? XCUIElement } + litPanelState.windows = (app, windows) + return windows + } + + func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) -> [XCUIElementQuery] { + elementSearchRoots(app: app).map { $0.descendants(matching: type) } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index 8b77e09315..2af1c4faf0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -434,7 +434,7 @@ extension RunnerTests { fps: command.fps.map { Int32($0) } ) // The pump outlives this request, so the clip keeps the panel its `recordStart` named. - let clipLitPanel = requestSurfacePanel + let clipLitPanel = requestLitPanel try startRecording(recorder) { [weak self] in guard let self else { return .failure(.unresolvedScreen) } return self.captureRunnerFrameResult(app: activeApp, litPanel: clipLitPanel) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 013a202bc0..03e78fd8ed 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -16,10 +16,7 @@ extension RunnerTests { } // A screenshot resolves its own surface, the target app and then the system surface. if command.command != .screenshot, let refusal = bookRequestLitPanelSurface(app: activeApp) { - return Response( - ok: false, - error: ErrorPayload(code: refusal.rawValue, message: refusal.message, hint: refusal.hint) - ) + return refusal.response } switch command.command { case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime, @@ -496,7 +493,7 @@ extension RunnerTests { // which app is foregrounded in order to see which display it is on (#2728). switch captureObservedScreen( app: resolveAppWithoutActivation(command: command), - litPanel: requestSurfacePanel + litPanel: requestLitPanel ) { case .failure(let failure): return Response( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 9d77369660..243ce1c208 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -168,8 +168,10 @@ extension RunnerTests { func findElement(app: XCUIApplication, text: String) -> XCUIElement? { let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text) - let element = app.descendants(matching: .any).matching(predicate).firstMatch - return element.exists ? element : nil + return elementQueries(app: app, matching: .any) + .lazy + .map { $0.matching(predicate).firstMatch } + .first { $0.exists } } func findElement( @@ -198,7 +200,8 @@ extension RunnerTests { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } - let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex + let matches = elementQueries(app: app, matching: .any) + .flatMap { $0.matching(predicate).allElementsBoundByIndex } .filter(\.exists) let facts = matches.map { element in SelectorCandidateFacts( @@ -236,7 +239,8 @@ extension RunnerTests { if frame.isEmpty { return false } - return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: app.frame) + let windowFrame = requestLitPanel == nil ? app.frame : onScreenWindowFrame(app: app) + return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: windowFrame) } // The tappable on-screen viewport. app.frame is unsuitable: it unions @@ -260,7 +264,8 @@ extension RunnerTests { /// (`RunnerResolveWindowDisplayID`). func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) { guard let resolved = resolveRunnerWindowElement(app: app) else { - return (nil, app.frame) + // A lit-panel request never falls back to the application, whose frame names another panel. + return (nil, requestLitPanel == nil ? app.frame : .null) } return (resolved.window, resolved.frame) } @@ -269,11 +274,10 @@ extension RunnerTests { /// back to `app.frame`: when the windows query fails because XCTest cannot serialize the app's /// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { - if requestSurfacePanel != nil { - if bookedLitPanelSurface == nil { - _ = bookRequestLitPanelSurface(app: app) - } - return bookedLitPanelSurface.map { ($0.window, $0.frame) } + if requestLitPanel != nil { + _ = bookRequestLitPanelSurface(app: app) + guard let booked = litPanelState.surface, booked.app === app else { return nil } + return (booked.window, booked.frame) } for window in app.windows.allElementsBoundByIndex where window.exists { let frame = window.frame @@ -366,7 +370,8 @@ extension RunnerTests { } } - let candidates = app.descendants(matching: .any).allElementsBoundByIndex + let candidates = elementQueries(app: app, matching: .any) + .flatMap { $0.allElementsBoundByIndex } .filter { element in element.exists && !element.frame.isEmpty && element.frame.contains(point) } @@ -632,7 +637,7 @@ extension RunnerTests { // anchor misses the button. Reference frames and the synthesized display ID still resolve // through `resolveRunnerWindow`. A foldable anchors at the lit panel's window instead, because // the app origin names the outer panel whichever one is lit. - if requestSurfacePanel == nil { + if requestLitPanel == nil { let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 2597b27669..7d08d239fe 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -52,7 +52,7 @@ extension RunnerTests { /// session window falls to the system surface's window, which is what the home screen is. macOS /// keeps the host display it always recorded. func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? { - switch captureRunnerFrameResult(app: app, litPanel: requestSurfacePanel) { + switch captureRunnerFrameResult(app: app, litPanel: requestLitPanel) { case .success(let captured): return captured.image case .failure: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 1687991d0f..db1f9f9a94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -255,9 +255,10 @@ extension RunnerTests { ) -> Result { #if os(iOS) return Self.navigationVisualSample( - from: context.resolvedWindow == nil + // A lit-panel request never takes the sole-screen route: a foldable has no sole screen. + from: context.resolvedWindow == nil && requestLitPanel == nil ? captureSoleAppScreen(app: app) - : captureResolvedAppScreen(app: app, litPanel: requestSurfacePanel), + : captureResolvedAppScreen(app: app, litPanel: requestLitPanel), encoding: { runnerPngData(for: $0.image) } ) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index d271fac001..3fe1721c82 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -211,20 +211,11 @@ extension RunnerTests { index: 0, parentIndex: nil ) - // A multi-panel device lists windows on both panels under one application, and the application's - // frame names one panel. The screen is the lit panel's native box and only the windows proven on - // its display (ADR 0025). - let litPanel = requestSurfacePanel - var rootChildren = context.rootSnapshot.children - if let litPanel { - root.rect = SnapshotRect(CGRect(x: 0, y: 0, width: litPanel.widthPt, height: litPanel.heightPt)) - rootChildren = Self.windowsOnLitDisplay(rootChildren, litDisplayID: litPanel.displayID) { - Self.snapshotDisplayID($0) - } - if rootChildren.isEmpty { - throw RunnerAppScreenCaptureFailure.unresolvedScreen - } + // The application's frame names one panel; the screen is the booked window's box (ADR 0025). + if requestLitPanel != nil, let surface = litPanelState.surface { + root.rect = SnapshotRect(surface.frame) } + let rootChildren = try litPanelRootChildren(context.rootSnapshot) var nodes: [RawAXNode] = [root] if Self.canDescendAtRawDepth(0, hint: hint) { appendCollapsedTabFallbackNodes( @@ -306,6 +297,19 @@ extension RunnerTests { ) } + /// A multi-panel device lists windows on both panels under one application, so a lit-panel + /// snapshot descends only into the windows proven on the lit display, and refuses when none is. + func litPanelRootChildren(_ root: XCUIElementSnapshot) throws -> [XCUIElementSnapshot] { + guard let litPanel = requestLitPanel else { return root.children } + let lit = Self.windowsOnLitDisplay(root.children, litDisplayID: litPanel.displayID) { + Self.snapshotDisplayID($0) + } + if lit.isEmpty { + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } + return lit + } + /// The windows a lit-panel snapshot keeps: only those whose reported display is the lit one. A /// window that reports no display is dropped, because it may be on the dark panel. static func windowsOnLitDisplay( @@ -440,7 +444,8 @@ extension RunnerTests { func walk( _ snapshot: XCUIElementSnapshot, depth: Int, - parentIndex: Int? + parentIndex: Int?, + children: [XCUIElementSnapshot]? = nil ) throws { if let limit = hint.rawTraversalDepth, depth > limit { return } @@ -459,8 +464,7 @@ extension RunnerTests { ) ) - let children = snapshot.children - for child in children { + for child in children ?? snapshot.children { try walk( child, depth: depth + 1, @@ -469,10 +473,12 @@ extension RunnerTests { } } + // Raw keeps the root's reported frame, and still never descends into the dark panel. try walk( context.rootSnapshot, depth: 0, - parentIndex: nil + parentIndex: nil, + children: try litPanelRootChildren(context.rootSnapshot) ) return SnapshotAcquisition( hint: hint, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index f25d1ad2ff..7d96142dad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -162,10 +162,17 @@ extension RunnerTests { preferredBackend: options.preferredBackend ) #endif - // Only the tree tier can prove which panel a window is on, so a multi-panel request runs it alone. - let plan = plan == Self.regularVisiblePlan && requestSurfacePanel != nil ? [.recursiveTree] : plan + // Only the tree tier can prove which panel a window is on, so a multi-panel request runs it + // alone, and a pin to another backend is refused rather than dropped. + var surfacePlan = plan + if target.litPanelSurface != nil { + if let pinned = options.preferredBackend, pinned != SnapshotBackendKind.recursiveTree.rawValue { + throw Self.litPanelBackendPinFailure(pinned) + } + surfacePlan = [.recursiveTree] + } let effective = Self.effectiveSnapshotCapturePlan( - plan, + surfacePlan, xCTestChannelPenalized: xCTestChannelPenalized, xCTestRecoveryProbeDue: xCTestRecoveryProbeDue, availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)), @@ -491,6 +498,14 @@ extension RunnerTests { /// at its own deadline is rejected as a timeout whatever it collected: keeping its payload as the /// fallback is right, accepting it is not, and a node count cannot tell a finished capture from a /// sweep the slice cut short (#2781). + static func litPanelBackendPinFailure(_ pinned: String) -> SnapshotCaptureFailure { + SnapshotCaptureFailure( + code: "UNSUPPORTED_OPERATION", + message: "a multi-panel snapshot runs only the tree backend, so the \(pinned) backend cannot be pinned", + hint: "Snapshot without a backend pin: only the tree backend proves which panel a window is on (ADR 0025)." + ) + } + static func snapshotTierRejectionReason( outcome: SnapshotTierOutcome, kind: SnapshotBackendKind, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift index 0096953434..7a9b5eaaaf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -12,6 +12,8 @@ struct SnapshotCaptureTarget { let app: XCUIApplication let bundleId: String? let processIdentifier: Int? + /// The lit panel's window the request booked; set only on a multi-panel request. + var litPanelSurface: LitPanelSurface? = nil } /// What snapshot command preparation hands the off-main capture. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index 7e9b24e8c8..f6f06d4772 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -18,14 +18,19 @@ extension RunnerTests { case .response(let response): return .response(response) case .context(let context): + if let refusal = self.bookRequestLitPanelSurface(app: context.app) { + return .response(refusal.response) + } + var target = context.observation == nil + ? self.takeSnapshotCaptureTarget(app: context.app) + : SnapshotCaptureTarget( + app: context.app, + bundleId: command.appBundleId?.trimmedNonEmpty, + processIdentifier: Self.processIdentifier(of: context.app) + ) + target.litPanelSurface = self.requestLitPanel == nil ? nil : self.litPanelState.surface return .capture( - context.observation == nil - ? self.takeSnapshotCaptureTarget(app: context.app) - : SnapshotCaptureTarget( - app: context.app, - bundleId: command.appBundleId?.trimmedNonEmpty, - processIdentifier: Self.processIdentifier(of: context.app) - ), + target, systemSurface: context.systemSurface, observation: context.observation ) @@ -99,14 +104,6 @@ extension RunnerTests { ) throws -> Response { let options = Self.presentationOptions(from: command) do { - let litPanelRefusal = try runMainThreadWork( - "snapshot_lit_panel", - timeout: Self.mainThreadExecutionTimeout, - timeoutError: Self.mainThreadExecutionTimeoutError - ) { - self.bookRequestLitPanelSurface(app: target.app) - } - if let litPanelRefusal { throw litPanelRefusal } var payload: DataPayload if options.raw { payload = try snapshotRaw(target: target, options: options) @@ -137,10 +134,7 @@ extension RunnerTests { setNeedsPostSnapshotInteractionDelay() return Response(ok: true, data: payload) } catch let refusal as RunnerAppScreenCaptureFailure { - return Response( - ok: false, - error: ErrorPayload(code: refusal.rawValue, message: refusal.message, hint: refusal.hint) - ) + return refusal.response } catch let failure as SnapshotCaptureFailure { invalidateCachedTargetAfterSnapshotFailure() return Response( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index be057cc408..e7ac87107b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -178,7 +178,8 @@ extension RunnerTests { shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] - for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] { + let inputTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView] + for query in inputTypes.flatMap({ elementQueries(app: app, matching: $0) }) { if shouldStop() { break } let candidates = query.allElementsBoundByIndex #if AGENT_DEVICE_RUNNER_UNIT_TESTS diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index ad556e2519..c2f3fb2470 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -89,13 +89,10 @@ final class RunnerTests: XCTestCase { let app = XCUIApplication() lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId) let mainOwned = RunnerMainOwnedState() + let litPanelState = RunnerLitPanelState() // Set while serving a command that had to re-activate the bound app, and stamped onto that // command's response before it leaves the execution queue (#2682). var pendingTargetActivation: TargetActivationFactPayload? - // Written on the command queue and read on main; see RunnerRequestPanel.swift. - let requestPanelLock = NSLock() - var requestLitPanel: RunnerLitPanel? - var requestLitSurface: LitPanelSurface? // iOS does not reliably expose hasKeyboardFocus for a bare type request, especially when // hardware-keyboard input hides the software keyboard. A successful tap on a concrete text // input is a scoped witness for the immediately-following bare type; lifecycle and non-text diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 03330ab3bc..309b4e7462 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -77,6 +77,11 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( RunnerApplicationScreenFailure *failure ); +// The application's windows that report `litPanel`'s display, in list order. A window that is gone +// or names no display is left out, because it may be on the dark panel. Element reads on a +// multi-panel request search only under these windows. +FOUNDATION_EXPORT NSArray *RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); + // `RunnerResolveApplicationScreen` for a multi-panel device: the window comes from // `RunnerResolveLitPanelWindow` instead of the application's first window. FOUNDATION_EXPORT BOOL RunnerResolveLitPanelScreen( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 38d85dd9c2..4c55078b4c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -224,6 +224,16 @@ static RunnerApplicationScreenFailure RunnerReadLitPanelCandidate( return RunnerReadResolvedWindowScreen(candidate, readFrame, screen, displayID); } +// The application's window list, or nil when the query raises or answers no list. +static NSArray *_Nullable RunnerCopyWindowList(id application) { + __block NSArray *windows = nil; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id elements = [[application valueForKey:@"windows"] valueForKey:@"allElementsBoundByIndex"]; + windows = [elements isKindOfClass:NSArray.class] ? elements : nil; + }]; + return exception == nil ? windows : nil; +} + static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( id application, RunnerLitPanel litPanel, @@ -234,12 +244,8 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( *window = nil; *windowFrame = CGRectNull; *screen = nil; - __block NSArray *windows = nil; - NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ - id elements = [[application valueForKey:@"windows"] valueForKey:@"allElementsBoundByIndex"]; - windows = [elements isKindOfClass:NSArray.class] ? elements : nil; - }]; - if (exception != nil || windows == nil) { + NSArray *windows = RunnerCopyWindowList(application); + if (windows == nil) { return RunnerApplicationScreenFailureUnresolvedWindow; } // A window the walk could not place may be the panel's surface, so it refuses every booking. A @@ -309,6 +315,21 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( return outcome; } +NSArray *RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel) { + NSMutableArray *lit = [NSMutableArray array]; + for (id candidate in RunnerCopyWindowList(application)) { + CGRect frame = CGRectNull; + id screen = nil; + NSUInteger displayID = 0; + if (RunnerReadLitPanelCandidate(candidate, &frame, &screen, &displayID) == + RunnerApplicationScreenFailureNone && + displayID == litPanel.displayID) { + [lit addObject:candidate]; + } + } + return lit; +} + RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( id application, RunnerLitPanel litPanel, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index b667a5ea13..7b9ce2ab6e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -6,12 +6,12 @@ extension RunnerTests { /// must not resolve against the first's panel. func testRequestPanelFollowsTheCommandAndAnUnnamedPanelClearsIt() throws { setRequestLitPanel(from: try command(namingLitPanel: #"{"displayId":3,"widthPt":669,"heightPt":951}"#)) - XCTAssertEqual(requestSurfacePanel?.displayID, 3) - XCTAssertEqual(requestSurfacePanel?.widthPt, 669) - XCTAssertEqual(requestSurfacePanel?.heightPt, 951) + XCTAssertEqual(requestLitPanel?.displayID, 3) + XCTAssertEqual(requestLitPanel?.widthPt, 669) + XCTAssertEqual(requestLitPanel?.heightPt, 951) setRequestLitPanel(from: try command(namingLitPanel: nil)) - XCTAssertNil(requestSurfacePanel) + XCTAssertNil(requestLitPanel) } func testRequestPanelTreatsAPanelNamingNoUsableBoxAsNoPanel() throws { @@ -21,10 +21,37 @@ extension RunnerTests { #"{"displayId":3,"widthPt":669,"heightPt":-951}"#, ] { setRequestLitPanel(from: try command(namingLitPanel: panel)) - XCTAssertNil(requestSurfacePanel, panel) + XCTAssertNil(requestLitPanel, panel) } } + /// Main work a command abandoned runs before the next command's panel is written, so it still + /// reads its own command's panel. + func testRequestLitPanelIsWrittenBehindMainWorkAlreadyQueued() throws { + let unnamed = try command(namingLitPanel: nil) + let named = try command(namingLitPanel: #"{"displayId":3,"widthPt":669,"heightPt":951}"#) + setRequestLitPanel(from: unnamed) + let drained = expectation(description: "main queue drained") + var queuedEarlier: UInt? + var queuedLater: UInt? + DispatchQueue.global().async { + DispatchQueue.main.async { queuedEarlier = self.requestLitPanel?.displayID ?? 0 } + self.setRequestLitPanel(from: named) + DispatchQueue.main.async { + queuedLater = self.requestLitPanel?.displayID ?? 0 + drained.fulfill() + } + } + wait(for: [drained], timeout: 5) + setRequestLitPanel(from: unnamed) + XCTAssertEqual(queuedEarlier, 0) + XCTAssertEqual(queuedLater, 3) + } + + func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() { + XCTAssertEqual(Self.litPanelBackendPinFailure("private-ax").code, "UNSUPPORTED_OPERATION") + } + /// Decoded from JSON, so a renamed wire field fails here. private func command(namingLitPanel panel: String?) throws -> Command { let fields = ["\"command\":\"tap\"", panel.map { "\"litPanel\":\($0)" }].compactMap { $0 } @@ -79,6 +106,35 @@ extension RunnerTests { XCTAssertFalse(fellBack, "no read in the request may fall back to the app") } + /// With a lit panel named, selector and point reads search only the windows on that display; the + /// launched app's windows are all on another one, so neither read may answer. + @MainActor + func testElementReadsSearchOnlyTheLitDisplaysWindows() throws { + var found: (unnamed: Bool, named: Bool, text: String?, namedText: String?) = (false, false, nil, nil) + _ = try responseWithTheAppOffTheLitPanel( + #"{"command":"tap","x":1,"y":1,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + ) { _ in + found.named = findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") + .element != nil + let unnamed = try runnerCommandFixture(#"{"command":"tap"}"#) + setRequestLitPanel(from: unnamed) + let match = findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") + found.unnamed = match.element != nil + if let frame = match.element?.frame { + found.text = readTextAt(app: app, x: frame.midX, y: frame.midY) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + found.namedText = readTextAt(app: app, x: frame.midX, y: frame.midY) + } + return Response(ok: true) + } + XCTAssertTrue(found.unnamed, "the label exists in the app's own window") + XCTAssertNotNil(found.text) + XCTAssertFalse(found.named, "a selector read must not search off the lit display") + XCTAssertNil(found.namedText, "a point read must not search off the lit display") + } + @MainActor func testSnapshotRefusesWhenTheAppIsOffTheLitPanel() throws { let response = try responseWithTheAppOffTheLitPanel( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index f05252dcf0..a19353fe93 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -343,6 +343,22 @@ extension RunnerTests { XCTAssertFalse(absent.resolved) } + /// The one owner of the windows an element read may search: those that report the lit display. + /// A window that is gone or names no display is left out. + func testLitPanelWindowsAreTheWindowsThatReportTheLitDisplay() { + let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let dock = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let windows: [NSObject] = [ + DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1), home, ScreenlessWindowFixture(), + DisplayWindowFixture(frame: Self.innerHome, displayID: 3, present: false), dock, + ] + let lit = RunnerCopyLitPanelWindows( + DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel) + XCTAssertEqual(lit.count, 2) + XCTAssertTrue((lit.first as AnyObject) === home) + XCTAssertTrue((lit.last as AnyObject) === dock) + } + func testLitPanelScreenIsTheScreenTheChosenWindowReported() { let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) var screen: AnyObject? diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index be67f6d5a3..9b07aa65b2 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -715,13 +715,13 @@ Android reports the app window on the next snapshot instead of a panel. SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -Decision: the host names the lit panel in each runner request that resolves a screen. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. A request without the field keeps the historic first-window route unchanged. +Decision: the host names the lit panel in each runner request that resolves a screen. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. -The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window, so they never fall back to the application's own frame. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. +The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Element reads (`readText`, the text-input lookup, and selector queries) search only under the application's windows that report the lit display. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. -A regular snapshot runs only the tree tier, because only that tier can prove which display a window is on. It traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the panel's native box. +A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and keeps the root's reported frame. Rejected alternatives: @@ -733,5 +733,5 @@ Rejected alternatives: Consequences: - `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame, and `SYNTHESIZED_RECORD displayID=` must equal `lit=`. -- A recording keeps the panel that its `recordStart` named. +- A recording keeps the panel that its `recordStart` named. A fold during the recording leaves that panel dark, and later frames are refused. Start a new recording after a fold. - Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index a1ac67f379..cc43689c2b 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -33,13 +33,20 @@ export async function runAppleRunnerCommand( return await runHostBoundAppleRunnerCommand(device, command, options); } const { resolveRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); + const startedAt = Date.now(); const litPanel = await resolveRunnerRequestLitPanel(device, { signal: resolveRunnerRequestSignal(options), requestTimeoutMs: command.timeoutMs, }); + if (!litPanel) return await runHostBoundAppleRunnerCommand(device, command, options); + // The runner gets what the panel read left of the request's budget. + const timeoutMs = + command.timeoutMs === undefined + ? undefined + : Math.max(1, command.timeoutMs - (Date.now() - startedAt)); return await runHostBoundAppleRunnerCommand( device, - litPanel ? { ...command, litPanel } : command, + timeoutMs === undefined ? { ...command, litPanel } : { ...command, litPanel, timeoutMs }, options, ); } diff --git a/packages/platform-apple/src/foldable/orientation.test.ts b/packages/platform-apple/src/foldable/orientation.test.ts index 725ef43db5..85420a7236 100644 --- a/packages/platform-apple/src/foldable/orientation.test.ts +++ b/packages/platform-apple/src/foldable/orientation.test.ts @@ -217,7 +217,7 @@ test('a rotation is not routed by a pending panel-table retry', async () => { throw new Error(`unexpected tool call: ${args.join(' ')}`); }, }); - await withAppleToolProvider(failing, () => readFoldablePanelTable(device, {})); + await withAppleToolProvider(failing, () => readFoldablePanelTable(device, { timeoutMs: 5_000 })); expect(tableFailures).toBe(1); await expect(sim.rotate('landscape-left')).resolves.toEqual({ orientation: 'landscape-left' }); diff --git a/packages/platform-apple/src/foldable/orientation.ts b/packages/platform-apple/src/foldable/orientation.ts index e3b58bcb25..a67802bab1 100644 --- a/packages/platform-apple/src/foldable/orientation.ts +++ b/packages/platform-apple/src/foldable/orientation.ts @@ -5,6 +5,7 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { createTtlMemo } from '@agent-device/kernel/ttl-memo'; import { + IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_READ_TIMEOUT_MS, IOS_FOLDABLE_ORIENTATION_SETTLE_ATTEMPTS, } from '../core/config.ts'; @@ -29,8 +30,8 @@ const ORIENTATION_UNVERIFIED_HINT = /** Resolved panel counts are hardware facts, so each simulator is probed once per daemon. */ const multiPanelByDevice = new Map(); -// Every surface command reads the panel table, so one that would not answer is not asked again -// for a few seconds: a failing probe can cost its whole timeout. +// A device whose panel count was never read routes as single-panel. A full-budget read that failed +// for it is not repeated for a few seconds, because every surface command would pay it again. const unansweredPanelTables = createTtlMemo({ ttlMs: 5_000 }); // Reads overlap, so a failure may finish after a read that started later and already resolved. // Each read takes a sequence number, and a failure never overrides a newer resolved read. @@ -66,27 +67,38 @@ export async function isAppleFoldableSimulator( return inventory.multiScreen; } +export type FoldablePanelTableRead = + | { kind: 'multi-panel'; inventory: AppleDisplayInventory } + /** A device the `fold` fact refuses, one with one panel, or one whose panel count is unknown. */ + | { kind: 'single-panel' } + /** A device known to have more than one panel whose table did not answer this time. */ + | { kind: 'unreadable' }; + /** - * One panel-table read of a multi-panel simulator, or undefined for a device the `fold` fact refuses, - * one known to have a single panel, or a table that would not answer. It records the panel count it - * reads, so a single-panel simulator pays one read per daemon. + * One panel-table read for a surface request. It records the panel count it reads, so a + * single-panel simulator pays one read per daemon. */ export async function readFoldablePanelTable( device: DeviceInfo, - options: { signal?: AbortSignal; timeoutMs?: number }, -): Promise { - if (!appleFoldableFacts(device).setFoldPose.available) return undefined; - if (multiPanelByDevice.get(device.id) === false) return undefined; - if (unansweredPanelTables.get(device.id)) return undefined; + options: { signal?: AbortSignal; timeoutMs: number }, +): Promise { + if (!appleFoldableFacts(device).setFoldPose.available) return { kind: 'single-panel' }; + const known = multiPanelByDevice.get(device.id); + if (known === false) return { kind: 'single-panel' }; + if (known === undefined && unansweredPanelTables.get(device.id)) return { kind: 'single-panel' }; const read = ++panelTableReads; const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) { + if (known) return { kind: 'unreadable' }; const newerReadResolved = (latestResolvedRead.get(device.id) ?? 0) > read; - if (!options.signal?.aborted && !newerReadResolved) unansweredPanelTables.set(device.id, true); - return undefined; + const fullBudget = options.timeoutMs >= IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS; + if (fullBudget && !options.signal?.aborted && !newerReadResolved) { + unansweredPanelTables.set(device.id, true); + } + return { kind: 'single-panel' }; } recordPanelTable(device, inventory, read); - return inventory.multiScreen ? inventory : undefined; + return inventory.multiScreen ? { kind: 'multi-panel', inventory } : { kind: 'single-panel' }; } /** Rotates a foldable simulator through guest HID and verifies the rotation through CoreDevice. */ diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index 580e7a103b..e0465e98e8 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -3,6 +3,7 @@ import { beforeEach, test, vi } from 'vitest'; import { bindTestCommandHost } from '@agent-device/test-kit'; import { withAppleRunnerProvider } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { AppError } from '@agent-device/kernel/errors'; vi.mock('../foldable/fold-helper-cache.ts', () => ({ ensureFoldHelperBinary: vi.fn(async () => ({ path: '/cache/fold-helper' })), @@ -156,7 +157,10 @@ test('the probe takes the request timeout when it is the shorter one', async () }); test('a surface command probes the panel table within its own timeout', async () => { - answering(IPHONE_DUO_CLOSED); + mockPanelTable.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { ok: true as const, payload: IPHONE_DUO_CLOSED }; + }); const device = simulator({ name: 'iPhone Duo' }); const calls: RecordedRunnerCall[] = []; await withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => @@ -164,6 +168,30 @@ test('a surface command probes the panel table within its own timeout', async () ); assert.equal(mockPanelTable.mock.lastCall?.[0].timeoutMs, 1_200); assert.equal(calls[0]?.command.litPanel?.displayId, 1); + const sentTimeoutMs = calls[0]?.command.timeoutMs ?? Number.NaN; + assert.ok(sentTimeoutMs > 0 && sentTimeoutMs < 1_200, 'the runner gets what the read left'); +}); + +test('a known multi-panel device refuses a surface command when its panel table goes unread', async () => { + const device = simulator({ name: 'iPhone Duo' }); + const calls: RecordedRunnerCall[] = []; + const run = () => + withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20 }), + ); + answering(IPHONE_DUO_CLOSED); + await run(); + assert.equal(calls[0]?.command.litPanel?.displayId, 1); + + mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); + await assert.rejects(run(), (error: AppError) => { + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'panel-table-unreadable'); + assert.equal(error.details?.runnerErrorCode, 'APP_SCREEN_UNRESOLVED'); + assert.equal(error.details?.dispatched, 'no'); + return true; + }); + assert.equal(calls.length, 1, 'the refused command never reached the runner'); }); test('a slow failed read does not back off over a newer read that resolved', async () => { diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 48cf0a8ce8..5bdd06ab25 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -1,25 +1,42 @@ import type { RunnerCommand } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; import { IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS } from '../core/config.ts'; import { appleLitPanel } from '../core/display-inventory.ts'; import { readFoldablePanelTable } from './orientation.ts'; +const PANEL_TABLE_UNREADABLE_HINT = + 'The panel table is unreadable, so no lit panel can be named and nothing was sent. Retry; if it persists, run xcrun devicectl device info displays --device .'; + /** * The lit panel a runner request names, read fresh per request because a fold changes it (ADR - * 0025). The read never outlasts the request's own `timeoutMs`. Undefined keeps the runner's - * historic first-window route. + * 0025). The read never outlasts the request's `timeoutMs`. Undefined keeps the runner's historic + * first-window route, which only a device not known to have more than one panel may take. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo, options: { signal?: AbortSignal; requestTimeoutMs?: number }, ): Promise { - const inventory = await readFoldablePanelTable(device, { + const read = await readFoldablePanelTable(device, { signal: options.signal, timeoutMs: Math.min( options.requestTimeoutMs ?? IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, ), }); - return inventory ? appleLitPanel(inventory) : undefined; + if (read.kind === 'unreadable') { + throw new AppError( + 'COMMAND_FAILED', + `The panel table of ${device.name} did not answer, so the lit panel cannot be named`, + { + deviceId: device.id, + reason: 'panel-table-unreadable', + runnerErrorCode: 'APP_SCREEN_UNRESOLVED', + dispatched: 'no', + hint: PANEL_TABLE_UNREADABLE_HINT, + }, + ); + } + return read.kind === 'multi-panel' ? appleLitPanel(read.inventory) : undefined; } From 11565141335ae6dd8945f24bdfa40803eeec07ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 1 Nov 2026 20:00:22 +0100 Subject: [PATCH 08/29] fix(ios): keep raw, unreadable-list and pin refusals on the lit panel - `--raw` publishes the booked window's frame as its root, so raw and regular coordinates share one space. - RunnerCopyLitPanelWindows returns nil for an unreadable window list. A lit-panel element read then refuses with APP_SCREEN_UNRESOLVED instead of finding nothing. - A backend pin on a lit-panel snapshot is refused in command preparation, so it leaves the cached target alone. - ADR 0025 says what a recording does after a fold: it keeps capturing the panel recordStart named. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 21 ++++++++++--- .../RunnerTests+CommandExecution.swift | 6 ++++ .../RunnerTests+Snapshot.swift | 17 +++++++---- .../RunnerTests+SnapshotCapturePlan.swift | 18 ++--------- .../RunnerTests+SnapshotExecution.swift | 22 ++++++++++++++ .../RunnerXCTestEventBridge.h | 5 ++-- .../RunnerXCTestEventBridge.m | 6 +++- .../UnitTests/RunnerRequestPanelTests.swift | 30 ++++++++++++++++++- .../RunnerTests+XCTestEventBridgeTests.swift | 11 +++++-- docs/adr/0025-foldable-apple-panels.md | 4 +-- 10 files changed, 106 insertions(+), 34 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 9454401a9b..e24fb10b48 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -14,7 +14,8 @@ struct LitPanelSurface { final class RunnerLitPanelState { var panel: RunnerLitPanel? var surface: LitPanelSurface? - var windows: (app: XCUIApplication, elements: [XCUIElement])? + /// Nil elements: the window list could not be read. + var windows: (app: XCUIApplication, elements: [XCUIElement]?)? } extension RunnerTests { @@ -58,11 +59,23 @@ extension RunnerTests { } /// Where an element read searches: the application, or on a lit-panel request only the windows - /// proven on the lit display, so a dark-panel element can never answer. + /// proven on the lit display, so a dark-panel element can never answer. An unreadable window list + /// searches nothing; `litPanelWindowsRefusal` says so before the read. func elementSearchRoots(app: XCUIApplication) -> [XCUIElement] { - guard let litPanel = requestLitPanel else { return [app] } + guard requestLitPanel != nil else { return [app] } + return litPanelWindows(app: app) ?? [] + } + + /// Why an element read on a lit-panel request cannot search, or nil when it can. + func litPanelWindowsRefusal(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { + guard requestLitPanel != nil, litPanelWindows(app: app) == nil else { return nil } + return .unresolvedScreen + } + + private func litPanelWindows(app: XCUIApplication) -> [XCUIElement]? { + guard let litPanel = requestLitPanel else { return nil } if let cached = litPanelState.windows, cached.app === app { return cached.elements } - let windows = RunnerCopyLitPanelWindows(app, litPanel).compactMap { $0 as? XCUIElement } + let windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap { $0 as? XCUIElement } litPanelState.windows = (app, windows) return windows } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 03e78fd8ed..f30a920178 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -18,6 +18,12 @@ extension RunnerTests { if command.command != .screenshot, let refusal = bookRequestLitPanelSurface(app: activeApp) { return refusal.response } + // An element read whose lit-panel window list cannot be read refuses instead of finding nothing. + if command.command == .readText || command.selectorKey != nil, + let refusal = litPanelWindowsRefusal(app: activeApp) + { + return refusal.response + } switch command.command { case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime, .appState, .pasteboardWrite, .snapshot: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 3fe1721c82..4487fa6a5b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -211,10 +211,7 @@ extension RunnerTests { index: 0, parentIndex: nil ) - // The application's frame names one panel; the screen is the booked window's box (ADR 0025). - if requestLitPanel != nil, let surface = litPanelState.surface { - root.rect = SnapshotRect(surface.frame) - } + publishLitPanelRoot(&root) let rootChildren = try litPanelRootChildren(context.rootSnapshot) var nodes: [RawAXNode] = [root] if Self.canDescendAtRawDepth(0, hint: hint) { @@ -297,6 +294,14 @@ extension RunnerTests { ) } + /// The application's frame names one panel, so a lit-panel snapshot's root, raw or regular, is the + /// booked window's box (ADR 0025). + func publishLitPanelRoot(_ root: inout RawAXNode) { + if requestLitPanel != nil, let surface = litPanelState.surface { + root.rect = SnapshotRect(surface.frame) + } + } + /// A multi-panel device lists windows on both panels under one application, so a lit-panel /// snapshot descends only into the windows proven on the lit display, and refuses when none is. func litPanelRootChildren(_ root: XCUIElementSnapshot) throws -> [XCUIElementSnapshot] { @@ -473,13 +478,15 @@ extension RunnerTests { } } - // Raw keeps the root's reported frame, and still never descends into the dark panel. try walk( context.rootSnapshot, depth: 0, parentIndex: nil, children: try litPanelRootChildren(context.rootSnapshot) ) + if !nodes.isEmpty { + publishLitPanelRoot(&nodes[0]) + } return SnapshotAcquisition( hint: hint, nodes: nodes, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 7d96142dad..ad8ac82a84 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -163,14 +163,8 @@ extension RunnerTests { ) #endif // Only the tree tier can prove which panel a window is on, so a multi-panel request runs it - // alone, and a pin to another backend is refused rather than dropped. - var surfacePlan = plan - if target.litPanelSurface != nil { - if let pinned = options.preferredBackend, pinned != SnapshotBackendKind.recursiveTree.rawValue { - throw Self.litPanelBackendPinFailure(pinned) - } - surfacePlan = [.recursiveTree] - } + // alone; preparation already refused a pin to another backend. + let surfacePlan = target.litPanelSurface == nil ? plan : [.recursiveTree] let effective = Self.effectiveSnapshotCapturePlan( surfacePlan, xCTestChannelPenalized: xCTestChannelPenalized, @@ -498,14 +492,6 @@ extension RunnerTests { /// at its own deadline is rejected as a timeout whatever it collected: keeping its payload as the /// fallback is right, accepting it is not, and a node count cannot tell a finished capture from a /// sweep the slice cut short (#2781). - static func litPanelBackendPinFailure(_ pinned: String) -> SnapshotCaptureFailure { - SnapshotCaptureFailure( - code: "UNSUPPORTED_OPERATION", - message: "a multi-panel snapshot runs only the tree backend, so the \(pinned) backend cannot be pinned", - hint: "Snapshot without a backend pin: only the tree backend proves which panel a window is on (ADR 0025)." - ) - } - static func snapshotTierRejectionReason( outcome: SnapshotTierOutcome, kind: SnapshotBackendKind, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index f6f06d4772..87ca3e4953 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -21,6 +21,11 @@ extension RunnerTests { if let refusal = self.bookRequestLitPanelSurface(app: context.app) { return .response(refusal.response) } + if self.requestLitPanel != nil, + let refusal = Self.litPanelBackendPinRefusal(Self.presentationOptions(from: command).preferredBackend) + { + return .response(refusal) + } var target = context.observation == nil ? self.takeSnapshotCaptureTarget(app: context.app) : SnapshotCaptureTarget( @@ -166,6 +171,23 @@ extension RunnerTests { } } + /// A multi-panel snapshot runs only the tree backend, the one that proves which panel a window is + /// on, so a pin to another backend is refused as a request. It never reaches the capture plan and + /// leaves the cached target alone. + static func litPanelBackendPinRefusal(_ preferredBackend: String?) -> Response? { + guard let pinned = preferredBackend, pinned != SnapshotBackendKind.recursiveTree.rawValue else { + return nil + } + return Response( + ok: false, + error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "a multi-panel snapshot runs only the tree backend, so the \(pinned) backend cannot be pinned", + hint: "Snapshot without a backend pin: only the tree backend proves which panel a window is on (ADR 0025)." + ) + ) + } + func invalidateCachedTargetAfterSnapshotFailure() { applyMainOwnedSnapshotState("target_invalidation") { self.invalidateCachedTarget(reason: "ax_snapshot_failure") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 309b4e7462..2382c926d8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -79,8 +79,9 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( // The application's windows that report `litPanel`'s display, in list order. A window that is gone // or names no display is left out, because it may be on the dark panel. Element reads on a -// multi-panel request search only under these windows. -FOUNDATION_EXPORT NSArray *RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); +// multi-panel request search only under these windows. Returns nil when the window list itself +// cannot be read, which is not the same answer as no window on the lit display. +FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); // `RunnerResolveApplicationScreen` for a multi-panel device: the window comes from // `RunnerResolveLitPanelWindow` instead of the application's first window. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 4c55078b4c..380ea089ce 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -316,8 +316,12 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( } NSArray *RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel) { + NSArray *windows = RunnerCopyWindowList(application); + if (windows == nil) { + return nil; + } NSMutableArray *lit = [NSMutableArray array]; - for (id candidate in RunnerCopyWindowList(application)) { + for (id candidate in windows) { CGRect frame = CGRectNull; id screen = nil; NSUInteger displayID = 0; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 7b9ce2ab6e..125bf1e438 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -49,7 +49,10 @@ extension RunnerTests { } func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() { - XCTAssertEqual(Self.litPanelBackendPinFailure("private-ax").code, "UNSUPPORTED_OPERATION") + XCTAssertEqual(Self.litPanelBackendPinRefusal("private-ax")?.error?.code, "UNSUPPORTED_OPERATION") + XCTAssertEqual(Self.litPanelBackendPinRefusal("queries")?.error?.code, "UNSUPPORTED_OPERATION") + XCTAssertNil(Self.litPanelBackendPinRefusal("tree")) + XCTAssertNil(Self.litPanelBackendPinRefusal(nil)) } /// Decoded from JSON, so a renamed wire field fails here. @@ -135,6 +138,31 @@ extension RunnerTests { XCTAssertNil(found.namedText, "a point read must not search off the lit display") } + /// A refused pin is a request refusal: it never reaches the capture plan, so it does not + /// invalidate the cached target the next command relies on. + @MainActor + func testLitPanelSnapshotPinRefusalKeepsTheCachedTarget() throws { + app.launch() + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + defer { + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let window = app.windows.firstMatch + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + let size = window.frame.size + let command = try runnerCommandFixture( + #"{"appBundleId":"com.callstack.agentdevice.runner","command":"snapshot","preferredBackend":"private-ax","litPanel":{"displayId":\#(displayID),"widthPt":\#(size.width),"heightPt":\#(size.height)}}"# + ) + setRequestLitPanel(from: command) + let response = try executeSnapshotDispatched(command: command) + XCTAssertEqual(response.error?.code, "UNSUPPORTED_OPERATION") + XCTAssertTrue(mainOwned.app === app, "a refused pin must not invalidate the cached target") + } + @MainActor func testSnapshotRefusesWhenTheAppIsOffTheLitPanel() throws { let response = try responseWithTheAppOffTheLitPanel( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index a19353fe93..d27b33d160 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -354,9 +354,14 @@ extension RunnerTests { ] let lit = RunnerCopyLitPanelWindows( DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel) - XCTAssertEqual(lit.count, 2) - XCTAssertTrue((lit.first as AnyObject) === home) - XCTAssertTrue((lit.last as AnyObject) === dock) + XCTAssertEqual(lit?.count, 2) + XCTAssertTrue((lit?.first as AnyObject) === home) + XCTAssertTrue((lit?.last as AnyObject) === dock) + XCTAssertNil( + RunnerCopyLitPanelWindows( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel), + "an unreadable window list is not an empty lit display" + ) } func testLitPanelScreenIsTheScreenTheChosenWindowReported() { diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 9b07aa65b2..0362a2199d 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -721,7 +721,7 @@ With a named panel, the runner walks at most eight windows. It books the first w The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Element reads (`readText`, the text-input lookup, and selector queries) search only under the application's windows that report the lit display. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. -A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and keeps the root's reported frame. +A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. Element reads whose window list cannot be read refuse with `APP_SCREEN_UNRESOLVED` instead of finding nothing. Rejected alternatives: @@ -733,5 +733,5 @@ Rejected alternatives: Consequences: - `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame, and `SYNTHESIZED_RECORD displayID=` must equal `lit=`. -- A recording keeps the panel that its `recordStart` named. A fold during the recording leaves that panel dark, and later frames are refused. Start a new recording after a fold. +- A recording keeps the panel that its `recordStart` named. The runner cannot read panel power, so it does not notice a fold during the recording: SpringBoard keeps windows on the panel that went dark, and later frames show that dark panel. Start a new recording after a fold. - Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. From bc78936a42df2559130f98775f2a26dde2642449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 19:24:27 +0100 Subject: [PATCH 09/29] fix(ios): let element search own the lit-panel refusal Every element read on a lit-panel request searches the windows proven on the lit display, or `elementSearchRoots` throws APP_SCREEN_UNRESOLVED, which `executeOnMainPrepared` returns as the reply. This replaces the command-name precheck, so a coordinate `type` whose window list cannot be read refuses instead of tapping a point with no target. The snapshot path reads the booked window only from its capture target. `findText` and `querySelector` join the lit-panel-scoped commands, so the traits table is the one list. A `customActions` snapshot names `customActions` in its refusal. The client no longer short-circuits on a `litPanel` field no caller sets. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 40 +++++++------ .../RunnerTests+CommandExecution.swift | 43 +++++++++----- .../RunnerTests+Interaction.swift | 18 +++--- .../RunnerTests+Snapshot.swift | 25 ++++---- .../RunnerTests+SnapshotAcquisition.swift | 6 +- .../RunnerTests+SnapshotCapturePlan.swift | 3 +- .../RunnerTests+SnapshotExecution.swift | 24 +++++--- .../RunnerTests+SynthesizedTextEntry.swift | 4 +- .../RunnerTests+TextEntry.swift | 11 ++-- .../RunnerTests+TextEntryFocus.swift | 4 +- .../RunnerTests+TextInputProbe.swift | 36 +++++++----- .../RunnerTests+TextTyping.swift | 4 +- .../RunnerTests+TypeExecution.swift | 8 +-- .../RunnerTests.swift | 3 + .../UnitTests/RunnerRequestPanelTests.swift | 57 ++++++++++++++++--- ...RunnerTests+CoordinateTextEntryTests.swift | 6 +- ...unnerTests+SynthesizedTextEntryTests.swift | 12 ++-- ...nnerTests+TextEntryConfirmationTests.swift | 6 +- .../RunnerTests+TextEntryLatencyTests.swift | 10 ++-- .../RunnerTests+TextEntryPolicyTests.swift | 10 ++-- .../RunnerTests+TextInputProbeTests.swift | 8 +-- .../RunnerTests+TextTypingTests.swift | 46 +++++++-------- contracts/fixtures/runner-requests.json | 1 + docs/adr/0025-foldable-apple-panels.md | 6 +- .../apple-runner/src/runner-command-traits.ts | 7 ++- ...nteractor-foldable-runner-requests.test.ts | 5 ++ .../platform-apple/src/core/runner-client.ts | 2 +- 27 files changed, 244 insertions(+), 161 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index e24fb10b48..cde16c9881 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -6,6 +6,7 @@ import XCTest // its command, so `recordStart` reads the panel once and carries it. struct LitPanelSurface { let app: XCUIApplication + let displayID: UInt let window: XCUIElement let frame: CGRect } @@ -14,8 +15,7 @@ struct LitPanelSurface { final class RunnerLitPanelState { var panel: RunnerLitPanel? var surface: LitPanelSurface? - /// Nil elements: the window list could not be read. - var windows: (app: XCUIApplication, elements: [XCUIElement]?)? + var windows: (app: XCUIApplication, elements: [XCUIElement])? } extension RunnerTests { @@ -54,33 +54,31 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - litPanelState.surface = LitPanelSurface(app: app, window: element, frame: frame) + litPanelState.surface = LitPanelSurface(app: app, displayID: litPanel.displayID, window: element, frame: frame) return nil } /// Where an element read searches: the application, or on a lit-panel request only the windows - /// proven on the lit display, so a dark-panel element can never answer. An unreadable window list - /// searches nothing; `litPanelWindowsRefusal` says so before the read. - func elementSearchRoots(app: XCUIApplication) -> [XCUIElement] { - guard requestLitPanel != nil else { return [app] } - return litPanelWindows(app: app) ?? [] - } - - /// Why an element read on a lit-panel request cannot search, or nil when it can. - func litPanelWindowsRefusal(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { - guard requestLitPanel != nil, litPanelWindows(app: app) == nil else { return nil } - return .unresolvedScreen - } - - private func litPanelWindows(app: XCUIApplication) -> [XCUIElement]? { - guard let litPanel = requestLitPanel else { return nil } + /// proven on the lit display, so a dark-panel element can never answer. A lit-panel window list + /// that cannot be read throws `APP_SCREEN_UNRESOLVED`, so no read can mistake it for no match. + func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] { + guard let litPanel = requestLitPanel else { return [app] } if let cached = litPanelState.windows, cached.app === app { return cached.elements } - let windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap { $0 as? XCUIElement } +#if AGENT_DEVICE_RUNNER_UNIT_TESTS + let unanswerable = litPanelWindowListUnanswerableForTesting +#else + let unanswerable = false +#endif + guard !unanswerable, + let windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap({ $0 as? XCUIElement }) + else { + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } litPanelState.windows = (app, windows) return windows } - func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) -> [XCUIElementQuery] { - elementSearchRoots(app: app).map { $0.descendants(matching: type) } + func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) throws -> [XCUIElementQuery] { + try elementSearchRoots(app: app).map { $0.descendants(matching: type) } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index f30a920178..d94291f2f9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -2,12 +2,33 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { + /// Runs `command` against its prepared app. An element read that cannot search the lit panel + /// throws its refusal, which answers the request (ADR 0025). @MainActor func executeOnMainPrepared( command: Command, activeApp: XCUIApplication, activeAppBundleId: String? = nil, alertDeadline: Date? = nil + ) throws -> Response { + do { + return try executePrepared( + command: command, + activeApp: activeApp, + activeAppBundleId: activeAppBundleId, + alertDeadline: alertDeadline + ) + } catch let refusal as RunnerAppScreenCaptureFailure { + return refusal.response + } + } + + @MainActor + private func executePrepared( + command: Command, + activeApp: XCUIApplication, + activeAppBundleId: String?, + alertDeadline: Date? ) throws -> Response { var activeApp = activeApp mainOwned.soleDisplayPortraitSize = nil @@ -18,12 +39,6 @@ extension RunnerTests { if command.command != .screenshot, let refusal = bookRequestLitPanelSurface(app: activeApp) { return refusal.response } - // An element read whose lit-panel window list cannot be read refuses instead of finding nothing. - if command.command == .readText || command.selectorKey != nil, - let refusal = litPanelWindowsRefusal(app: activeApp) - { - return refusal.response - } switch command.command { case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime, .appState, .pasteboardWrite, .snapshot: @@ -45,7 +60,7 @@ extension RunnerTests { } else { expectedPoint = nil } - let match = findElement( + let match = try findElement( app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue, @@ -163,7 +178,7 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y) + textInput = try coordinateTapTextInputAt(app: activeApp, x: x, y: y) } else { textInput = nil // Clear before dispatch so a throwing gesture cannot retain the preceding field's witness. @@ -416,11 +431,11 @@ extension RunnerTests { } return Response(ok: true, data: DataPayload(message: "remote pressed")) case .type: - var response: Response? + var outcome: Result? withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { - response = executeTypeCommand(activeApp: activeApp, command: command) + outcome = Result { try executeTypeCommand(activeApp: activeApp, command: command) } } - return response ?? Response(ok: false, error: ErrorPayload(message: "type produced no response")) + return try outcome?.get() ?? Response(ok: false, error: ErrorPayload(message: "type produced no response")) case .swipe: guard let direction = command.direction else { return Response(ok: false, error: ErrorPayload(message: "swipe requires direction")) @@ -441,18 +456,18 @@ extension RunnerTests { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "findText requires text")) } - let found = findElement(app: activeApp, text: text) != nil + let found = try findElement(app: activeApp, text: text) != nil return Response(ok: true, data: DataPayload(found: found)) case .querySelector: guard let selectorKey = command.selectorKey, let selectorValue = command.selectorValue else { return Response(ok: false, error: ErrorPayload(message: "querySelector requires selectorKey and selectorValue")) } - return queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + return try queryElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) case .readText: guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "readText requires x and y")) } - guard let text = readTextAt(app: activeApp, x: x, y: y) else { + guard let text = try readTextAt(app: activeApp, x: x, y: y) else { return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text")) } return Response(ok: true, data: DataPayload(text: text)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 243ce1c208..8d5c4bdd2e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -166,9 +166,9 @@ extension RunnerTests { #endif } - func findElement(app: XCUIApplication, text: String) -> XCUIElement? { + func findElement(app: XCUIApplication, text: String) throws -> XCUIElement? { let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text) - return elementQueries(app: app, matching: .any) + return try elementQueries(app: app, matching: .any) .lazy .map { $0.matching(predicate).firstMatch } .first { $0.exists } @@ -181,7 +181,7 @@ extension RunnerTests { allowNonHittableFallback: Bool = false, expectedPoint: CGPoint? = nil, rawMatchPolicy: DirectSelectorRawMatchPolicy = .rejectDistinctMatches - ) -> SelectorElementMatch { + ) throws -> SelectorElementMatch { let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty else { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) @@ -200,7 +200,7 @@ extension RunnerTests { return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } - let matches = elementQueries(app: app, matching: .any) + let matches = try elementQueries(app: app, matching: .any) .flatMap { $0.matching(predicate).allElementsBoundByIndex } .filter(\.exists) let facts = matches.map { element in @@ -292,13 +292,13 @@ extension RunnerTests { resolveRunnerWindow(app: app).frame } - func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { + func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response { // querySelector is a read — it backs get/is/wait and the offscreen-refusal // double-check, none of which mutate. The fail-closed raw-match rule exists // to stop a mutation acting on an unseen duplicate; applying it here would // instead turn a decorative non-hittable duplicate into an AMBIGUOUS_MATCH // for readers that previously resolved the hittable element. - let match = findElement( + let match = try findElement( app: app, selectorKey: selectorKey, selectorValue: selectorValue, @@ -360,9 +360,9 @@ extension RunnerTests { return left.elementType.rawValue < right.elementType.rawValue } - func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { + func readTextAt(app: XCUIApplication, x: Double, y: Double) throws -> String? { let point = CGPoint(x: x, y: y) - let textInputCandidates = textInputCandidatesAt(app: app, point: point, contentTranslation: liveContentTranslation(app: app)) + let textInputCandidates = try textInputCandidatesAt(app: app, point: point, contentTranslation: liveContentTranslation(app: app)) .map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { @@ -370,7 +370,7 @@ extension RunnerTests { } } - let candidates = elementQueries(app: app, matching: .any) + let candidates = try elementQueries(app: app, matching: .any) .flatMap { $0.allElementsBoundByIndex } .filter { element in element.exists && !element.frame.isEmpty && element.frame.contains(point) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 4487fa6a5b..3f8867aa22 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -31,6 +31,8 @@ extension RunnerTests { * (#2660). Nil only where the platform has no iOS keyboard to measure. */ let keyboardBand: RunnerKeyboardBandFact? + /// The lit panel's window the request booked; set only on a multi-panel request (ADR 0025). + var litPanelSurface: LitPanelSurface? = nil } struct SnapshotEvaluation { @@ -211,8 +213,8 @@ extension RunnerTests { index: 0, parentIndex: nil ) - publishLitPanelRoot(&root) - let rootChildren = try litPanelRootChildren(context.rootSnapshot) + Self.publishLitPanelRoot(&root, surface: context.litPanelSurface) + let rootChildren = try Self.litPanelRootChildren(context.rootSnapshot, surface: context.litPanelSurface) var nodes: [RawAXNode] = [root] if Self.canDescendAtRawDepth(0, hint: hint) { appendCollapsedTabFallbackNodes( @@ -296,19 +298,20 @@ extension RunnerTests { /// The application's frame names one panel, so a lit-panel snapshot's root, raw or regular, is the /// booked window's box (ADR 0025). - func publishLitPanelRoot(_ root: inout RawAXNode) { - if requestLitPanel != nil, let surface = litPanelState.surface { + static func publishLitPanelRoot(_ root: inout RawAXNode, surface: LitPanelSurface?) { + if let surface { root.rect = SnapshotRect(surface.frame) } } /// A multi-panel device lists windows on both panels under one application, so a lit-panel /// snapshot descends only into the windows proven on the lit display, and refuses when none is. - func litPanelRootChildren(_ root: XCUIElementSnapshot) throws -> [XCUIElementSnapshot] { - guard let litPanel = requestLitPanel else { return root.children } - let lit = Self.windowsOnLitDisplay(root.children, litDisplayID: litPanel.displayID) { - Self.snapshotDisplayID($0) - } + static func litPanelRootChildren( + _ root: XCUIElementSnapshot, + surface: LitPanelSurface? + ) throws -> [XCUIElementSnapshot] { + guard let surface else { return root.children } + let lit = windowsOnLitDisplay(root.children, litDisplayID: surface.displayID) { snapshotDisplayID($0) } if lit.isEmpty { throw RunnerAppScreenCaptureFailure.unresolvedScreen } @@ -482,10 +485,10 @@ extension RunnerTests { context.rootSnapshot, depth: 0, parentIndex: nil, - children: try litPanelRootChildren(context.rootSnapshot) + children: try Self.litPanelRootChildren(context.rootSnapshot, surface: context.litPanelSurface) ) if !nodes.isEmpty { - publishLitPanelRoot(&nodes[0]) + Self.publishLitPanelRoot(&nodes[0], surface: context.litPanelSurface) } return SnapshotAcquisition( hint: hint, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 6f31243dea..509bc23a45 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -36,7 +36,8 @@ extension RunnerTests { app: XCUIApplication, hint: CaptureHint, captureDeadline: Date = .distantFuture, - treeCaptureSliceBudgetOverride: TimeInterval? = nil + treeCaptureSliceBudgetOverride: TimeInterval? = nil, + litPanelSurface: LitPanelSurface? = nil ) throws -> SnapshotTraversalContext? { let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) @@ -64,7 +65,8 @@ extension RunnerTests { rootSnapshot: rootSnapshot, rootReadDuration: rootReadDuration, viewport: viewport, - keyboardBand: keyboardBand + keyboardBand: keyboardBand, + litPanelSurface: litPanelSurface ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index ad8ac82a84..f2325fa606 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -347,7 +347,8 @@ extension RunnerTests { app: app, hint: hint, captureDeadline: deadline, - treeCaptureSliceBudgetOverride: step.treeCaptureSliceBudgetOverride + treeCaptureSliceBudgetOverride: step.treeCaptureSliceBudgetOverride, + litPanelSurface: target.litPanelSurface ) else { return (nil, .completed) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index 87ca3e4953..6874f3dacb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -21,9 +21,7 @@ extension RunnerTests { if let refusal = self.bookRequestLitPanelSurface(app: context.app) { return .response(refusal.response) } - if self.requestLitPanel != nil, - let refusal = Self.litPanelBackendPinRefusal(Self.presentationOptions(from: command).preferredBackend) - { + if self.requestLitPanel != nil, let refusal = Self.litPanelBackendPinRefusal(command) { return .response(refusal) } var target = context.observation == nil @@ -33,7 +31,7 @@ extension RunnerTests { bundleId: command.appBundleId?.trimmedNonEmpty, processIdentifier: Self.processIdentifier(of: context.app) ) - target.litPanelSurface = self.requestLitPanel == nil ? nil : self.litPanelState.surface + target.litPanelSurface = self.litPanelState.surface return .capture( target, systemSurface: context.systemSurface, @@ -173,17 +171,25 @@ extension RunnerTests { /// A multi-panel snapshot runs only the tree backend, the one that proves which panel a window is /// on, so a pin to another backend is refused as a request. It never reaches the capture plan and - /// leaves the cached target alone. - static func litPanelBackendPinRefusal(_ preferredBackend: String?) -> Response? { - guard let pinned = preferredBackend, pinned != SnapshotBackendKind.recursiveTree.rawValue else { + /// leaves the cached target alone. `customActions` without a pin implies the private-ax pin, so + /// the refusal names it. + static func litPanelBackendPinRefusal(_ command: Command) -> Response? { + guard let pinned = presentationOptions(from: command).preferredBackend, + pinned != SnapshotBackendKind.recursiveTree.rawValue + else { return nil } + let impliedByCustomActions = command.preferredBackend == nil return Response( ok: false, error: ErrorPayload( code: "UNSUPPORTED_OPERATION", - message: "a multi-panel snapshot runs only the tree backend, so the \(pinned) backend cannot be pinned", - hint: "Snapshot without a backend pin: only the tree backend proves which panel a window is on (ADR 0025)." + message: impliedByCustomActions + ? "a multi-panel snapshot runs only the tree backend, so customActions, which only the \(pinned) backend reads, cannot be served" + : "a multi-panel snapshot runs only the tree backend, so the \(pinned) backend cannot be pinned", + hint: impliedByCustomActions + ? "Snapshot without customActions: only the tree backend proves which panel a window is on (ADR 0025)." + : "Snapshot without a backend pin: only the tree backend proves which panel a window is on (ADR 0025)." ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 16e7020d20..3aea84ba1a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -326,7 +326,7 @@ extension RunnerTests { @MainActor func runSynthesizedReplacementRoute( _ request: SynthesizedReplacementRequest - ) -> SynthesizedReplacementRouteOutcome { + ) throws -> SynthesizedReplacementRouteOutcome { #if os(iOS) NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder-replacement") let plan = Self.synthesizedTextPlan( @@ -418,7 +418,7 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable") guard let point = request.target.refreshPoint else { return .notApplicable } return .fallback( - focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y) + try focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y) ) } // The private synthesize call returns at post time, not commit time, and this route never diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 3d3d4d3987..caf5a21441 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -516,7 +516,8 @@ extension RunnerTests { /// Resolves the target's input, with the snapshot the resolution read it through. A bound target /// accepts only an input whose snapshot carries its bound identity, so every post, read-back, - /// verification poll, and repair refuses an input that took the bound one's place. + /// verification poll, and repair refuses an input that took the bound one's place. It re-reads a + /// target the request's focus read already resolved, so a lit-panel refusal here is no answer. func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { guard let boundIdentity = target.boundIdentity else { return resolveUnboundTextEntryInput(app: app, target: target) @@ -538,7 +539,7 @@ extension RunnerTests { } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), + case .matches(let candidates)? = try? probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } @@ -573,12 +574,12 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { + switch try? probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { case .absent: return true case .matches(let candidates): return candidates.allSatisfy { $0.identity != boundIdentity } - case .unavailable: + case .unavailable, nil: return false } } @@ -617,7 +618,7 @@ extension RunnerTests { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y, contentTranslation: target.contentTranslation) { + let refreshed = try? textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y, contentTranslation: target.contentTranslation) { return refreshed } return focusedTextInput(app: app).flatMap(read) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 0f598bcb68..86c4d8d85d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -104,7 +104,7 @@ extension RunnerTests { } @MainActor - func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { + func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) throws -> TextEntryTarget { let contentTranslation = liveContentTranslation(app: app) guard let x, let y else { let softwareKeyboardVisible = isKeyboardVisible(app: app) @@ -136,7 +136,7 @@ extension RunnerTests { } let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) + let input = try textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index e7ac87107b..87bd015de4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -20,18 +20,19 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double, contentTranslation: TextEntryContentTranslation) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first + func textInputAt(app: XCUIApplication, x: Double, y: Double, contentTranslation: TextEntryContentTranslation) throws -> ResolvedTextEntryInput? { + try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first } - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> [ResolvedTextEntryInput] { - safely("TEXT_INPUT_AT_POINT", []) { - queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) + func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) throws -> [ResolvedTextEntryInput] { + let queries = try textInputQueries(app: app) + return safely("TEXT_INPUT_AT_POINT", []) { + queryTextInputs(queries: queries, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } - func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: liveContentTranslation(app: app)) { + func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) throws -> XCUIElement? { + switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: liveContentTranslation(app: app)) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -58,8 +59,8 @@ extension RunnerTests { x: Double, y: Double, contentTranslation: TextEntryContentTranslation - ) -> CoordinateTextInputResolution { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { + ) throws -> CoordinateTextInputResolution { + switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): guard let input = inputs.first, let identity = input.identity else { return .unavailable } return .input(TextInputAtPoint(element: input.element, identity: identity)) @@ -93,14 +94,15 @@ extension RunnerTests { return read } - func probeTextInputs(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> TextInputProbeOutcome { + func probeTextInputs(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) throws -> TextInputProbeOutcome { + let queries = try textInputQueries(app: app) switch containingTextInputProbeIssues(fallback: TextInputProbeOutcome.unavailable(.emptyTree), { shouldStop in var treeReadable = app.descendants(matching: .any).firstMatch.exists #if AGENT_DEVICE_RUNNER_UNIT_TESTS if textInputProbeBlindForTesting { treeReadable = false } #endif guard treeReadable else { return .unavailable(.emptyTree) } - let inputs = queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) + let inputs = queryTextInputs(queries: queries, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) return inputs.isEmpty ? .absent : .matches(inputs) }) { case .success(let outcome): @@ -171,15 +173,21 @@ extension RunnerTests { return true } + /// The text-input queries of a read at a point. They are resolved before the read's issue + /// containment, so a lit-panel refusal answers the request instead of reading as no input. + private func textInputQueries(app: XCUIApplication) throws -> [XCUIElementQuery] { + let inputTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView] + return try inputTypes.flatMap { try elementQueries(app: app, matching: $0) } + } + private func queryTextInputs( - app: XCUIApplication, + queries: [XCUIElementQuery], point: CGPoint, contentTranslation: TextEntryContentTranslation, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] - let inputTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView] - for query in inputTypes.flatMap({ elementQueries(app: app, matching: $0) }) { + for query in queries { if shouldStop() { break } let candidates = query.allElementsBoundByIndex #if AGENT_DEVICE_RUNNER_UNIT_TESTS diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 142f9c4415..b2a26b45d3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -13,7 +13,7 @@ extension RunnerTests { xCTestChannelPenalized: Bool = false, synthesizer: any TextEntrySynthesizing, commandId: String? = nil - ) -> TextEntryResult { + ) throws -> TextEntryResult { let totalStartedAt = Date() let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } var reader = TextEntryReader(target: target, resolve: resolve) @@ -58,7 +58,7 @@ extension RunnerTests { xCTestChannelPenalized: xCTestChannelPenalized ) { - switch runSynthesizedReplacementRoute( + switch try runSynthesizedReplacementRoute( SynthesizedReplacementRequest( app: app, target: target, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index c02e2096c7..25e47ffcd6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -3,7 +3,7 @@ import AgentDeviceSnapshotPresentation extension RunnerTests { @MainActor - func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response { + func executeTypeCommand(activeApp: XCUIApplication, command: Command) throws -> Response { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "type requires text")) } @@ -33,7 +33,7 @@ extension RunnerTests { // answer both refuse here, each under its own reason. let contentTranslation = liveContentTranslation(app: activeApp) let inputAtPoint: TextInputAtPoint - switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { + switch try coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): inputAtPoint = input case .absent: @@ -68,7 +68,7 @@ extension RunnerTests { if let resolvedCoordinateTarget { target = resolvedCoordinateTarget } else { - target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + target = try focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) } NSLog( "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=focus durationMs=%.1f chars=%d mode=%@", @@ -95,7 +95,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: message)) } } - let textResult = typeTextReliably( + let textResult = try typeTextReliably( app: activeApp, target: target, text: text, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index c2f3fb2470..b2ba40d74d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -203,6 +203,9 @@ final class RunnerTests: XCTestCase { /// Simulates an app tree read that returns no descendants without an issue or exception. var textInputProbeBlindForTesting = false + /// Simulates a lit-panel window list the XCTest runtime cannot answer. + var litPanelWindowListUnanswerableForTesting = false + static let injectedTapFailureFlagPathForTesting = "/tmp/agent-device-inject-tap-recorded-failure-for-testing" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 125bf1e438..c7d5b0b1a4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -48,11 +48,18 @@ extension RunnerTests { XCTAssertEqual(queuedLater, 3) } - func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() { - XCTAssertEqual(Self.litPanelBackendPinRefusal("private-ax")?.error?.code, "UNSUPPORTED_OPERATION") - XCTAssertEqual(Self.litPanelBackendPinRefusal("queries")?.error?.code, "UNSUPPORTED_OPERATION") - XCTAssertNil(Self.litPanelBackendPinRefusal("tree")) - XCTAssertNil(Self.litPanelBackendPinRefusal(nil)) + func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() throws { + func refusal(_ fields: String) throws -> ErrorPayload? { + try Self.litPanelBackendPinRefusal(runnerCommandFixture(#"{"command":"snapshot"\#(fields)}"#))?.error + } + XCTAssertEqual(try refusal(#","preferredBackend":"private-ax""#)?.code, "UNSUPPORTED_OPERATION") + XCTAssertEqual(try refusal(#","preferredBackend":"queries""#)?.code, "UNSUPPORTED_OPERATION") + XCTAssertNil(try refusal(#","preferredBackend":"tree""#)) + XCTAssertNil(try refusal("")) + let implied = try XCTUnwrap(refusal(#","customActions":true"#)) + XCTAssertEqual(implied.code, "UNSUPPORTED_OPERATION") + XCTAssertTrue(implied.message.contains("customActions"), implied.message) + XCTAssertFalse(implied.hint?.contains("backend pin") ?? true, implied.hint ?? "") } /// Decoded from JSON, so a renamed wire field fails here. @@ -109,6 +116,38 @@ extension RunnerTests { XCTAssertFalse(fellBack, "no read in the request may fall back to the app") } + /// The app's window is booked on the lit panel, and then its window list stops answering: an + /// element read refuses with the typed code instead of finding nothing, and a coordinate `type` + /// refuses before it taps a point with no target. + @MainActor + func testElementReadsRefuseWhenTheLitPanelWindowListIsUnanswerable() throws { + app.launch() + mainOwned.app = app + defer { + litPanelWindowListUnanswerableForTesting = false + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let window = app.windows.firstMatch + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + let frame = window.frame + let panel = #""litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height)}"# + for json in [ + #"{"command":"readText","x":\#(frame.midX),"y":\#(frame.midY),\#(panel)}"#, + #"{"command":"type","text":"x","x":\#(frame.midX),"y":\#(frame.midY),\#(panel)}"#, + ] { + let command = try runnerCommandFixture(json) + setRequestLitPanel(from: command) + XCTAssertNil(bookRequestLitPanelSurface(app: app), "the window is on the lit panel") + litPanelWindowListUnanswerableForTesting = true + let response = try executeOnMainPrepared(command: command, activeApp: app) + litPanelWindowListUnanswerableForTesting = false + XCTAssertEqual(response.error?.code, "APP_SCREEN_UNRESOLVED", json) + } + } + /// With a lit panel named, selector and point reads search only the windows on that display; the /// launched app's windows are all on another one, so neither read may answer. @MainActor @@ -117,18 +156,18 @@ extension RunnerTests { _ = try responseWithTheAppOffTheLitPanel( #"{"command":"tap","x":1,"y":1,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# ) { _ in - found.named = findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") + found.named = try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") .element != nil let unnamed = try runnerCommandFixture(#"{"command":"tap"}"#) setRequestLitPanel(from: unnamed) - let match = findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") + let match = try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") found.unnamed = match.element != nil if let frame = match.element?.frame { - found.text = readTextAt(app: app, x: frame.midX, y: frame.midY) + found.text = try readTextAt(app: app, x: frame.midX, y: frame.midY) setRequestLitPanel(from: try runnerCommandFixture( #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# )) - found.namedText = readTextAt(app: app, x: frame.midX, y: frame.midY) + found.namedText = try readTextAt(app: app, x: frame.midX, y: frame.midY) } return Response(ok: true) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index a91f188fee..3e6666d6cd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { + switch try coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { case .input(let input): return input case .absent: @@ -65,7 +65,7 @@ extension RunnerTests { let typeCommand = try runnerCommandFixture( #"{"command":"type","commandId":"type-after-penalized-non-text-target","text":"must-not-type","textEntryMode":"append"}"# ) - let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + let typeResponse = try executeTypeCommand(activeApp: app, command: typeCommand) XCTAssertFalse(typeResponse.ok) XCTAssertEqual(typeResponse.error?.code, "TEXT_INPUT_NOT_FOCUSED") @@ -99,7 +99,7 @@ extension RunnerTests { #"{"command":"type","commandId":"fill-penalized-non-text-point","text":"must-not-replace","textEntryMode":"replace","x":\#(nonTextFrame.midX),"y":\#(nonTextFrame.midY)}"# ) let failureCountBefore = currentXCTestFailureCount() - let fillResponse = executeTypeCommand(activeApp: app, command: fillCommand) + let fillResponse = try executeTypeCommand(activeApp: app, command: fillCommand) XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertFalse(fillResponse.ok, String(describing: fillResponse.error)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 76bf0c0e77..03bf17d3f4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -77,7 +77,7 @@ extension RunnerTests { ]) ) let failuresBeforeType = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) return response } @@ -226,7 +226,7 @@ extension RunnerTests { let label = "\(testCase.seed.count) characters, all select-alls dropped" let seeded = try replaceSynthesizedFieldText(field, text: testCase.seed, commandId: "seed-\(testCase.seed.count)") XCTAssertTrue(seeded.ok, String(describing: seeded.error)) - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -324,7 +324,7 @@ extension RunnerTests { ] for (handle, inputAtRefreshPoint) in found { - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -357,7 +357,7 @@ extension RunnerTests { XCTAssertEqual(rebound.identifier, "agent-device-text-entry-neighbour") let neighbourFrame = neighbour.frame - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -392,7 +392,7 @@ extension RunnerTests { let neighbourFrame = neighbour.frame let failures = currentXCTestFailureCount() - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -453,7 +453,7 @@ extension RunnerTests { textInputProbeBlindForTesting = true let frame = textField.frame - guard case .unavailable(.emptyTree) = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) else { + guard case .unavailable(.emptyTree) = try probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) else { return XCTFail("A blind tree must be unavailable to every probe caller") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift index a3b4fe7a5a..b3749e1f02 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryConfirmationTests.swift @@ -298,7 +298,7 @@ extension RunnerTests { "x": frame.midX, "y": frame.midY, "appBundleId": "com.callstack.agentdevice.runner", ])) let failures = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "TEXT_ENTRY_MISMATCH") @@ -338,7 +338,7 @@ extension RunnerTests { "x": frame.midX, "y": frame.midY, "appBundleId": "com.callstack.agentdevice.runner", ])) let failures = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertTrue(response.ok, String(describing: response.error)) XCTAssertEqual(response.data?.message, "typed") @@ -435,7 +435,7 @@ extension RunnerTests { ) let failuresBeforeType = currentXCTestFailureCount() - let response = executeTypeCommand(activeApp: app, command: command) + let response = try executeTypeCommand(activeApp: app, command: command) XCTAssertFalse(didRecordXCTestFailure(since: failuresBeforeType)) XCTAssertTrue(response.ok, String(describing: response.error)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 9b94a9df90..78fc0037ab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -54,7 +54,7 @@ extension RunnerTests { ) let readsBefore = try XCTUnwrap(Int(valueReads.label)) - let response = executeTypeCommand(activeApp: app, command: fill) + let response = try executeTypeCommand(activeApp: app, command: fill) let readsAfter = try XCTUnwrap(Int(valueReads.label)) XCTAssertTrue(response.ok, String(describing: response.error)) @@ -76,14 +76,14 @@ extension RunnerTests { let valueReads = app.staticTexts["agent-device-text-entry-value-reads"] let frame = field.frame let point = CGPoint(x: frame.midX, y: frame.midY) - func countingReads(_ body: () -> T) throws -> (T, Int) { + func countingReads(_ body: () throws -> T) throws -> (T, Int) { let before = try XCTUnwrap(Int(valueReads.label)) - let value = body() + let value = try body() return (value, try XCTUnwrap(Int(valueReads.label)) - before) } let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false, contentTranslation: .none) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) } + let (lookup, lookupReads) = try countingReads { try textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -146,7 +146,7 @@ extension RunnerTests { let readsBefore = try XCTUnwrap(Int(valueReads.label)) let fillStartedAt = Date() - let response = executeTypeCommand(activeApp: app, command: fill) + let response = try executeTypeCommand(activeApp: app, command: fill) let readsAfter = try XCTUnwrap(Int(valueReads.label)) try XCTSkipUnless( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 20f62e56b9..00fb6fe970 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -733,7 +733,7 @@ extension RunnerTests { defer { tearDownSynthesizedReplacementField() } let synthesizer = RecordingTextEntrySynthesizer() let frame = field.frame - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -778,10 +778,10 @@ extension RunnerTests { // `_XCTestCaseInterruptionException`, which XCTest records as a test failure even when production // code catches it. Springboard is always running, so the query resolves to no input at the point. @MainActor - func testSynthesizedReplacementTypesNothingWhenItCannotReadTheFieldBack() { + func testSynthesizedReplacementTypesNothingWhenItCannotReadTheFieldBack() throws { let synthesizer = RecordingTextEntrySynthesizer() let failures = currentXCTestFailureCount() - let result = typeTextReliably( + let result = try typeTextReliably( app: springboard, target: TextEntryTarget( element: nil, @@ -809,8 +809,8 @@ extension RunnerTests { // fail closed: it used to fall through to the vacuous-typing early return and report // `verified: true` for a clear that never ran. @MainActor - func testEmptyReplacementWithoutResolvableTargetFailsClosed() { - let result = typeTextReliably( + func testEmptyReplacementWithoutResolvableTargetFailsClosed() throws { + let result = try typeTextReliably( app: springboard, target: TextEntryTarget(element: nil, refreshPoint: nil, prefersFocusedElement: false, contentTranslation: .none), text: "", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index 5d1fe3a468..0edffaf660 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: .none) + _ = try? self.probeTextInputs(app: self.app, point: point, contentTranslation: .none) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) - _ = probeTextInputs(app: app, point: point, contentTranslation: .none) + _ = try? textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) + _ = try? probeTextInputs(app: app, point: point, contentTranslation: .none) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) + let outcome = try probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 75764dcd42..e692526a79 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -10,7 +10,7 @@ extension RunnerTests { #"{"command":"type","commandId":"type-without-focus","text":"hello","textEntryMode":"append"}"# ) - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: XCUIApplication(bundleIdentifier: "com.example.agentdevice.missing-input"), command: command ) @@ -31,7 +31,7 @@ extension RunnerTests { try skipUnlessSoftwareKeyboardIsHidden() let failureCountBefore = currentXCTestFailureCount() - let typeResponse = executeTypeCommand( + let typeResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard","text":"hardware-keyboard","textEntryMode":"append"}"# @@ -46,7 +46,7 @@ extension RunnerTests { // The tap witness is one-shot: a second bare type without a new tap has no target. let unfocusedFailureCountBefore = currentXCTestFailureCount() - let unfocusedResponse = executeTypeCommand( + let unfocusedResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard-unfocused","text":"-again","textEntryMode":"append"}"# @@ -63,7 +63,7 @@ extension RunnerTests { // keyboard here is a product change, not an environment fact. XCTAssertFalse(isKeyboardVisible(app: app)) let appendFailureCountBefore = currentXCTestFailureCount() - let appendResponse = executeTypeCommand( + let appendResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard-again","text":"-again","textEntryMode":"append"}"# @@ -83,7 +83,7 @@ extension RunnerTests { try skipUnlessSoftwareKeyboardIsHidden() let failureCountBefore = currentXCTestFailureCount() - let submitResponse = executeTypeCommand( + let submitResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"type-hardware-keyboard-submit","text":"\n"}"# @@ -102,7 +102,7 @@ extension RunnerTests { try skipUnlessSoftwareKeyboardIsHidden() let failureCountBefore = currentXCTestFailureCount() - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: textField, @@ -146,7 +146,7 @@ extension RunnerTests { let typeCommand = try runnerCommandFixture( #"{"command":"type","commandId":"type-disappearing-input","text":"ab","delayMs":50,"textEntryMode":"append"}"# ) - let typeResponse = executeTypeCommand(activeApp: app, command: typeCommand) + let typeResponse = try executeTypeCommand(activeApp: app, command: typeCommand) XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertFalse(typeResponse.ok) @@ -163,7 +163,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-auto-submit", text: "123456", at: textField) ) @@ -183,7 +183,7 @@ extension RunnerTests { func testFillReplacesAValueLongerThanOneClearBurst() throws { let textField = try launchRemovableInputFixture("--agent-device-text-entry-long-prefill") - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-long-prefill", text: "Ada Lovelace", at: textField) ) @@ -197,7 +197,7 @@ extension RunnerTests { func testFillClearsAValueLongerThanOneClearBurst() throws { let textField = try launchRemovableInputFixture("--agent-device-text-entry-long-prefill") - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "clear-long-prefill", text: "", at: textField) ) @@ -216,7 +216,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - _ = executeTypeCommand( + _ = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-clear-replaced-input", text: "", at: textField) ) @@ -234,7 +234,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-auto-submit-without-successor") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-auto-submit-no-successor", text: "123456", at: textField) ) @@ -253,7 +253,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-disappear-after-input") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-disappearing-input", text: "123456", at: textField) ) @@ -272,7 +272,7 @@ extension RunnerTests { let textField = try launchRemovableInputFixture("--agent-device-text-entry-replace-after-input") let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-replaced-input", text: "123456", at: textField) ) @@ -294,7 +294,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-disappearing-masked-input", text: "123456", at: textField) ) @@ -314,7 +314,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-replaced-masked-input", text: "123456", at: textField) ) @@ -349,7 +349,7 @@ extension RunnerTests { defer { textEntryWarmupPauseForTesting = nil } let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-masked-replaced-in-pause", text: "123456", at: textField) ) @@ -375,7 +375,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-unnamed-successor", text: "123456", at: textField) ) @@ -398,7 +398,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-digit-count-auto-submit", text: "123456", at: textField) ) @@ -425,7 +425,7 @@ extension RunnerTests { let frameBefore = textField.frame let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-moves-on-input", text: "hello", at: textField) ) @@ -451,7 +451,7 @@ extension RunnerTests { ) let failureCountBefore = currentXCTestFailureCount() - let response = executeTypeCommand( + let response = try executeTypeCommand( activeApp: app, command: try fillCommandFixture(commandId: "fill-unnamed-replaced", text: "123456", at: textField) ) @@ -510,7 +510,7 @@ extension RunnerTests { let failureCountBefore = currentXCTestFailureCount() // The target the `type` command builds when it cannot resolve an input but the keyboard is up: // no element, no refresh point, focused-element preference. - let result = typeTextReliably( + let result = try typeTextReliably( app: app, target: TextEntryTarget( element: nil, @@ -564,7 +564,7 @@ extension RunnerTests { try tapHardwareKeyboardInput(commandId: "tap-masked-input") let failureCountBefore = currentXCTestFailureCount() - let fillResponse = executeTypeCommand( + let fillResponse = try executeTypeCommand( activeApp: app, command: try runnerCommandFixture( #"{"command":"type","commandId":"fill-masked-input","text":"not-a-secret-123","textEntryMode":"replace"}"# diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index ce15c0d5e3..f86080eabd 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -9,6 +9,7 @@ {"name": "ios-device.recording-stop.session", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "recordStop", "appBundleId": "com.example.app"}}, {"name": "ios-foldable.alert.get", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "alert", "action": "get", "appBundleId": "com.example.app", "timeoutMs": 10000, "commandId": ""}}, {"name": "ios-foldable.interactions-tap.synthesized", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "tap", "x": 10, "y": 20, "synthesized": true, "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, + {"name": "ios-foldable.interactor-find-text.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "findText", "text": "Search", "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, {"name": "ios-foldable.interactor-keyboard-dismiss.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, {"name": "ios-foldable.interactor-read-text.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, {"name": "ios-foldable.interactor-snapshot.lit-panel", "producer": "packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "litPanel": {"displayId": 3, "widthPt": 669, "heightPt": 951}, "commandId": ""}}, diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 0362a2199d..7925aa246e 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -715,13 +715,13 @@ Android reports the app window on the next snapshot instead of a panel. SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -Decision: the host names the lit panel in each runner request that resolves a screen. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. +Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. -The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Element reads (`readText`, the text-input lookup, and selector queries) search only under the application's windows that report the lit display. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. +The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search owns this rule: if the window list cannot be read, the search throws `APP_SCREEN_UNRESOLVED`, which answers the request instead of no match. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. -A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. Element reads whose window list cannot be read refuse with `APP_SCREEN_UNRESOLVED` instead of finding nothing. +A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. A `customActions` request implies the private AX pin, so its refusal names `customActions`. Rejected alternatives: diff --git a/packages/apple-runner/src/runner-command-traits.ts b/packages/apple-runner/src/runner-command-traits.ts index d2672344f8..ed11e98248 100644 --- a/packages/apple-runner/src/runner-command-traits.ts +++ b/packages/apple-runner/src/runner-command-traits.ts @@ -94,7 +94,8 @@ export const RUNNER_COMMAND_TRAITS = { /** * Whether the runner resolves a screen for the command: a window to book a gesture on, a viewport, - * or a display to sample. Such a request for a multi-panel device names the lit panel (ADR 0025). + * a display to sample, or windows to search for an element. Such a request for a multi-panel device + * names the lit panel (ADR 0025). This table is the one list of lit-panel-scoped commands. */ const RUNNER_COMMAND_RESOLVES_SURFACE = { tap: true, @@ -106,8 +107,8 @@ const RUNNER_COMMAND_RESOLVES_SURFACE = { swipe: true, scroll: true, desktopScroll: false, - findText: false, - querySelector: false, + findText: true, + querySelector: true, readText: true, snapshot: true, screenshot: true, diff --git a/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts b/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts index 6e3994e6e3..b78e96a010 100644 --- a/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-foldable-runner-requests.test.ts @@ -60,6 +60,11 @@ const INTERACTOR_SITES: Record = { DUO_SIMULATOR, (i) => i.readTextAtPoint!({ x: 10, y: 20 }, { appBundleId: APP }), ], + // An element search reads only the lit panel's windows, so the dark panel's duplicate cannot answer. + 'ios-foldable.interactor-find-text.lit-panel': [ + DUO_SIMULATOR, + (i) => i.findText!('Search', { appBundleId: APP }), + ], // A request that resolves no screen carries no panel: alert, lifecycle, readiness and clipboard // traffic must stay byte-for-byte the request it was. 'ios-foldable.alert.get': [DUO_SIMULATOR, (i) => i.readAlert!({ appBundleId: APP })], diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index cc43689c2b..0e0982812c 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -29,7 +29,7 @@ export async function runAppleRunnerCommand( command: RunnerCommand, options: AppleRunnerCommandOptions = {}, ): Promise> { - if (command.litPanel || !runnerCommandResolvesSurface(command)) { + if (!runnerCommandResolvesSurface(command)) { return await runHostBoundAppleRunnerCommand(device, command, options); } const { resolveRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); From 8bf25406ffb0457f6c4e1bddb2f34b93b3c5f35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 20:11:38 +0100 Subject: [PATCH 10/29] test(ios): share the lit-panel and penalized-clear test setups The new unreadable-list case and the pin case launched the app on its own panel the same way, and the added `try` made two penalized-clear calls a new clone. One helper each removes both clones. Co-Authored-By: Claude Opus 5.5 --- .../UnitTests/RunnerRequestPanelTests.swift | 57 ++++++------- ...unnerTests+SynthesizedTextEntryTests.swift | 83 ++++++++----------- 2 files changed, 61 insertions(+), 79 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index c7d5b0b1a4..55d8ad6f45 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -83,6 +83,27 @@ extension RunnerTests { } #if os(iOS) + /// Launches the app and names its own window's display and box as the lit panel: the + /// `litPanel` JSON member, and the window frame. + @MainActor + private func launchAppOnTheLitPanel() -> (panel: String, frame: CGRect) { + app.launch() + mainOwned.app = app + let window = app.windows.firstMatch + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) + let frame = window.frame + return (#""litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height)}"#, frame) + } + + @MainActor + private func tearDownLitPanelApp() { + litPanelWindowListUnanswerableForTesting = false + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + /// Runs `command` against the launched app while the request names display 999, which none of /// the app's windows is on. @MainActor @@ -92,11 +113,7 @@ extension RunnerTests { ) throws -> Response { app.launch() mainOwned.app = app - defer { - setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } + defer { tearDownLitPanelApp() } let command = try runnerCommandFixture(json) setRequestLitPanel(from: command) return try run(command) @@ -121,19 +138,8 @@ extension RunnerTests { /// refuses before it taps a point with no target. @MainActor func testElementReadsRefuseWhenTheLitPanelWindowListIsUnanswerable() throws { - app.launch() - mainOwned.app = app - defer { - litPanelWindowListUnanswerableForTesting = false - setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let window = app.windows.firstMatch - var displayID: UInt = 0 - XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) - let frame = window.frame - let panel = #""litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height)}"# + let (panel, frame) = launchAppOnTheLitPanel() + defer { tearDownLitPanelApp() } for json in [ #"{"command":"readText","x":\#(frame.midX),"y":\#(frame.midY),\#(panel)}"#, #"{"command":"type","text":"x","x":\#(frame.midX),"y":\#(frame.midY),\#(panel)}"#, @@ -181,20 +187,11 @@ extension RunnerTests { /// invalidate the cached target the next command relies on. @MainActor func testLitPanelSnapshotPinRefusalKeepsTheCachedTarget() throws { - app.launch() - mainOwned.app = app + let (panel, _) = launchAppOnTheLitPanel() mainOwned.bundleId = "com.callstack.agentdevice.runner" - defer { - setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let window = app.windows.firstMatch - var displayID: UInt = 0 - XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) - let size = window.frame.size + defer { tearDownLitPanelApp() } let command = try runnerCommandFixture( - #"{"appBundleId":"com.callstack.agentdevice.runner","command":"snapshot","preferredBackend":"private-ax","litPanel":{"displayId":\#(displayID),"widthPt":\#(size.width),"heightPt":\#(size.height)}}"# + #"{"appBundleId":"com.callstack.agentdevice.runner","command":"snapshot","preferredBackend":"private-ax",\#(panel)}"# ) setRequestLitPanel(from: command) let response = try executeSnapshotDispatched(command: command) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 03bf17d3f4..1758544ac5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -253,6 +253,27 @@ extension RunnerTests { } } + /// Clears through the penalized synthesized route at `frame`'s center, where the pre-tap lookup + /// found `input`. + @MainActor + private func clearPenalized(at frame: CGRect, finding input: TextInputAtPoint) throws -> TextEntryResult { + try typeTextReliably( + app: app, + target: TextEntryTarget( + element: nil, + refreshPoint: CGPoint(x: frame.midX, y: frame.midY), + prefersFocusedElement: false, + contentTranslation: .none, + inputAtRefreshPoint: input + ), + text: "", + delaySeconds: 0, + repairMode: .replacement, + xCTestChannelPenalized: true, + synthesizer: RecordingTextEntrySynthesizer() + ) + } + /// Launches the fixture whose field moves away when focused and leaves a neighbouring field under /// the point the focus tap hit, the way a React Native bottom sheet extends above the keyboard. /// The field is not focused yet, so the replacement's own tap starts the move. @@ -324,21 +345,7 @@ extension RunnerTests { ] for (handle, inputAtRefreshPoint) in found { - let result = try typeTextReliably( - app: app, - target: TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - inputAtRefreshPoint: inputAtRefreshPoint - ), - text: "", - delaySeconds: 0, - repairMode: .replacement, - xCTestChannelPenalized: true, - synthesizer: RecordingTextEntrySynthesizer() - ) + let result = try clearPenalized(at: neighbourFrame, finding: inputAtRefreshPoint) XCTAssertEqual(result.failure, .notFocused, handle) XCTAssertEqual(String(describing: neighbour.value ?? ""), "neighbour", handle) @@ -357,23 +364,12 @@ extension RunnerTests { XCTAssertEqual(rebound.identifier, "agent-device-text-entry-neighbour") let neighbourFrame = neighbour.frame - let result = try typeTextReliably( - app: app, - target: TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - inputAtRefreshPoint: TextInputAtPoint( - element: rebound, - identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) - ) - ), - text: "", - delaySeconds: 0, - repairMode: .replacement, - xCTestChannelPenalized: true, - synthesizer: RecordingTextEntrySynthesizer() + let result = try clearPenalized( + at: neighbourFrame, + finding: TextInputAtPoint( + element: rebound, + identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) + ) ) XCTAssertNil(result.failure) @@ -392,23 +388,12 @@ extension RunnerTests { let neighbourFrame = neighbour.frame let failures = currentXCTestFailureCount() - let result = try typeTextReliably( - app: app, - target: TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - inputAtRefreshPoint: TextInputAtPoint( - element: app.textFields.element, - identity: TextEntryInputIdentity(elementType: .textField, identifier: "") - ) - ), - text: "", - delaySeconds: 0, - repairMode: .replacement, - xCTestChannelPenalized: true, - synthesizer: RecordingTextEntrySynthesizer() + let result = try clearPenalized( + at: neighbourFrame, + finding: TextInputAtPoint( + element: app.textFields.element, + identity: TextEntryInputIdentity(elementType: .textField, identifier: "") + ) ) XCTAssertFalse(didRecordXCTestFailure(since: failures)) From f2452c3c042d50980346fbd95330472f471792ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 20:42:37 +0100 Subject: [PATCH 11/29] docs(adr): qualify when customActions implies the private AX pin Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 7925aa246e..ab600e58c7 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -721,7 +721,7 @@ With a named panel, the runner walks at most eight windows. It books the first w The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search owns this rule: if the window list cannot be read, the search throws `APP_SCREEN_UNRESOLVED`, which answers the request instead of no match. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. -A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. A `customActions` request implies the private AX pin, so its refusal names `customActions`. +A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. A `customActions` request with no explicit backend pin implies the private AX pin, so its refusal names `customActions`. Rejected alternatives: From 5b72da74e3e0376cc1b6fff7402410c7d7a93fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 20:52:10 +0100 Subject: [PATCH 12/29] fix(ios): refuse an unread lit-panel window list instead of falling back An unreadable window list on a lit-panel request resolved as "no window", which sends a capture to the system-surface fallback. It now refuses with APP_SCREEN_UNRESOLVED. A panel read that spends the whole request budget is refused before sending, instead of handing the runner a 1 ms timeout. Co-Authored-By: Claude Opus 5.5 --- .../RunnerXCTestEventBridge.h | 6 +++--- .../RunnerXCTestEventBridge.m | 3 ++- .../RunnerTests+XCTestEventBridgeTests.swift | 11 ++++++++++ docs/adr/0025-foldable-apple-panels.md | 2 +- .../src/foldable/runner-panel.test.ts | 20 +++++++++++++++++++ .../src/foldable/runner-panel.ts | 14 +++++++++---- 6 files changed, 47 insertions(+), 9 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 2382c926d8..6eeba802c7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -46,9 +46,9 @@ typedef struct { #define AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS 8 // Resolves the window on `litPanel`: the first window on that display whose frame is the panel's -// box either way up, else the largest window on that display. An unplaced window, or a list past -// the cap with no full-panel window in it, is `UnresolvedScreen`. Windows that are all on another -// display are `OffLitPanel`; no window at all is `UnresolvedWindow`. +// box either way up, else the largest window on that display. An unreadable list, an unplaced +// window, or a list past the cap with no full-panel window in it, is `UnresolvedScreen`. Windows +// that are all on another display are `OffLitPanel`; no window at all is `UnresolvedWindow`. FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( id application, RunnerLitPanel litPanel, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 380ea089ce..f837e60c0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -244,9 +244,10 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( *window = nil; *windowFrame = CGRectNull; *screen = nil; + // An unread list is not an app without windows, so it never reaches the system-surface fallback. NSArray *windows = RunnerCopyWindowList(application); if (windows == nil) { - return RunnerApplicationScreenFailureUnresolvedWindow; + return RunnerApplicationScreenFailureUnresolvedScreen; } // A window the walk could not place may be the panel's surface, so it refuses every booking. A // list past the cap refuses only when no full-panel window was found before it. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index d27b33d160..f9c81da9cb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -303,6 +303,17 @@ extension RunnerTests { XCTAssertEqual(resolveLitPanelWindow([]).failure, .unresolvedWindow, "no window at all") } + /// An unread window list is not an app without windows: it must not reach the system-surface + /// fallback, which only `UnresolvedWindow` takes. + func testLitPanelWindowRefusesAWindowListItCannotRead() { + var window: AnyObject? + var frame = CGRect.null + let failure = RunnerResolveLitPanelWindow( + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel, &window, &frame) + XCTAssertEqual(failure, .unresolvedScreen) + XCTAssertNil(window) + } + /// A window the walk could not place may itself be the panel's surface, so a full-panel window /// read after it is not booked. func testLitPanelWindowRefusesAFullPanelWindowReadAfterAnUnplacedOne() { diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index ab600e58c7..3a27396ba4 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -717,7 +717,7 @@ SpringBoard keeps windows on both panels. Its first window is the outer panel's Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. -With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. +With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search owns this rule: if the window list cannot be read, the search throws `APP_SCREEN_UNRESOLVED`, which answers the request instead of no match. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index e0465e98e8..d450686577 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -172,6 +172,26 @@ test('a surface command probes the panel table within its own timeout', async () assert.ok(sentTimeoutMs > 0 && sentTimeoutMs < 1_200, 'the runner gets what the read left'); }); +test('a panel read that spends the whole request budget refuses instead of sending', async () => { + mockPanelTable.mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)); + return { ok: true as const, payload: IPHONE_DUO_CLOSED }; + }); + const device = simulator({ name: 'iPhone Duo' }); + const calls: RecordedRunnerCall[] = []; + await assert.rejects( + withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 20 }), + ), + (error: AppError) => { + assert.equal(error.details?.reason, 'panel-table-unreadable'); + assert.equal(error.details?.dispatched, 'no'); + return true; + }, + ); + assert.equal(calls.length, 0, 'a request with no time left never reaches the runner'); +}); + test('a known multi-panel device refuses a surface command when its panel table goes unread', async () => { const device = simulator({ name: 'iPhone Duo' }); const calls: RecordedRunnerCall[] = []; diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 5bdd06ab25..51bdc0cb9d 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -11,13 +11,16 @@ const PANEL_TABLE_UNREADABLE_HINT = /** * The lit panel a runner request names, read fresh per request because a fold changes it (ADR - * 0025). The read never outlasts the request's `timeoutMs`. Undefined keeps the runner's historic - * first-window route, which only a device not known to have more than one panel may take. + * 0025). The read never outlasts the request's `timeoutMs`; a read that spends all of it is + * refused like an unread table, so the runner never gets a request with no time left. Undefined + * keeps the runner's historic first-window route, which only a device not known to have more than + * one panel may take. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo, options: { signal?: AbortSignal; requestTimeoutMs?: number }, ): Promise { + const startedAt = Date.now(); const read = await readFoldablePanelTable(device, { signal: options.signal, timeoutMs: Math.min( @@ -25,7 +28,10 @@ export async function resolveRunnerRequestLitPanel( IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, ), }); - if (read.kind === 'unreadable') { + if (read.kind === 'single-panel') return undefined; + const spentRequest = + options.requestTimeoutMs !== undefined && Date.now() - startedAt >= options.requestTimeoutMs; + if (read.kind === 'unreadable' || spentRequest) { throw new AppError( 'COMMAND_FAILED', `The panel table of ${device.name} did not answer, so the lit panel cannot be named`, @@ -38,5 +44,5 @@ export async function resolveRunnerRequestLitPanel( }, ); } - return read.kind === 'multi-panel' ? appleLitPanel(read.inventory) : undefined; + return appleLitPanel(read.inventory); } From 45a4ad49c6abaf22c43f88ebc3a43aafd63d6783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 23:05:19 +0100 Subject: [PATCH 13/29] fix(ios): contain the lit-panel window walk and keep its refusals closed The element search's window walk now runs inside an issue scope, so a window that vanishes mid-walk refuses with APP_SCREEN_UNRESOLVED instead of recording an XCTest failure. A text-entry re-read refused at its refresh point answers no input instead of the app-wide identifier query, and the Maestro-compat frame gate refuses a lit-panel request with no booked window. The snapshot pin refusal keeps one message that names its cause. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 19 ++++---- .../RunnerTests+Interaction.swift | 7 ++- .../RunnerTests+SnapshotExecution.swift | 14 ++---- .../RunnerTests+TextEntry.swift | 26 ++++++----- .../RunnerTests.swift | 4 +- .../UnitTests/RunnerRequestPanelTests.swift | 43 ++++++++++++++++--- .../src/foldable/runner-panel.ts | 6 +-- 7 files changed, 77 insertions(+), 42 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index cde16c9881..41236f3842 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -59,19 +59,22 @@ extension RunnerTests { } /// Where an element read searches: the application, or on a lit-panel request only the windows - /// proven on the lit display, so a dark-panel element can never answer. A lit-panel window list - /// that cannot be read throws `APP_SCREEN_UNRESOLVED`, so no read can mistake it for no match. + /// proven on the lit display. A list that cannot be read throws `APP_SCREEN_UNRESOLVED`. The walk + /// runs inside an issue scope, so a window that vanishes mid-walk refuses instead of ending the + /// session (#2995). func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] { guard let litPanel = requestLitPanel else { return [app] } if let cached = litPanelState.windows, cached.app === app { return cached.elements } + var windows: [XCUIElement]? + let walk = holdingRecordedIssues { + windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap { $0 as? XCUIElement } #if AGENT_DEVICE_RUNNER_UNIT_TESTS - let unanswerable = litPanelWindowListUnanswerableForTesting -#else - let unanswerable = false + if litPanelWindowWalkIssueForTesting { + record(XCTIssue(type: .assertionFailure, compactDescription: "Window vanished during the lit-panel walk")) + } #endif - guard !unanswerable, - let windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap({ $0 as? XCUIElement }) - else { + } + guard walk.issues.isEmpty, walk.exceptionMessage == nil, let windows else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } litPanelState.windows = (app, windows) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 8d5c4bdd2e..b6191bf1fd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -233,13 +233,18 @@ extension RunnerTests { // with no frame at all cannot be coordinate-tapped, otherwise the decision // is the shared TapPointPolicy center-in-frame rule (golden parity table // with the TS twin). app.frame is the frame source here — replay taps - // resolved bounds Maestro-style, so the union frame is intentional. + // resolved bounds Maestro-style, so the union frame is intentional. A + // lit-panel request uses its booked window, because app.frame names one + // panel, and refuses when no window is booked (ADR 0025). private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { let frame = element.frame if frame.isEmpty { return false } let windowFrame = requestLitPanel == nil ? app.frame : onScreenWindowFrame(app: app) + if requestLitPanel != nil, windowFrame.isNull { + return false + } return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: windowFrame) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index 6874f3dacb..471827e1c0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -170,26 +170,20 @@ extension RunnerTests { } /// A multi-panel snapshot runs only the tree backend, the one that proves which panel a window is - /// on, so a pin to another backend is refused as a request. It never reaches the capture plan and - /// leaves the cached target alone. `customActions` without a pin implies the private-ax pin, so - /// the refusal names it. + /// on, so a pin to another backend, or `customActions` implying one, is refused as a request. static func litPanelBackendPinRefusal(_ command: Command) -> Response? { guard let pinned = presentationOptions(from: command).preferredBackend, pinned != SnapshotBackendKind.recursiveTree.rawValue else { return nil } - let impliedByCustomActions = command.preferredBackend == nil + let cause = command.preferredBackend == nil ? "customActions" : "the \(pinned) backend pin" return Response( ok: false, error: ErrorPayload( code: "UNSUPPORTED_OPERATION", - message: impliedByCustomActions - ? "a multi-panel snapshot runs only the tree backend, so customActions, which only the \(pinned) backend reads, cannot be served" - : "a multi-panel snapshot runs only the tree backend, so the \(pinned) backend cannot be pinned", - hint: impliedByCustomActions - ? "Snapshot without customActions: only the tree backend proves which panel a window is on (ADR 0025)." - : "Snapshot without a backend pin: only the tree backend proves which panel a window is on (ADR 0025)." + message: "a multi-panel snapshot runs only the tree backend, so \(cause) cannot be served", + hint: "Snapshot without \(cause): only the tree backend proves which panel a window is on (ADR 0025)." ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index caf5a21441..3fc5e3dc2d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -516,8 +516,8 @@ extension RunnerTests { /// Resolves the target's input, with the snapshot the resolution read it through. A bound target /// accepts only an input whose snapshot carries its bound identity, so every post, read-back, - /// verification poll, and repair refuses an input that took the bound one's place. It re-reads a - /// target the request's focus read already resolved, so a lit-panel refusal here is no answer. + /// verification poll, and repair refuses an input that took the bound one's place. A lit-panel + /// refusal at the refresh point answers no input, never the app-wide reads after it. func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { guard let boundIdentity = target.boundIdentity else { return resolveUnboundTextEntryInput(app: app, target: target) @@ -537,11 +537,13 @@ extension RunnerTests { let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } - if target.inputAtRefreshPoint == nil, - let refreshPoint = target.refreshPoint, - case .matches(let candidates)? = try? probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), - let match = candidates.first(where: { $0.identity == boundIdentity }) { - return match + if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint { + guard let probe = try? probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) else { + return nil + } + if case .matches(let candidates) = probe, let match = candidates.first(where: { $0.identity == boundIdentity }) { + return match + } } if let focused = focusedTextInput(app: app), let input = boundInput(focused) { return input @@ -617,9 +619,13 @@ extension RunnerTests { if let element = target.element, let input = read(element) { return input } - if let refreshPoint = target.refreshPoint, - let refreshed = try? textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y, contentTranslation: target.contentTranslation) { - return refreshed + if let refreshPoint = target.refreshPoint { + guard let candidates = try? textInputCandidatesAt(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) else { + return nil + } + if let refreshed = candidates.first { + return refreshed + } } return focusedTextInput(app: app).flatMap(read) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b2ba40d74d..c4582e58b6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -203,8 +203,8 @@ final class RunnerTests: XCTestCase { /// Simulates an app tree read that returns no descendants without an issue or exception. var textInputProbeBlindForTesting = false - /// Simulates a lit-panel window list the XCTest runtime cannot answer. - var litPanelWindowListUnanswerableForTesting = false + /// Records an XCTest issue during the lit-panel window walk, as a window that vanishes mid-walk does. + var litPanelWindowWalkIssueForTesting = false static let injectedTapFailureFlagPathForTesting = "/tmp/agent-device-inject-tap-recorded-failure-for-testing" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 55d8ad6f45..449875273c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -98,7 +98,7 @@ extension RunnerTests { @MainActor private func tearDownLitPanelApp() { - litPanelWindowListUnanswerableForTesting = false + litPanelWindowWalkIssueForTesting = false setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() @@ -133,11 +133,11 @@ extension RunnerTests { XCTAssertFalse(fellBack, "no read in the request may fall back to the app") } - /// The app's window is booked on the lit panel, and then its window list stops answering: an - /// element read refuses with the typed code instead of finding nothing, and a coordinate `type` - /// refuses before it taps a point with no target. + /// The app's window is booked on the lit panel, and then a window vanishes during the element + /// search's walk: the issue is held, the read refuses with the typed code instead of finding + /// nothing, and a coordinate `type` refuses before it taps a point with no target. @MainActor - func testElementReadsRefuseWhenTheLitPanelWindowListIsUnanswerable() throws { + func testElementReadsRefuseWhenTheLitPanelWindowWalkFails() throws { let (panel, frame) = launchAppOnTheLitPanel() defer { tearDownLitPanelApp() } for json in [ @@ -147,13 +147,42 @@ extension RunnerTests { let command = try runnerCommandFixture(json) setRequestLitPanel(from: command) XCTAssertNil(bookRequestLitPanelSurface(app: app), "the window is on the lit panel") - litPanelWindowListUnanswerableForTesting = true + litPanelWindowWalkIssueForTesting = true let response = try executeOnMainPrepared(command: command, activeApp: app) - litPanelWindowListUnanswerableForTesting = false + litPanelWindowWalkIssueForTesting = false XCTAssertEqual(response.error?.code, "APP_SCREEN_UNRESOLVED", json) } } + /// A bound input re-read at its refresh point answers no input when the lit-panel walk fails: the + /// app-wide identifier query after it could name the dark panel's twin. + @MainActor + func testBoundInputReReadAnswersNoInputWhenTheLitPanelWalkFails() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + let (panel, _) = launchAppOnTheLitPanel() + defer { + app.launchArguments = [] + tearDownLitPanelApp() + } + let identifier = "agent-device-hardware-keyboard-input" + let field = app.textFields[identifier] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let frame = field.frame + let target = TextEntryTarget( + element: nil, + refreshPoint: CGPoint(x: frame.midX, y: frame.midY), + prefersFocusedElement: false, + contentTranslation: .none, + boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: identifier) + ) + let command = try runnerCommandFixture(#"{"command":"type",\#(panel)}"#) + setRequestLitPanel(from: command) + XCTAssertNotNil(resolveTextEntryInput(app: app, target: target), "the walk answers") + setRequestLitPanel(from: command) + litPanelWindowWalkIssueForTesting = true + XCTAssertNil(resolveTextEntryInput(app: app, target: target)) + } + /// With a lit panel named, selector and point reads search only the windows on that display; the /// launched app's windows are all on another one, so neither read may answer. @MainActor diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 51bdc0cb9d..35fcc3dd8b 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -11,10 +11,8 @@ const PANEL_TABLE_UNREADABLE_HINT = /** * The lit panel a runner request names, read fresh per request because a fold changes it (ADR - * 0025). The read never outlasts the request's `timeoutMs`; a read that spends all of it is - * refused like an unread table, so the runner never gets a request with no time left. Undefined - * keeps the runner's historic first-window route, which only a device not known to have more than - * one panel may take. + * 0025). A read that spends the request's `timeoutMs` is refused like an unread table. Undefined + * keeps the historic first-window route, for a device not known to have more than one panel. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo, From 7e79b03976227bcd5ca0e9be44a78aed790c2017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 23:10:32 +0100 Subject: [PATCH 14/29] refactor(ios): drop the single-caller textInputAt and trim lit-panel comments Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+CommandExecution.swift | 3 +-- .../RunnerTests+TextEntryFocus.swift | 2 +- .../RunnerTests+TextInputProbe.swift | 7 +------ .../AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h | 6 ++---- .../UnitTests/RunnerTests+TextEntryLatencyTests.swift | 2 +- .../UnitTests/RunnerTests+TextInputProbeTests.swift | 2 +- 6 files changed, 7 insertions(+), 15 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index d94291f2f9..d9f1a6ed85 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -2,8 +2,7 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { - /// Runs `command` against its prepared app. An element read that cannot search the lit panel - /// throws its refusal, which answers the request (ADR 0025). + /// Runs `command` against its prepared app; a thrown lit-panel refusal answers it (ADR 0025). @MainActor func executeOnMainPrepared( command: Command, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 86c4d8d85d..2a370d49bb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -136,7 +136,7 @@ extension RunnerTests { } let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = try textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) + let input = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 87bd015de4..33a2c6ff3d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -20,10 +20,6 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double, contentTranslation: TextEntryContentTranslation) throws -> ResolvedTextEntryInput? { - try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first - } - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) throws -> [ResolvedTextEntryInput] { let queries = try textInputQueries(app: app) return safely("TEXT_INPUT_AT_POINT", []) { @@ -173,8 +169,7 @@ extension RunnerTests { return true } - /// The text-input queries of a read at a point. They are resolved before the read's issue - /// containment, so a lit-panel refusal answers the request instead of reading as no input. + /// Resolved before the probe's issue containment, so a lit-panel refusal is not read as no input. private func textInputQueries(app: XCUIApplication) throws -> [XCUIElementQuery] { let inputTypes: [XCUIElement.ElementType] = [.textField, .secureTextField, .searchField, .textView] return try inputTypes.flatMap { try elementQueries(app: app, matching: $0) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index 6eeba802c7..a121261687 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -77,10 +77,8 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( RunnerApplicationScreenFailure *failure ); -// The application's windows that report `litPanel`'s display, in list order. A window that is gone -// or names no display is left out, because it may be on the dark panel. Element reads on a -// multi-panel request search only under these windows. Returns nil when the window list itself -// cannot be read, which is not the same answer as no window on the lit display. +// The application's windows that report `litPanel`'s display, in list order; a window that is gone +// or names no display is left out. Nil when the window list itself cannot be read. FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); // `RunnerResolveApplicationScreen` for a multi-panel device: the window comes from diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 78fc0037ab..9f2b062abd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -83,7 +83,7 @@ extension RunnerTests { } let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false, contentTranslation: .none) - let (lookup, lookupReads) = try countingReads { try textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) } + let (lookup, lookupReads) = try countingReads { try textInputCandidatesAt(app: app, point: point, contentTranslation: .none).first } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index 0edffaf660..1a618c3621 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -174,7 +174,7 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = try? textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) + _ = try? textInputCandidatesAt(app: app, point: point, contentTranslation: .none) _ = try? probeTextInputs(app: app, point: point, contentTranslation: .none) record(expected) } From 3e09282fdfe1c0fea4fdbb4b060200cbb9dc3f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 01:09:24 +0100 Subject: [PATCH 15/29] fix(ios): route every lit-panel text-entry read through the element search The bound-input identifier query and the in-app back control searched the whole application, so on an unfolded Duo a same-identifier input or back button on the dark panel could answer. Both now search the element search's roots, and its refusal propagates: text-entry resolution throws instead of answering nil, and boundTextEntryInputIsGone answers "cannot prove gone". One owner returns the request's remaining budget, so the panel read and the runner timeout share one clock. The unreachable hasTappableFrame guard is removed. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+CommandExecution.swift | 2 +- .../RunnerTests+Interaction.swift | 5 +- .../RunnerTests+Navigation.swift | 10 ++- ...unnerTests+SynthesizedCommitDeadline.swift | 10 +-- .../RunnerTests+SynthesizedTextEntry.swift | 24 ++--- .../RunnerTests+TextEntry.swift | 89 ++++++++++--------- .../RunnerTests+TextEntryFocus.swift | 4 +- .../RunnerTests+TextEntryReadiness.swift | 6 +- .../RunnerTests+TextInputProbe.swift | 3 +- .../RunnerTests+TextTyping.swift | 68 +++++++------- .../UnitTests/RunnerRequestPanelTests.swift | 47 ++++++++-- .../RunnerTests+NavigationTests.swift | 14 +-- .../RunnerTests+TextEntryLatencyTests.swift | 2 +- .../RunnerTests+TextEntryPolicyTests.swift | 2 +- .../platform-apple/src/core/runner-client.ts | 15 +--- .../src/foldable/runner-panel.ts | 60 +++++++++---- 16 files changed, 209 insertions(+), 152 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index d9f1a6ed85..40d0d1792c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -548,7 +548,7 @@ extension RunnerTests { ) #endif case .backInApp: - switch tapInAppBackControl(app: activeApp) { + switch try tapInAppBackControl(app: activeApp) { case .performed: return Response(ok: true, data: DataPayload(message: "backInApp")) case .unavailable: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index b6191bf1fd..1ce69bcf25 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -235,16 +235,13 @@ extension RunnerTests { // with the TS twin). app.frame is the frame source here — replay taps // resolved bounds Maestro-style, so the union frame is intentional. A // lit-panel request uses its booked window, because app.frame names one - // panel, and refuses when no window is booked (ADR 0025). + // panel (ADR 0025). private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { let frame = element.frame if frame.isEmpty { return false } let windowFrame = requestLitPanel == nil ? app.frame : onScreenWindowFrame(app: app) - if requestLitPanel != nil, windowFrame.isNull { - return false - } return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: windowFrame) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index db1f9f9a94..c40d4c7562 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -34,7 +34,7 @@ extension RunnerTests { } @MainActor - func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome { + func tapInAppBackControl(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) if let back = macOSNavigationBackElement(app: app) { tapElementCenter(app: app, element: back) @@ -57,7 +57,7 @@ extension RunnerTests { || isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) { NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", mainOwned.bundleId ?? "") - } else if let frame = topNavigationBackFrame(app: app) { + } else if let frame = try topNavigationBackFrame(app: app) { return tapResolvedNavigationBackControl(app: app, frame: frame) } return tapTopLeadingNavigationFallback( @@ -112,10 +112,12 @@ extension RunnerTests { } } - private func topNavigationBackFrame(app: XCUIApplication) -> CGRect? { + private func topNavigationBackFrame(app: XCUIApplication) throws -> CGRect? { #if os(iOS) let window = onScreenWindowFrame(app: app) - let candidates = app.buttons.matching(Self.navigationBackPredicate()).allElementsBoundByIndex.compactMap { + let candidates = try elementQueries(app: app, matching: .button).flatMap { + $0.matching(Self.navigationBackPredicate()).allElementsBoundByIndex + }.compactMap { element -> (frame: CGRect, rank: Int)? in guard element.exists, element.isHittable else { return nil } let frame = element.frame diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift index 8417e035c2..792bc9d0d5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift @@ -73,13 +73,13 @@ extension RunnerTests { app: XCUIApplication, target: TextEntryTarget, expectedText: String - ) -> SynthesizedTextCommitOutcome { + ) throws -> SynthesizedTextCommitOutcome { guard !expectedText.contains("\n"), !expectedText.contains("\r") else { return .unobservable } let waitStartedAt = Date() NSLog("[DEBUG-1874] wait start expectedLen=%ld route=replacement", expectedText.count) - let ingredients = synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) + let ingredients = try synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) let outcome = Self.awaitSynthesizedReplacementCommitOutcome( expectedText: expectedText, placeholder: ingredients.placeholder, @@ -132,13 +132,13 @@ extension RunnerTests { app: XCUIApplication, target: TextEntryTarget, expectedText: String - ) -> ( + ) throws -> ( placeholder: String?, fieldIsUnreadable: Bool, observe: () -> TextValueRead, waitForNextObservation: () -> Void ) { - let field = resolveTextEntryInput(app: app, target: target)?.attributes + let field = try resolveTextEntryInput(app: app, target: target)?.attributes let placeholder = field?.placeholderValue let waitStartedAt = Date() return ( @@ -146,7 +146,7 @@ extension RunnerTests { fieldIsUnreadable: textValueRead(for: field) == .unreadable, observe: { let observed = self.textValueRead( - for: self.resolveTextEntryInput(app: app, target: target)?.attributes, + for: (try? self.resolveTextEntryInput(app: app, target: target))??.attributes, treatingPlaceholderAsEmpty: true ) var observedText: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 3aea84ba1a..18380a7725 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -261,12 +261,12 @@ extension RunnerTests { /// Clears until the field reads empty, at most `maxClearPassCount` passes. A read that cannot /// answer ends the clear, so the post never types over text it could not see removed. static func clearUntilEmpty( - clearOnce: () -> SynthesizedStepDispatch, - read: () -> TextValueRead - ) -> SynthesizedClearOutcome { + clearOnce: () throws -> SynthesizedStepDispatch, + read: () throws -> TextValueRead + ) rethrows -> SynthesizedClearOutcome { for pass in 1...SynthesizedTextPlan.Step.maxClearPassCount { - guard clearOnce() == .posted else { return .stopped } - switch read() { + guard try clearOnce() == .posted else { return .stopped } + switch try read() { case .text(let value) where value.isEmpty: return .cleared case .text: @@ -300,22 +300,22 @@ extension RunnerTests { func runSynthesizedTextPlan( _ plan: SynthesizedTextPlan, text: String, - post: (_ characters: String, _ step: SynthesizedTextPlan.Step) -> SynthesizedStepDispatch, - waitAfterWarmupCharacter: (_ characters: String) -> Void, + post: (_ characters: String, _ step: SynthesizedTextPlan.Step) throws -> SynthesizedStepDispatch, + waitAfterWarmupCharacter: (_ characters: String) throws -> Void, didPostStep: (_ step: SynthesizedTextPlan.Step) -> Void = { _ in } - ) -> SynthesizedPlanRun { + ) rethrows -> SynthesizedPlanRun { let characters = Array(text) var postedCount = 0 for step in plan.steps { let nextCount = postedCount + step.characterCount - if post(String(characters[postedCount.. 0 { sleepFor(step.pauseAfterSeconds) } @@ -355,7 +355,7 @@ extension RunnerTests { var clearOutcome: SynthesizedClearOutcome? // A private synthesis channel that is gone mid-plan leaves the command the same // point-and-focus fallback it had before the plan existed. - let synthesisAvailable = runSynthesizedTextPlan( + let synthesisAvailable = try runSynthesizedTextPlan( plan, text: request.text, post: { slice, step in @@ -426,7 +426,7 @@ extension RunnerTests { // still-in-flight character at all. Wait here, on the same request.target // (element nil, refreshPoint set) that gated this route, so each poll re-resolves via the // refresh point rather than trusting a stale element handle. - let commit = awaitSynthesizedReplacementCommit( + let commit = try awaitSynthesizedReplacementCommit( app: request.app, target: request.target, expectedText: request.text diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 3fc5e3dc2d..258bee27b5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -286,19 +286,19 @@ extension RunnerTests { struct TextEntryReader { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? - private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? + private let resolve: (TextEntryTarget) throws -> ResolvedTextEntryInput? - init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) { + init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) throws -> ResolvedTextEntryInput?) { self.target = target.withResolvedInput(target.postFocusRead) lastRead = target.postFocusRead self.resolve = resolve } - mutating func input() -> ResolvedTextEntryInput? { + mutating func input() throws -> ResolvedTextEntryInput? { if let lastRead { return lastRead } - let read = resolve(target) + let read = try resolve(target) target = target.withResolvedInput(read) lastRead = read return read @@ -390,13 +390,13 @@ extension RunnerTests { } /// Clears the reader's input. Every post goes through `reader.dispatch` and every read back through - /// `reader.input()`, so a bound target's clear stops at an input that took its place. On iOS a + /// `try reader.input()`, so a bound target's clear stops at an input that took its place. On iOS a /// readable value is selected whole and deleted, wherever the caret is. Otherwise, or when text is /// left, delete bursts run from the caret, and a value longer than one burst is read again after /// each burst until it reads empty, stops shortening, or has had at least one burst more than it /// needs. - func clearTextInput(app: XCUIApplication, reader: inout TextEntryReader) { - guard let input = reader.input() else { + func clearTextInput(app: XCUIApplication, reader: inout TextEntryReader) throws { + guard let input = try reader.input() else { return } // Skip the clear (edge-tap, select all, delete bursts) ONLY when we can confirm the @@ -418,7 +418,7 @@ extension RunnerTests { reader.dispatch { moveCaretToEnd(app: app, input: input) } #endif // An input that stops answering once focused (#3060) cannot be read back, so it gets one burst. - guard reader.input() != nil else { + guard try reader.input() != nil else { let burst = Self.textClearBurst(valueLength: valueLength) reader.dispatch { input.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } return @@ -430,9 +430,9 @@ extension RunnerTests { // bursts alone. if existing != nil { var passes = 0 - let outcome = Self.clearUntilEmpty( + let outcome = try Self.clearUntilEmpty( clearOnce: { - guard let input = reader.input() else { + guard let input = try reader.input() else { return .stop } passes += 1 @@ -443,13 +443,13 @@ extension RunnerTests { return .posted }, read: { - textValueRead(for: reader.input()?.attributes, treatingPlaceholderAsEmpty: true) + textValueRead(for: try reader.input()?.attributes, treatingPlaceholderAsEmpty: true) } ) NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR route=element passes=%d outcome=%@", passes, String(describing: outcome)) // Text left after every pass goes to the bursts; an input that no longer resolves is not typed into. guard outcome == .notCleared, - let remaining = reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }) + let remaining = try reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }) else { return } @@ -457,14 +457,14 @@ extension RunnerTests { } #endif for _ in 0...(valueLength / Self.maxTextClearBurst + 1) { - guard let input = reader.input() else { + guard let input = try reader.input() else { return } let burst = Self.textClearBurst(valueLength: valueLength) reader.dispatch { input.element.typeText(String(repeating: XCUIKeyboardKey.delete.rawValue, count: burst)) } // A burst that covers the value needs no read. guard burst < valueLength + Self.textClearSlack, - let remaining = reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }), + let remaining = try reader.input().flatMap({ editableTextValue(for: $0.attributes, treatingPlaceholderAsEmpty: true) }), Self.textClearContinues(clearedLength: valueLength, remainingLength: remaining.count) else { return @@ -516,11 +516,11 @@ extension RunnerTests { /// Resolves the target's input, with the snapshot the resolution read it through. A bound target /// accepts only an input whose snapshot carries its bound identity, so every post, read-back, - /// verification poll, and repair refuses an input that took the bound one's place. A lit-panel - /// refusal at the refresh point answers no input, never the app-wide reads after it. - func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { + /// verification poll, and repair refuses an input that took the bound one's place. Every search runs + /// under the element search's roots, so a lit-panel refusal propagates. + func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) throws -> ResolvedTextEntryInput? { guard let boundIdentity = target.boundIdentity else { - return resolveUnboundTextEntryInput(app: app, target: target) + return try resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { guard let input = ResolvedTextEntryInput(reading: candidate, contentTranslation: target.contentTranslation), @@ -537,18 +537,16 @@ extension RunnerTests { let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } - if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint { - guard let probe = try? probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) else { - return nil - } - if case .matches(let candidates) = probe, let match = candidates.first(where: { $0.identity == boundIdentity }) { - return match - } + if target.inputAtRefreshPoint == nil, + let refreshPoint = target.refreshPoint, + case .matches(let candidates) = try probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), + let match = candidates.first(where: { $0.identity == boundIdentity }) { + return match } if let focused = focusedTextInput(app: app), let input = boundInput(focused) { return input } - if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity), + if let byIdentifier = try boundTextEntryInputQuery(app: app, identity: boundIdentity), let input = boundInput(byIdentifier) { return input } @@ -557,12 +555,19 @@ extension RunnerTests { /// Whether a bound target's input is proven gone: XCTest found no match for it, or a different /// input answers in its place, and nothing at its point carries its identity. A failed probe - /// proves nothing. Unbound targets and unidentified inputs without an element cannot be proven - /// gone. + /// proves nothing, and neither does a lit-panel refusal. Unbound targets and unidentified inputs + /// without an element cannot be proven gone. func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) -> Bool { - guard let boundIdentity = target.boundIdentity, - let query = boundTextEntryInputQuery(app: app, identity: boundIdentity) ?? target.element - else { + guard let boundIdentity = target.boundIdentity else { + return false + } + let byIdentifier: XCUIElement? + do { + byIdentifier = try boundTextEntryInputQuery(app: app, identity: boundIdentity) + } catch { + return false + } + guard let query = byIdentifier ?? target.element else { return false } switch probeTextEntryInput(query) { @@ -586,12 +591,16 @@ extension RunnerTests { } } - /// The app-wide query for an input whose identifier is unique enough to search by. - private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) -> XCUIElement? { + /// The query, under the element search's roots, for an input whose identifier is unique enough + /// to search by. Of several lit windows, the first one holding a match answers. + private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) throws -> XCUIElement? { guard identity.isDistinguishable else { return nil } - return app.descendants(matching: identity.elementType).matching(identifier: identity.identifier).element + let queries = try elementQueries(app: app, matching: identity.elementType).map { + $0.matching(identifier: identity.identifier) + } + return (queries.count == 1 ? queries.first : queries.first { $0.count > 0 })?.element } /// Snapshots one candidate: its identity, a proven no-match, or a failure that proves nothing. @@ -609,7 +618,7 @@ extension RunnerTests { } /// Resolution for a target not bound to an input yet: the first element that exists. - private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { + private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) throws -> ResolvedTextEntryInput? { let read = { (element: XCUIElement) in ResolvedTextEntryInput(reading: element, contentTranslation: target.contentTranslation) } @@ -619,13 +628,9 @@ extension RunnerTests { if let element = target.element, let input = read(element) { return input } - if let refreshPoint = target.refreshPoint { - guard let candidates = try? textInputCandidatesAt(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) else { - return nil - } - if let refreshed = candidates.first { - return refreshed - } + if let refreshPoint = target.refreshPoint, + let refreshed = try textInputCandidatesAt(app: app, point: refreshPoint, contentTranslation: target.contentTranslation).first { + return refreshed } return focusedTextInput(app: app).flatMap(read) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 2a370d49bb..f0e6cbee74 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -123,7 +123,7 @@ extension RunnerTests { contentTranslation: contentTranslation ) } - let focused = waitForTextEntryReadiness( + let focused = try waitForTextEntryReadiness( app: app, target: TextEntryTarget( element: focusedTextInput(app: app), @@ -171,7 +171,7 @@ extension RunnerTests { ) let element = stabilized.focusConfirmed ? readyTarget.element - : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) + : (try waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index b30d584165..97b3beb889 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -30,9 +30,9 @@ extension RunnerTests { app: XCUIApplication, target: TextEntryTarget, timeout: TimeInterval = TextEntryTiming.readinessTimeout - ) -> XCUIElement? { + ) throws -> XCUIElement? { #if os(iOS) - let candidate = resolveTextEntryInput(app: app, target: target)?.element + let candidate = try resolveTextEntryInput(app: app, target: target)?.element let keyboardVisibleAtEntry = isKeyboardVisible(app: app) let deadline = Date().addingTimeInterval(timeout) var hardwareKeyboardFallback = Date().addingTimeInterval( @@ -67,7 +67,7 @@ extension RunnerTests { } return candidate #else - return resolveTextEntryInput(app: app, target: target)?.element + return try resolveTextEntryInput(app: app, target: target)?.element #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 33a2c6ff3d..bf2464f43f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -84,7 +84,8 @@ extension RunnerTests { /// containment, with a placeholder read as empty. A read that cannot answer is `.unavailable`. func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) -> TextValueRead { let probed = containingTextInputProbeIssues(fallback: TextValueRead.unavailable) { _ in - textValueRead(for: resolveTextEntryInput(app: app, target: target)?.attributes, treatingPlaceholderAsEmpty: true) + guard let input = try? resolveTextEntryInput(app: app, target: target) else { return .unavailable } + return textValueRead(for: input.attributes, treatingPlaceholderAsEmpty: true) } guard case .success(let read) = probed else { return .unavailable } return read diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index b2a26b45d3..0adae35fb3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -15,7 +15,7 @@ extension RunnerTests { commandId: String? = nil ) throws -> TextEntryResult { let totalStartedAt = Date() - let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } + let resolve = { (target: TextEntryTarget) in try self.resolveTextEntryInput(app: app, target: target) } var reader = TextEntryReader(target: target, resolve: resolve) guard !text.isEmpty else { // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing @@ -23,7 +23,7 @@ extension RunnerTests { // place while the response reported "typed". Append (`type`) and unrepaired entry stay // no-ops, which is what an empty payload means for them. if repairMode == .replacement { - guard reader.input() != nil else { + guard try reader.input() != nil else { // No resolvable input means no clear happened — including the synthesized // first-responder route, whose target carries no element. Success here would claim a // clear the runner never performed. @@ -36,10 +36,10 @@ extension RunnerTests { failure: .notFocused ) } - clearTextInput(app: app, reader: &reader) + try clearTextInput(app: app, reader: &reader) // nil means the value is unreadable (secure fields), not "not empty": leave `verified` // nil there rather than reporting a mismatch the runner cannot actually observe. - let observed = editableTextValue(for: reader.input()?.attributes, treatingPlaceholderAsEmpty: true) + let observed = editableTextValue(for: try reader.input()?.attributes, treatingPlaceholderAsEmpty: true) logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode) return TextEntryResult( verified: observed.map { $0.isEmpty }, @@ -84,7 +84,7 @@ extension RunnerTests { softwareKeyboardVisible: isKeyboardVisible(app: app) ) let initialResolveStartedAt = Date() - let initialTarget = reader.input() + let initialTarget = try reader.input() let currentText = editableTextValue(for: initialTarget?.attributes, treatingPlaceholderAsEmpty: true) let initialText = repairMode == .append ? currentText : nil let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText) @@ -125,7 +125,7 @@ extension RunnerTests { } if currentText == nil || currentText?.isEmpty == false { let clearStartedAt = Date() - clearTextInput(app: app, reader: &reader) + try clearTextInput(app: app, reader: &reader) logTextEntryPhase( commandId: commandId, phase: "clear", @@ -134,7 +134,7 @@ extension RunnerTests { mode: repairMode ) } - entryBaseline = textEntryObservation(for: reader.input()) + entryBaseline = textEntryObservation(for: try reader.input()) } // The shape this command posts, decided once: a spaced per-character plan, a peeled warmup plus @@ -170,10 +170,10 @@ extension RunnerTests { return nil } - func typeIntoCurrentTarget(_ value: String) -> TextEntryFailure? { + func typeIntoCurrentTarget(_ value: String) throws -> TextEntryFailure? { #if os(iOS) if shouldUseSynthesizedFirstResponderType { - guard reader.input() != nil else { + guard try reader.input() != nil else { return .notFocused } textEntryRoute = "synthesized-first-responder" @@ -191,7 +191,7 @@ extension RunnerTests { } } #endif - if let currentTarget = reader.input() { + if let currentTarget = try reader.input() { textEntryRoute = "xctest-element" reader.dispatch { currentTarget.element.typeText(value) } return nil @@ -243,7 +243,7 @@ extension RunnerTests { reader.dispatch { sleepFor(TextEntryTiming.pollInterval) } } - func readBackWarmupCharacter(_ peeledCharacter: String) { + func readBackWarmupCharacter(_ peeledCharacter: String) throws { // A route with an element reads the peeled character back and waits up to `warmupValueTimeout` // for the app to accept it. The route that had to be budgeted has no element, so its expected // value is nil and this is the plan's one charged poll. @@ -258,7 +258,7 @@ extension RunnerTests { } let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout) while Date() < deadline { - let observed = textValueRead(for: reader.input()?.attributes) + let observed = textValueRead(for: try reader.input()?.attributes) if !observed.warmupReadIsPending(expectedText: warmupExpectedText) { if observed == .unreadable { pauseAfterWarmupRead() } return @@ -273,12 +273,12 @@ extension RunnerTests { var planFailure: TextEntryResult? var stepStartedAt = Date() let pacedStartedAt = Date() - runSynthesizedTextPlan( + try runSynthesizedTextPlan( synthesizedTypePlan, text: text, post: { slice, _ in stepStartedAt = Date() - if let failure = typeIntoCurrentTarget(slice) { + if let failure = try typeIntoCurrentTarget(slice) { planFailure = dispatchFailureResult(failure) return .stop } @@ -286,7 +286,7 @@ extension RunnerTests { }, waitAfterWarmupCharacter: { peeledCharacter in let warmupStartedAt = Date() - readBackWarmupCharacter(peeledCharacter) + try readBackWarmupCharacter(peeledCharacter) logTextEntryPhase( commandId: commandId, phase: "warmup", @@ -344,9 +344,9 @@ extension RunnerTests { let verifyStartedAt = Date() var result: TextEntryResult if repairMode == .replacement { - result = observeReplacementTextEntry(app: app, reader: &reader, requested: text, baseline: entryBaseline) + result = try observeReplacementTextEntry(app: app, reader: &reader, requested: text, baseline: entryBaseline) } else { - result = verifyAppendTextEntryWithRepairIfNeeded(app: app, reader: &reader, expectedText: expectedText) + result = try verifyAppendTextEntryWithRepairIfNeeded(app: app, reader: &reader, expectedText: expectedText) } logTextEntryPhase( commandId: commandId, @@ -393,10 +393,10 @@ extension RunnerTests { reader: inout TextEntryReader, requested: String, baseline: TextEntryObservation? - ) -> TextEntryResult { + ) throws -> TextEntryResult { var confirmation = ReplacementTextEntryConfirmation(requested: requested, baseline: baseline, startedAt: Date()) while true { - guard let input = reader.input() else { + guard let input = try reader.input() else { let target = reader.target let failure: TextEntryFailure? = target.boundIdentity != nil && !boundTextEntryInputIsGone(app: app, target: target) ? .commitNotObserved : nil @@ -422,8 +422,8 @@ extension RunnerTests { app: XCUIApplication, reader: inout TextEntryReader, expectedText: String? - ) -> TextEntryResult { - if reader.input() == nil, reader.target.boundIdentity != nil { + ) throws -> TextEntryResult { + if try reader.input() == nil, reader.target.boundIdentity != nil { guard boundTextEntryInputIsGone(app: app, target: reader.target) else { return TextEntryResult( verified: nil, @@ -438,7 +438,7 @@ extension RunnerTests { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil) } - let initialResult = verifyTextEntry( + let initialResult = try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: false @@ -449,11 +449,11 @@ extension RunnerTests { else { return initialResult } - guard shouldRepairAppendTextEntry( + guard try shouldRepairAppendTextEntry( reader: &reader, expectedText: expectedText ) else { - return verifyTextEntry( + return try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: false @@ -461,14 +461,14 @@ extension RunnerTests { } guard Self.textEntryRepairCanTarget(boundIdentity: reader.target.boundIdentity) else { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_REPAIR_REFUSED reason=unidentified-input") - return verifyTextEntry( + return try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: false ) } - guard let repairTarget = reader.input() else { + guard let repairTarget = try reader.input() else { return initialResult } let observedText = editableTextValue(for: repairTarget.attributes) ?? "" @@ -477,9 +477,9 @@ extension RunnerTests { expectedText.count, observedText.count ) - clearTextInput(app: app, reader: &reader) + try clearTextInput(app: app, reader: &reader) // An input the clear removed or let another take the place of has nowhere for the repair to go. - guard let input = reader.input() else { + guard let input = try reader.input() else { return TextEntryResult( verified: nil, repaired: false, @@ -489,7 +489,7 @@ extension RunnerTests { ) } reader.dispatch { input.element.typeText(expectedText) } - return verifyTextEntry( + return try verifyTextEntry( reader: &reader, expectedText: expectedText, repaired: true @@ -503,8 +503,8 @@ extension RunnerTests { reader: inout TextEntryReader, expectedText: String?, repaired: Bool - ) -> TextEntryResult { - let targetInput = reader.input() + ) throws -> TextEntryResult { + let targetInput = try reader.input() guard let expectedText else { return TextEntryResult( verified: nil, @@ -532,7 +532,7 @@ extension RunnerTests { var latestObservedText = observedText while Date() < stableDeadline { reader.dispatch { sleepFor(TextEntryTiming.pollInterval) } - let nextInput = reader.input() + let nextInput = try reader.input() guard let nextObservedText = editableTextValue(for: nextInput?.attributes) else { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } @@ -593,13 +593,13 @@ extension RunnerTests { private func shouldRepairAppendTextEntry( reader: inout TextEntryReader, expectedText: String - ) -> Bool { + ) throws -> Bool { #if os(iOS) var latestObservedText: String? var latestElementType: XCUIElement.ElementType? let deadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow) repeat { - let input = reader.input() + let input = try reader.input() guard let observedText = editableTextValue(for: input?.attributes) else { return false } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 449875273c..35587435b3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -154,10 +154,10 @@ extension RunnerTests { } } - /// A bound input re-read at its refresh point answers no input when the lit-panel walk fails: the - /// app-wide identifier query after it could name the dark panel's twin. + /// A bound input re-read on a lit-panel request whose walk fails propagates the refusal, and the + /// input cannot be proven gone. @MainActor - func testBoundInputReReadAnswersNoInputWhenTheLitPanelWalkFails() throws { + func testBoundInputReReadRefusesWhenTheLitPanelWalkFails() throws { app.launchArguments = ["--agent-device-text-entry-regression"] let (panel, _) = launchAppOnTheLitPanel() defer { @@ -177,10 +177,47 @@ extension RunnerTests { ) let command = try runnerCommandFixture(#"{"command":"type",\#(panel)}"#) setRequestLitPanel(from: command) - XCTAssertNotNil(resolveTextEntryInput(app: app, target: target), "the walk answers") + XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the walk answers") setRequestLitPanel(from: command) litPanelWindowWalkIssueForTesting = true - XCTAssertNil(resolveTextEntryInput(app: app, target: target)) + XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertFalse(boundTextEntryInputIsGone(app: app, target: target)) + } + + /// The walk answers but finds no input at the refresh point, so resolution reaches the + /// identifier query: on a lit-panel request it searches only the lit windows, so a field the + /// app-wide query finds, standing in for the dark panel's twin, is not resolved. + @MainActor + func testBoundInputIdentifierQuerySearchesOnlyTheLitWindows() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + app.launch() + mainOwned.app = app + defer { + app.launchArguments = [] + tearDownLitPanelApp() + } + let identifier = "agent-device-hardware-keyboard-input" + let field = app.textFields[identifier] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let frame = field.frame + let target = TextEntryTarget( + element: nil, + refreshPoint: CGPoint(x: frame.midX, y: frame.midY), + prefersFocusedElement: false, + contentTranslation: .none, + boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: identifier) + ) + setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"type"}"#)) + XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the app-wide query finds it") + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"type","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + guard case .absent = try probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) else { + return XCTFail("the walk answers with no input at the point") + } + XCTAssertNil(try resolveTextEntryInput(app: app, target: target)) } /// With a lit panel named, selector and point reads search only the windows on that display; the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index f881f622e4..1f72efc51d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -22,7 +22,7 @@ extension RunnerTests { let status = try launchBackFixture(["--agent-device-back-regression"]) defer { endBackFixture() } - let outcome = try withRejectedAppTree(framesReadable: false) { tapInAppBackControl(app: app) } + let outcome = try withRejectedAppTree(framesReadable: false) { try tapInAppBackControl(app: app) } guard soleScreenRouteIsOpen(for: app) else { guard case .unavailable = outcome else { @@ -43,7 +43,7 @@ extension RunnerTests { _ = try launchBackFixture([]) defer { endBackFixture() } - let outcome = try withRejectedAppTree(framesReadable: false) { tapInAppBackControl(app: app) } + let outcome = try withRejectedAppTree(framesReadable: false) { try tapInAppBackControl(app: app) } guard case .unavailable = outcome else { return XCTFail("a top-leading tap on an app without a back control must not read as performed") @@ -61,7 +61,7 @@ extension RunnerTests { defer { endBackFixture() } let outcome = try withRejectedAppTree(framesReadable: true, rejections: 1) { - tapInAppBackControl(app: app) + try tapInAppBackControl(app: app) } guard case .performed = outcome else { @@ -94,7 +94,7 @@ extension RunnerTests { ) let outcome = try withRejectedAppTree(framesReadable: true, rejections: 1) { - tapInAppBackControl(app: app) + try tapInAppBackControl(app: app) } guard case .performed = outcome else { return XCTFail("a refused sole-screen route must still dispatch back through the resolved window") @@ -156,7 +156,7 @@ extension RunnerTests { private func withRejectedAppTree( framesReadable: Bool, rejections: Int = .max, - _ body: () -> T + _ body: () throws -> T ) throws -> T { var restorations: [() -> Void] = [] defer { restorations.reversed().forEach { $0() } } @@ -188,7 +188,7 @@ extension RunnerTests { imp_removeBlock(unavailableFrame) } } - return body() + return try body() } @MainActor @@ -217,7 +217,7 @@ extension RunnerTests { } } - guard case .performed = tapInAppBackControl(app: app) else { + guard case .performed = try tapInAppBackControl(app: app) else { return XCTFail("the selected back control must dispatch") } let changed = expectation(for: NSPredicate(format: "label == %@", "Back performed"), evaluatedWith: status) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 9f2b062abd..19e4cdd194 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -84,7 +84,7 @@ extension RunnerTests { let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false, contentTranslation: .none) let (lookup, lookupReads) = try countingReads { try textInputCandidatesAt(app: app, point: point, contentTranslation: .none).first } - let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } + let (input, resolutionReads) = try countingReads { try resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") XCTAssertEqual(input?.snapshot?.identifier, "agent-device-hardware-keyboard-input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 00fb6fe970..b01c2ebe1f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -830,7 +830,7 @@ extension RunnerTests { func testSynthesizedReplacementCommitSkipsSubmitKeyText() { for expectedText in ["ada@example.test\n", "ada@example.test\r"] { XCTAssertEqual( - awaitSynthesizedReplacementCommit( + try awaitSynthesizedReplacementCommit( app: XCUIApplication(), target: TextEntryTarget(element: nil, refreshPoint: CGPoint(x: 10, y: 20), prefersFocusedElement: false, contentTranslation: .none), expectedText: expectedText diff --git a/packages/platform-apple/src/core/runner-client.ts b/packages/platform-apple/src/core/runner-client.ts index 0e0982812c..3ed071c725 100644 --- a/packages/platform-apple/src/core/runner-client.ts +++ b/packages/platform-apple/src/core/runner-client.ts @@ -32,21 +32,10 @@ export async function runAppleRunnerCommand( if (!runnerCommandResolvesSurface(command)) { return await runHostBoundAppleRunnerCommand(device, command, options); } - const { resolveRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); - const startedAt = Date.now(); - const litPanel = await resolveRunnerRequestLitPanel(device, { - signal: resolveRunnerRequestSignal(options), - requestTimeoutMs: command.timeoutMs, - }); - if (!litPanel) return await runHostBoundAppleRunnerCommand(device, command, options); - // The runner gets what the panel read left of the request's budget. - const timeoutMs = - command.timeoutMs === undefined - ? undefined - : Math.max(1, command.timeoutMs - (Date.now() - startedAt)); + const { nameRunnerRequestLitPanel } = await import('../foldable/runner-panel.ts'); return await runHostBoundAppleRunnerCommand( device, - timeoutMs === undefined ? { ...command, litPanel } : { ...command, litPanel, timeoutMs }, + await nameRunnerRequestLitPanel(device, command, resolveRunnerRequestSignal(options)), options, ); } diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 35fcc3dd8b..53c47dd433 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -9,16 +9,26 @@ import { readFoldablePanelTable } from './orientation.ts'; const PANEL_TABLE_UNREADABLE_HINT = 'The panel table is unreadable, so no lit panel can be named and nothing was sent. Retry; if it persists, run xcrun devicectl device info displays --device .'; +/** Refuses a request whose lit panel cannot be named; nothing was sent. */ +function panelTableRefusal(device: DeviceInfo, message: string): AppError { + return new AppError('COMMAND_FAILED', message, { + deviceId: device.id, + reason: 'panel-table-unreadable', + runnerErrorCode: 'APP_SCREEN_UNRESOLVED', + dispatched: 'no', + hint: PANEL_TABLE_UNREADABLE_HINT, + }); +} + /** * The lit panel a runner request names, read fresh per request because a fold changes it (ADR - * 0025). A read that spends the request's `timeoutMs` is refused like an unread table. Undefined - * keeps the historic first-window route, for a device not known to have more than one panel. + * 0025). The read never outlasts the request's `timeoutMs`. Undefined keeps the historic + * first-window route, for a device not known to have more than one panel. */ export async function resolveRunnerRequestLitPanel( device: DeviceInfo, options: { signal?: AbortSignal; requestTimeoutMs?: number }, ): Promise { - const startedAt = Date.now(); const read = await readFoldablePanelTable(device, { signal: options.signal, timeoutMs: Math.min( @@ -26,21 +36,37 @@ export async function resolveRunnerRequestLitPanel( IOS_APPLE_DISPLAY_PROBE_TIMEOUT_MS, ), }); - if (read.kind === 'single-panel') return undefined; - const spentRequest = - options.requestTimeoutMs !== undefined && Date.now() - startedAt >= options.requestTimeoutMs; - if (read.kind === 'unreadable' || spentRequest) { - throw new AppError( - 'COMMAND_FAILED', + if (read.kind === 'unreadable') { + throw panelTableRefusal( + device, `The panel table of ${device.name} did not answer, so the lit panel cannot be named`, - { - deviceId: device.id, - reason: 'panel-table-unreadable', - runnerErrorCode: 'APP_SCREEN_UNRESOLVED', - dispatched: 'no', - hint: PANEL_TABLE_UNREADABLE_HINT, - }, ); } - return appleLitPanel(read.inventory); + return read.kind === 'multi-panel' ? appleLitPanel(read.inventory) : undefined; +} + +/** + * `command` with the lit panel named and the request budget the read left, measured on one clock. + * A read that leaves no budget is refused before anything is sent. + */ +export async function nameRunnerRequestLitPanel( + device: DeviceInfo, + command: RunnerCommand, + signal: AbortSignal | undefined, +): Promise { + const startedAt = Date.now(); + const litPanel = await resolveRunnerRequestLitPanel(device, { + signal, + requestTimeoutMs: command.timeoutMs, + }); + if (!litPanel) return command; + if (command.timeoutMs === undefined) return { ...command, litPanel }; + const timeoutMs = command.timeoutMs - (Date.now() - startedAt); + if (timeoutMs < 1) { + throw panelTableRefusal( + device, + `Reading the panel table of ${device.name} spent the request's ${command.timeoutMs} ms, so nothing was sent`, + ); + } + return { ...command, litPanel, timeoutMs }; } From f80aeef726ddf5cec7d688b3182bc7c4520b919a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 01:18:33 +0100 Subject: [PATCH 16/29] fix(ios): book the lit panel's whole box for a window that leaves a strip On the unfolded Duo, an app's window is 871x669 beside the inner panel's status strip, and the booking used that window frame as the synthesized dispatch reference. The orientation flip then mapped points against 871 instead of the 951-point screen, so taps landed 80 points off, and a fill's focus tap missed. The booked frame is now the panel's box turned like the window, at its origin, which is unchanged for full-panel windows. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 18 ++++++++++++++++-- .../UnitTests/RunnerRequestPanelTests.swift | 14 ++++++++++++++ docs/adr/0025-foldable-apple-panels.md | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 41236f3842..59ea96c735 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -44,7 +44,9 @@ extension RunnerTests { /// Books `app`'s window on the lit panel for the rest of the request, or returns why it cannot. A /// request that named no panel books nothing. Later surface reads of `app` answer with this - /// window, so none of them can fall back to the application's own coordinates. + /// window, so none of them can fall back to the application's own coordinates. The booked frame + /// is the panel's box turned like the window, at the window's origin: a window that leaves part + /// of the panel uncovered still maps points into the whole screen. func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { guard let litPanel = requestLitPanel else { return nil } if let booked = litPanelState.surface, booked.app === app { return nil } @@ -54,10 +56,22 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - litPanelState.surface = LitPanelSurface(app: app, displayID: litPanel.displayID, window: element, frame: frame) + litPanelState.surface = LitPanelSurface( + app: app, + displayID: litPanel.displayID, + window: element, + frame: Self.litPanelScreenFrame(litPanel, windowFrame: frame) + ) return nil } + static func litPanelScreenFrame(_ panel: RunnerLitPanel, windowFrame: CGRect) -> CGRect { + let long = max(panel.widthPt, panel.heightPt) + let short = min(panel.widthPt, panel.heightPt) + let turned = windowFrame.width > windowFrame.height + return CGRect(origin: windowFrame.origin, size: turned ? CGSize(width: long, height: short) : CGSize(width: short, height: long)) + } + /// Where an element read searches: the application, or on a lit-panel request only the windows /// proven on the lit display. A list that cannot be read throws `APP_SCREEN_UNRESOLVED`. The walk /// runs inside an issue scope, so a window that vanishes mid-walk refuses instead of ending the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 35587435b3..86610635b5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -48,6 +48,20 @@ extension RunnerTests { XCTAssertEqual(queuedLater, 3) } + /// The Duo leaves a status strip beside an app's window on the inner panel; a point must still map + /// into the whole screen, so the booked frame is the panel's box turned like the window. + func testLitPanelBookingMapsIntoTheWholeTurnedPanel() { + let inner = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) + XCTAssertEqual( + Self.litPanelScreenFrame(inner, windowFrame: CGRect(x: 0, y: 0, width: 871, height: 669)), + CGRect(x: 0, y: 0, width: 951, height: 669) + ) + XCTAssertEqual( + Self.litPanelScreenFrame(inner, windowFrame: CGRect(x: 0, y: 0, width: 669, height: 951)), + CGRect(x: 0, y: 0, width: 669, height: 951) + ) + } + func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() throws { func refusal(_ fields: String) throws -> ErrorPayload? { try Self.litPanelBackendPinRefusal(runnerCommandFixture(#"{"command":"snapshot"\#(fields)}"#))?.error diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 3a27396ba4..513b050472 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -732,6 +732,6 @@ Rejected alternatives: Consequences: -- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame, and `SYNTHESIZED_RECORD displayID=` must equal `lit=`. +- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` is the lit panel's box turned like that window, at its origin. A window that leaves a strip of the panel uncovered, like an app beside the inner panel's status strip, still maps points into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. - A recording keeps the panel that its `recordStart` named. The runner cannot read panel power, so it does not notice a fold during the recording: SpringBoard keeps windows on the panel that went dark, and later frames show that dark panel. Start a new recording after a fold. - Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. From 160cd757ccca74dc5d247a6ec6b636fad2d0a45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 01:19:03 +0100 Subject: [PATCH 17/29] docs(ios): tighten the lit-panel booking comment Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerUITests/RunnerRequestPanel.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 59ea96c735..86f06a17e2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -45,8 +45,7 @@ extension RunnerTests { /// Books `app`'s window on the lit panel for the rest of the request, or returns why it cannot. A /// request that named no panel books nothing. Later surface reads of `app` answer with this /// window, so none of them can fall back to the application's own coordinates. The booked frame - /// is the panel's box turned like the window, at the window's origin: a window that leaves part - /// of the panel uncovered still maps points into the whole screen. + /// is the panel's box turned like the window, so a partial window maps into the whole screen. func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { guard let litPanel = requestLitPanel else { return nil } if let booked = litPanelState.surface, booked.app === app { return nil } From 319c2dc2dddcbecf082d72d7d56abb2fb3d0e960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 01:20:55 +0100 Subject: [PATCH 18/29] test(ios): share the bound-input fixture target Co-Authored-By: Claude Opus 5.5 --- .../UnitTests/RunnerRequestPanelTests.swift | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 86610635b5..63e38e5f20 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -168,27 +168,33 @@ extension RunnerTests { } } - /// A bound input re-read on a lit-panel request whose walk fails propagates the refusal, and the - /// input cannot be proven gone. + /// A target bound to the regression fixture's field by identifier, refreshed at its center. @MainActor - func testBoundInputReReadRefusesWhenTheLitPanelWalkFails() throws { - app.launchArguments = ["--agent-device-text-entry-regression"] - let (panel, _) = launchAppOnTheLitPanel() - defer { - app.launchArguments = [] - tearDownLitPanelApp() - } + private func boundFixtureTarget() throws -> TextEntryTarget { let identifier = "agent-device-hardware-keyboard-input" let field = app.textFields[identifier] XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame - let target = TextEntryTarget( + return TextEntryTarget( element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, contentTranslation: .none, boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: identifier) ) + } + + /// A bound input re-read on a lit-panel request whose walk fails propagates the refusal, and the + /// input cannot be proven gone. + @MainActor + func testBoundInputReReadRefusesWhenTheLitPanelWalkFails() throws { + app.launchArguments = ["--agent-device-text-entry-regression"] + let (panel, _) = launchAppOnTheLitPanel() + defer { + app.launchArguments = [] + tearDownLitPanelApp() + } + let target = try boundFixtureTarget() let command = try runnerCommandFixture(#"{"command":"type",\#(panel)}"#) setRequestLitPanel(from: command) XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the walk answers") @@ -212,23 +218,15 @@ extension RunnerTests { app.launchArguments = [] tearDownLitPanelApp() } - let identifier = "agent-device-hardware-keyboard-input" - let field = app.textFields[identifier] - XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) - let frame = field.frame - let target = TextEntryTarget( - element: nil, - refreshPoint: CGPoint(x: frame.midX, y: frame.midY), - prefersFocusedElement: false, - contentTranslation: .none, - boundIdentity: TextEntryInputIdentity(elementType: .textField, identifier: identifier) - ) + let target = try boundFixtureTarget() setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"type"}"#)) XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the app-wide query finds it") setRequestLitPanel(from: try runnerCommandFixture( #"{"command":"type","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# )) - guard case .absent = try probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) else { + guard let point = target.refreshPoint, + case .absent = try probeTextInputs(app: app, point: point, contentTranslation: .none) + else { return XCTFail("the walk answers with no input at the point") } XCTAssertNil(try resolveTextEntryInput(app: app, target: target)) From a739321c0a80a2f02711f2f3f36c2f43edf868d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 02:30:36 +0100 Subject: [PATCH 19/29] fix(ios): propagate lit-panel refusals from the commit poll and clear reads The synthesized commit wait's observe closure and the replacement clear's read swallowed a lit-panel refusal with `try?`, so the wait timed out or the clear reported TEXT_INPUT_CLEAR_NOT_OBSERVED. Both now throw, and the refusal answers the request. The clear read also keeps main's answer for an input that is simply gone. The bridge exports one lit-panel resolver that returns the window, frame and screen, and lit-panel comments are trimmed. Co-Authored-By: Claude Opus 5.5 --- .../RunnerAppScreenCapture.swift | 11 +++++-- .../RunnerRequestPanel.swift | 21 +++++-------- .../RunnerTests+Snapshot.swift | 9 ++---- ...unnerTests+SynthesizedCommitDeadline.swift | 6 ++-- .../RunnerTests+SynthesizedTextEntry.swift | 16 +++++----- .../RunnerTests+TextEntry.swift | 5 ++-- .../RunnerTests+TextInputProbe.swift | 15 +++++++--- .../RunnerXCTestEventBridge.h | 19 ++++-------- .../RunnerXCTestEventBridge.m | 30 +------------------ .../UnitTests/RunnerRequestPanelTests.swift | 1 + .../RunnerTests+XCTestEventBridgeTests.swift | 25 ++++++---------- 11 files changed, 59 insertions(+), 99 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index fa81a85bf4..dff4f67bb5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -140,10 +140,15 @@ extension RunnerTests { var screen: AnyObject? var displayID: UInt = 0 var resolution: RunnerApplicationScreenFailure = .unresolvedScreen - let resolved = if let litPanel { - RunnerResolveLitPanelScreen(app, litPanel, &screen, &displayID, &resolution) + var resolved: Bool + if let litPanel { + var window: AnyObject? + var frame = CGRect.null + resolution = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame, &screen) + resolved = resolution == .none + displayID = litPanel.displayID } else { - RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution) + resolved = RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution) } guard resolved, let captureScreen = screen as? XCUIScreen diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 86f06a17e2..eaf4b7afe5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -1,9 +1,7 @@ import XCTest -// The lit panel of the command in flight (ADR 0025). It is main-owned, and `executeAccepted` writes -// it onto the main queue ahead of the command's own main-thread work. Main work a previous command -// abandoned therefore runs first and still reads that command's panel. A recording pump outlives -// its command, so `recordStart` reads the panel once and carries it. +// The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a +// previous command left queued, so that work still reads its own command's panel. struct LitPanelSurface { let app: XCUIApplication let displayID: UInt @@ -42,16 +40,15 @@ extension RunnerTests { /// The lit panel the current request named, or nil for a single-panel request. Main only. var requestLitPanel: RunnerLitPanel? { litPanelState.panel } - /// Books `app`'s window on the lit panel for the rest of the request, or returns why it cannot. A - /// request that named no panel books nothing. Later surface reads of `app` answer with this - /// window, so none of them can fall back to the application's own coordinates. The booked frame - /// is the panel's box turned like the window, so a partial window maps into the whole screen. + /// Books `app`'s lit-panel window for the request, or returns why it cannot. The booked frame is + /// the panel's box turned like the window, so a partial window maps into the whole screen. func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { guard let litPanel = requestLitPanel else { return nil } if let booked = litPanelState.surface, booked.app === app { return nil } var window: AnyObject? var frame = CGRect.null - let failure = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame) + var screen: AnyObject? + let failure = RunnerResolveLitPanelWindow(app, litPanel, &window, &frame, &screen) guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } @@ -71,10 +68,8 @@ extension RunnerTests { return CGRect(origin: windowFrame.origin, size: turned ? CGSize(width: long, height: short) : CGSize(width: short, height: long)) } - /// Where an element read searches: the application, or on a lit-panel request only the windows - /// proven on the lit display. A list that cannot be read throws `APP_SCREEN_UNRESOLVED`. The walk - /// runs inside an issue scope, so a window that vanishes mid-walk refuses instead of ending the - /// session (#2995). + /// The roots of every element read: the app, or the windows proven on the lit display. A walk that + /// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`. func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] { guard let litPanel = requestLitPanel else { return [app] } if let cached = litPanelState.windows, cached.app === app { return cached.elements } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 3f8867aa22..d6466341ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -296,16 +296,14 @@ extension RunnerTests { ) } - /// The application's frame names one panel, so a lit-panel snapshot's root, raw or regular, is the - /// booked window's box (ADR 0025). + /// A lit-panel snapshot's root, raw or regular, is the booked box (ADR 0025). static func publishLitPanelRoot(_ root: inout RawAXNode, surface: LitPanelSurface?) { if let surface { root.rect = SnapshotRect(surface.frame) } } - /// A multi-panel device lists windows on both panels under one application, so a lit-panel - /// snapshot descends only into the windows proven on the lit display, and refuses when none is. + /// A lit-panel snapshot descends only into windows proven on the lit display; none refuses. static func litPanelRootChildren( _ root: XCUIElementSnapshot, surface: LitPanelSurface? @@ -318,8 +316,7 @@ extension RunnerTests { return lit } - /// The windows a lit-panel snapshot keeps: only those whose reported display is the lit one. A - /// window that reports no display is dropped, because it may be on the dark panel. + /// A window that reports no display is dropped: it may be on the dark panel. static func windowsOnLitDisplay( _ windows: [Window], litDisplayID: UInt, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift index 792bc9d0d5..7947052a5e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift @@ -80,7 +80,7 @@ extension RunnerTests { let waitStartedAt = Date() NSLog("[DEBUG-1874] wait start expectedLen=%ld route=replacement", expectedText.count) let ingredients = try synthesizedCommitPollingIngredients(app: app, target: target, expectedText: expectedText) - let outcome = Self.awaitSynthesizedReplacementCommitOutcome( + let outcome = try Self.awaitSynthesizedReplacementCommitOutcome( expectedText: expectedText, placeholder: ingredients.placeholder, fieldIsUnreadable: ingredients.fieldIsUnreadable, @@ -135,7 +135,7 @@ extension RunnerTests { ) throws -> ( placeholder: String?, fieldIsUnreadable: Bool, - observe: () -> TextValueRead, + observe: () throws -> TextValueRead, waitForNextObservation: () -> Void ) { let field = try resolveTextEntryInput(app: app, target: target)?.attributes @@ -146,7 +146,7 @@ extension RunnerTests { fieldIsUnreadable: textValueRead(for: field) == .unreadable, observe: { let observed = self.textValueRead( - for: (try? self.resolveTextEntryInput(app: app, target: target))??.attributes, + for: try self.resolveTextEntryInput(app: app, target: target)?.attributes, treatingPlaceholderAsEmpty: true ) var observedText: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift index 18380a7725..6ab772d269 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift @@ -359,7 +359,7 @@ extension RunnerTests { plan, text: request.text, post: { slice, step in - func enter(_ text: String, replacingExistingText: Bool) -> SynthesizedStepDispatch { + func enter(_ text: String, replacingExistingText: Bool) throws -> SynthesizedStepDispatch { switch request.synthesizer.enterText( app: request.app, text: text, @@ -379,18 +379,18 @@ extension RunnerTests { } if step.replacesExistingText { var passes = 0 - let outcome = Self.clearUntilEmpty( + let outcome = try Self.clearUntilEmpty( clearOnce: { passes += 1 - return enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true) + return try enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true) }, - read: { self.clearedFieldRead(app: request.app, target: request.target) } + read: { try self.clearedFieldRead(app: request.app, target: request.target) } ) clearOutcome = outcome NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR passes=%d outcome=%@", passes, String(describing: outcome)) guard outcome == .cleared || outcome == .unverified else { return .stop } } - return enter(slice, replacingExistingText: false) + return try enter(slice, replacingExistingText: false) }, // A replacement never peels a warmup character, so no step asks for a read-back. waitAfterWarmupCharacter: { _ in } @@ -523,9 +523,9 @@ extension RunnerTests { stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout, ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling, now: () -> Date = { Date() }, - observe: () -> TextValueRead, + observe: () throws -> TextValueRead, waitForNextObservation: () -> Void - ) -> SynthesizedTextCommitOutcome { + ) rethrows -> SynthesizedTextCommitOutcome { // An unreadable field never exposes its value, so every read could only expire the wait. The // element route leaves such a field unverified rather than failed; so does this one. A masked // value appears only once the burst lands, so a read that turns unreadable mid-wait ends it @@ -545,7 +545,7 @@ extension RunnerTests { // landed during the final poll sleep — the exact loaded-host timing this wait exists for. while true { let observedText: String? - switch observe() { + switch try observe() { case .unreadable: return .unobservable case .text(let value): diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 258bee27b5..76e8c4ac03 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -390,7 +390,7 @@ extension RunnerTests { } /// Clears the reader's input. Every post goes through `reader.dispatch` and every read back through - /// `try reader.input()`, so a bound target's clear stops at an input that took its place. On iOS a + /// `reader.input()`, so a bound target's clear stops at an input that took its place. On iOS a /// readable value is selected whole and deleted, wherever the caret is. Otherwise, or when text is /// left, delete bursts run from the caret, and a value longer than one burst is read again after /// each burst until it reads empty, stops shortening, or has had at least one burst more than it @@ -591,8 +591,7 @@ extension RunnerTests { } } - /// The query, under the element search's roots, for an input whose identifier is unique enough - /// to search by. Of several lit windows, the first one holding a match answers. + /// The query, under the element search's roots, for an input whose identifier is distinctive. private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) throws -> XCUIElement? { guard identity.isDistinguishable else { return nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index bf2464f43f..2106ac54c3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -81,12 +81,19 @@ extension RunnerTests { } /// What `target`'s input reads between a replacement's clear passes, read inside the same issue - /// containment, with a placeholder read as empty. A read that cannot answer is `.unavailable`. - func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) -> TextValueRead { + /// containment, with a placeholder read as empty. A read that cannot answer is `.unavailable`; + /// a lit-panel refusal propagates. + func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) throws -> TextValueRead { + var refusal: Error? let probed = containingTextInputProbeIssues(fallback: TextValueRead.unavailable) { _ in - guard let input = try? resolveTextEntryInput(app: app, target: target) else { return .unavailable } - return textValueRead(for: input.attributes, treatingPlaceholderAsEmpty: true) + do { + return textValueRead(for: try resolveTextEntryInput(app: app, target: target)?.attributes, treatingPlaceholderAsEmpty: true) + } catch { + refusal = error + return .unavailable + } } + if let refusal { throw refusal } guard case .success(let read) = probed else { return .unavailable } return read } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index a121261687..d977ca2717 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -33,8 +33,7 @@ typedef NS_ENUM(NSUInteger, RunnerApplicationScreenFailure) { RunnerApplicationScreenFailureOffLitPanel, }; -// The panel the host reports as lit (ADR 0025): its display and its native box in points. The -// system presents a panel turned, so a window on it may report the box with its sides swapped. +// The lit panel (ADR 0025): its display and native box in points, sides swapped on a turned window. typedef struct { NSUInteger displayID; CGFloat widthPt; @@ -48,12 +47,14 @@ typedef struct { // Resolves the window on `litPanel`: the first window on that display whose frame is the panel's // box either way up, else the largest window on that display. An unreadable list, an unplaced // window, or a list past the cap with no full-panel window in it, is `UnresolvedScreen`. Windows -// that are all on another display are `OffLitPanel`; no window at all is `UnresolvedWindow`. +// that are all on another display are `OffLitPanel`; no window at all is `UnresolvedWindow`. The +// screen is the one the chosen window reports. FOUNDATION_EXPORT RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( id application, RunnerLitPanel litPanel, id _Nullable *window, - CGRect *windowFrame + CGRect *windowFrame, + id _Nullable *screen ); // Resolves the shared core against the live XCTest runtime. Returns nil on success, or a @@ -81,16 +82,6 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( // or names no display is left out. Nil when the window list itself cannot be read. FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); -// `RunnerResolveApplicationScreen` for a multi-panel device: the window comes from -// `RunnerResolveLitPanelWindow` instead of the application's first window. -FOUNDATION_EXPORT BOOL RunnerResolveLitPanelScreen( - id application, - RunnerLitPanel litPanel, - id _Nullable *screen, - NSUInteger *displayID, - RunnerApplicationScreenFailure *failure -); - // Reads the display ID off an already-resolved window, keeping a synthesized record's display and // the caller's booked reference frame on the same window. The frame is still read before the // screen; a window that answers empty or raises fails closed with a refusal message naming the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index f837e60c0f..4f43e177dd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -234,7 +234,7 @@ static RunnerApplicationScreenFailure RunnerReadLitPanelCandidate( return exception == nil ? windows : nil; } -static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( +RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( id application, RunnerLitPanel litPanel, id _Nullable *window, @@ -335,34 +335,6 @@ static RunnerApplicationScreenFailure RunnerResolveLitPanelWindowScreen( return lit; } -RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( - id application, - RunnerLitPanel litPanel, - id _Nullable *window, - CGRect *windowFrame -) { - id screen = nil; - return RunnerResolveLitPanelWindowScreen(application, litPanel, window, windowFrame, &screen); -} - -BOOL RunnerResolveLitPanelScreen( - id application, - RunnerLitPanel litPanel, - id _Nullable *screen, - NSUInteger *displayID, - RunnerApplicationScreenFailure *failure -) { - *displayID = 0; - id window = nil; - CGRect windowFrame = CGRectNull; - *failure = RunnerResolveLitPanelWindowScreen(application, litPanel, &window, &windowFrame, screen); - if (*failure != RunnerApplicationScreenFailureNone) { - return NO; - } - *displayID = litPanel.displayID; - return YES; -} - BOOL RunnerResolveApplicationScreen( id application, id _Nullable *screen, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 63e38e5f20..898907bcd7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -203,6 +203,7 @@ extension RunnerTests { XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) } + XCTAssertThrowsError(try clearedFieldRead(app: app, target: target), "a clear pass read propagates it too") XCTAssertFalse(boundTextEntryInputIsGone(app: app, target: target)) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index f9c81da9cb..1918d62b50 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -263,12 +263,13 @@ extension RunnerTests { private func resolveLitPanelWindow( _ windows: [NSObject], panel: RunnerLitPanel = innerPanel - ) -> (failure: RunnerApplicationScreenFailure, window: AnyObject?, frame: CGRect) { + ) -> (failure: RunnerApplicationScreenFailure, window: AnyObject?, frame: CGRect, screen: AnyObject?) { var window: AnyObject? var frame = CGRect.null + var screen: AnyObject? let failure = RunnerResolveLitPanelWindow( - DisplayApplicationFixture(windows: WindowListFixture(windows)), panel, &window, &frame) - return (failure, window, frame) + DisplayApplicationFixture(windows: WindowListFixture(windows)), panel, &window, &frame, &screen) + return (failure, window, frame, screen) } func testLitPanelWindowSkipsTheFirstWindowOnTheDarkPanel() { @@ -308,8 +309,9 @@ extension RunnerTests { func testLitPanelWindowRefusesAWindowListItCannotRead() { var window: AnyObject? var frame = CGRect.null + var screen: AnyObject? let failure = RunnerResolveLitPanelWindow( - DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel, &window, &frame) + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel, &window, &frame, &screen) XCTAssertEqual(failure, .unresolvedScreen) XCTAssertNil(window) } @@ -377,18 +379,9 @@ extension RunnerTests { func testLitPanelScreenIsTheScreenTheChosenWindowReported() { let home = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) - var screen: AnyObject? - var displayID: UInt = 0 - var failure = RunnerApplicationScreenFailure.unresolvedScreen - XCTAssertTrue( - RunnerResolveLitPanelScreen( - DisplayApplicationFixture(windows: WindowListFixture([ - DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1), home, - ])), - Self.innerPanel, &screen, &displayID, &failure) - ) - XCTAssertEqual(displayID, 3) - XCTAssertTrue(screen === home.screen) + let walk = resolveLitPanelWindow([DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1), home]) + XCTAssertEqual(walk.failure, .none) + XCTAssertTrue(walk.screen === home.screen) } } #endif From bd46ee9de73197a87fc8cfded75a351ff13f9e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 08:54:59 +0100 Subject: [PATCH 20/29] fix(ios): book the lit panel's screen box at the screen origin The booked box was inferred from the window: its origin and whether it was wider than tall. A window that the inner panel's status strip pushes to a non-zero origin would then book a box 80 points past the screen, and every tap, reference and snapshot root would land off. The box is now the panel's native portrait box laid out in the app's interface orientation, at the screen origin, through the existing soleScreenReferenceFrame. The XCUICoordinate anchor keeps the window's own origin. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 25 ++++++++++++------ .../RunnerTests+Interaction.swift | 7 +++-- .../UnitTests/RunnerRequestPanelTests.swift | 26 ++++++++++--------- docs/adr/0025-foldable-apple-panels.md | 2 +- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index eaf4b7afe5..f161de8f40 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -6,6 +6,8 @@ struct LitPanelSurface { let app: XCUIApplication let displayID: UInt let window: XCUIElement + let windowOrigin: CGPoint + /// The lit panel's screen in the app's interface orientation, whatever the window's shape. let frame: CGRect } @@ -40,8 +42,7 @@ extension RunnerTests { /// The lit panel the current request named, or nil for a single-panel request. Main only. var requestLitPanel: RunnerLitPanel? { litPanelState.panel } - /// Books `app`'s lit-panel window for the request, or returns why it cannot. The booked frame is - /// the panel's box turned like the window, so a partial window maps into the whole screen. + /// Books `app`'s lit-panel window for the request, or returns why it cannot. func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { guard let litPanel = requestLitPanel else { return nil } if let booked = litPanelState.surface, booked.app === app { return nil } @@ -52,20 +53,28 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } + guard let screenFrame = Self.litPanelScreenFrame( + litPanel, + interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + ) else { + return .unresolvedScreen + } litPanelState.surface = LitPanelSurface( app: app, displayID: litPanel.displayID, window: element, - frame: Self.litPanelScreenFrame(litPanel, windowFrame: frame) + windowOrigin: frame.origin, + frame: screenFrame ) return nil } - static func litPanelScreenFrame(_ panel: RunnerLitPanel, windowFrame: CGRect) -> CGRect { - let long = max(panel.widthPt, panel.heightPt) - let short = min(panel.widthPt, panel.heightPt) - let turned = windowFrame.width > windowFrame.height - return CGRect(origin: windowFrame.origin, size: turned ? CGSize(width: long, height: short) : CGSize(width: short, height: long)) + /// The panel's native portrait box laid out in `interfaceOrientation`, at the screen origin. + static func litPanelScreenFrame(_ panel: RunnerLitPanel, interfaceOrientation: Int) -> CGRect? { + soleScreenReferenceFrame( + nativePortraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), + interfaceOrientation: interfaceOrientation + ) } /// The roots of every element read: the app, or the windows proven on the lit display. A walk that diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 1ce69bcf25..033913a302 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -647,8 +647,11 @@ extension RunnerTests { let resolved = resolveRunnerWindow(app: app) let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) - let offsetX = x - Double(resolved.frame.origin.x) - let offsetY = y - Double(resolved.frame.origin.y) + // A lit-panel window can sit inside the booked screen box, so the anchor is its own origin. + let booked = litPanelState.surface.flatMap { $0.app === app ? $0 : nil } + let anchor = booked?.windowOrigin ?? resolved.frame.origin + let offsetX = x - Double(anchor.x) + let offsetY = y - Double(anchor.y) return origin.withOffset(CGVector(dx: offsetX, dy: offsetY)) } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 898907bcd7..ca5f8ef246 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -1,3 +1,4 @@ +import AgentDeviceSnapshotPresentation import XCTest #if AGENT_DEVICE_RUNNER_UNIT_TESTS @@ -48,18 +49,17 @@ extension RunnerTests { XCTAssertEqual(queuedLater, 3) } - /// The Duo leaves a status strip beside an app's window on the inner panel; a point must still map - /// into the whole screen, so the booked frame is the panel's box turned like the window. - func testLitPanelBookingMapsIntoTheWholeTurnedPanel() { + /// The Duo's 80 pt status strip can put an app's window at a non-zero origin on the inner panel. + /// The booked box takes no window input: it is the panel's screen in the interface orientation, + /// at the screen origin, so no window origin or shape can move it. + func testLitPanelBookingIsTheScreenInTheInterfaceOrientation() { let inner = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) - XCTAssertEqual( - Self.litPanelScreenFrame(inner, windowFrame: CGRect(x: 0, y: 0, width: 871, height: 669)), - CGRect(x: 0, y: 0, width: 951, height: 669) - ) - XCTAssertEqual( - Self.litPanelScreenFrame(inner, windowFrame: CGRect(x: 0, y: 0, width: 669, height: 951)), - CGRect(x: 0, y: 0, width: 669, height: 951) - ) + let landscape = CGRect(x: 0, y: 0, width: 951, height: 669) + let portrait = CGRect(x: 0, y: 0, width: 669, height: 951) + XCTAssertEqual(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft), landscape) + XCTAssertEqual(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight), landscape) + XCTAssertEqual(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.portrait), portrait) + XCTAssertNil(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.unknown), "an unknown orientation books nothing") } func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() throws { @@ -203,7 +203,9 @@ extension RunnerTests { XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) } - XCTAssertThrowsError(try clearedFieldRead(app: app, target: target), "a clear pass read propagates it too") + XCTAssertThrowsError(try clearedFieldRead(app: app, target: target), "a clear pass read propagates it too") { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } XCTAssertFalse(boundTextEntryInputIsGone(app: app, target: target)) } diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 513b050472..7288129d6d 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -732,6 +732,6 @@ Rejected alternatives: Consequences: -- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` is the lit panel's box turned like that window, at its origin. A window that leaves a strip of the panel uncovered, like an app beside the inner panel's status strip, still maps points into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. +- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` is the lit panel's screen in the app's interface orientation, at the screen origin, whatever the booked window's origin or shape: an app beside the inner panel's status strip still maps points into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. - A recording keeps the panel that its `recordStart` named. The runner cannot read panel power, so it does not notice a fold during the recording: SpringBoard keeps windows on the panel that went dark, and later frames show that dark panel. Start a new recording after a fold. - Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. From c01b4c9375037c81b5f2b126d7cb17a73b5b8a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 09:28:57 +0100 Subject: [PATCH 21/29] fix(ios): turn lit-panel taps about the screen, keep the window's translation The previous commit booked the screen box, which broke the placed-window translation: XCTest reports a window at (0,80) from its own top-left, and the snapshot shifts those frames into screen space only when the viewport is that window's frame. The booking is the window frame again, as for a placed iPad window (#3899), and on a lit-panel request the synthesized turn is about the lit panel's screen in the interface orientation instead of the window's shape. That replaces the aspect guess and the window origin inference. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 19 +++----------- .../RunnerTests+Interaction.swift | 7 ++--- .../RunnerTests+SynthesizedInteraction.swift | 7 +++-- .../UnitTests/RunnerRequestPanelTests.swift | 26 ++++++++++++++++--- docs/adr/0025-foldable-apple-panels.md | 2 +- 5 files changed, 33 insertions(+), 28 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index f161de8f40..6439973bad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -6,8 +6,6 @@ struct LitPanelSurface { let app: XCUIApplication let displayID: UInt let window: XCUIElement - let windowOrigin: CGPoint - /// The lit panel's screen in the app's interface orientation, whatever the window's shape. let frame: CGRect } @@ -53,23 +51,12 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - guard let screenFrame = Self.litPanelScreenFrame( - litPanel, - interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - ) else { - return .unresolvedScreen - } - litPanelState.surface = LitPanelSurface( - app: app, - displayID: litPanel.displayID, - window: element, - windowOrigin: frame.origin, - frame: screenFrame - ) + litPanelState.surface = LitPanelSurface(app: app, displayID: litPanel.displayID, window: element, frame: frame) return nil } - /// The panel's native portrait box laid out in `interfaceOrientation`, at the screen origin. + /// The lit panel's screen in `interfaceOrientation`, at the screen origin: the box a synthesized + /// turn swaps sides about, whatever the booked window's origin or shape. static func litPanelScreenFrame(_ panel: RunnerLitPanel, interfaceOrientation: Int) -> CGRect? { soleScreenReferenceFrame( nativePortraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 033913a302..1ce69bcf25 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -647,11 +647,8 @@ extension RunnerTests { let resolved = resolveRunnerWindow(app: app) let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) - // A lit-panel window can sit inside the booked screen box, so the anchor is its own origin. - let booked = litPanelState.surface.flatMap { $0.app === app ? $0 : nil } - let anchor = booked?.windowOrigin ?? resolved.frame.origin - let offsetX = x - Double(anchor.x) - let offsetY = y - Double(anchor.y) + let offsetX = x - Double(resolved.frame.origin.x) + let offsetY = y - Double(resolved.frame.origin.y) return origin.withOffset(CGVector(dx: offsetX, dy: offsetY)) } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 7a20a5de0f..3296d7c64c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -512,14 +512,17 @@ extension RunnerTests { } /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a - /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the - /// resolved window `fallback`, which names its panel (ADR 0025). When the device's only screen + /// placed iPad window's smaller box is not the basis (#3899). A named lit panel supplies its full + /// screen box independently of the booked window translation (ADR 0025). When the device's only screen /// will not report its size, `fallback` is the basis only while the app provably fills that /// screen; otherwise this is nil and the gesture refuses before it dispatches. Portrait and an /// unknown orientation leave the point as it is, so they keep `fallback` without a read; landscape /// swaps the box's sides and upside-down mirrors the point across them, so both read it. @MainActor func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { + if let panel = requestLitPanel { + return Self.litPanelScreenFrame(panel, interfaceOrientation: interfaceOrientation) ?? fallback + } guard XCUIScreen.screens.count == 1 else { return fallback } switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index ca5f8ef246..da964512a2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -49,10 +49,9 @@ extension RunnerTests { XCTAssertEqual(queuedLater, 3) } - /// The Duo's 80 pt status strip can put an app's window at a non-zero origin on the inner panel. - /// The booked box takes no window input: it is the panel's screen in the interface orientation, - /// at the screen origin, so no window origin or shape can move it. - func testLitPanelBookingIsTheScreenInTheInterfaceOrientation() { + /// The box a lit-panel turn swaps sides about is the panel's screen in the interface orientation, + /// at the screen origin; it takes no window input. + func testLitPanelScreenFrameIsTheScreenInTheInterfaceOrientation() { let inner = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) let landscape = CGRect(x: 0, y: 0, width: 951, height: 669) let portrait = CGRect(x: 0, y: 0, width: 669, height: 951) @@ -97,6 +96,25 @@ extension RunnerTests { } #if os(iOS) + /// The Duo's 80 pt status strip puts a portrait app's window at (0,80) on the inner panel. Points + /// stay in the window's translated space, and the turn is about the whole lit screen, not the + /// window's shape. + func testLitPanelRotationIsAboutTheScreenForAWindowAtANonZeroOrigin() throws { + defer { setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) } + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + let window = CGRect(x: 0, y: 80, width: 669, height: 871) + XCTAssertEqual( + synthesizedRotationFrame(fallback: window, interfaceOrientation: RunnerInterfaceOrientation.portrait), + CGRect(x: 0, y: 0, width: 669, height: 951) + ) + XCTAssertEqual( + synthesizedRotationFrame(fallback: CGRect(x: 0, y: 0, width: 871, height: 669), interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft), + CGRect(x: 0, y: 0, width: 951, height: 669) + ) + } + /// Launches the app and names its own window's display and box as the lit panel: the /// `litPanel` JSON member, and the window frame. @MainActor diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 7288129d6d..425324e5fd 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -732,6 +732,6 @@ Rejected alternatives: Consequences: -- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` is the lit panel's screen in the app's interface orientation, at the screen origin, whatever the booked window's origin or shape: an app beside the inner panel's status strip still maps points into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. +- `runner.log` prints `AGENT_DEVICE_RUNNER_LIT_PANEL_WINDOW lit=(x) windows= outcome= frame=(…)`. `SYNTHESIZED_DISPATCH reference=` must equal that frame. Snapshot points are translated by that frame's origin, as for a placed iPad window, and a synthesized turn is about the lit panel's screen in the app's interface orientation, not the window's shape: an app beside the inner panel's 80 pt status strip still maps into the whole screen. `SYNTHESIZED_RECORD displayID=` must equal `lit=`. - A recording keeps the panel that its `recordStart` named. The runner cannot read panel power, so it does not notice a fold during the recording: SpringBoard keeps windows on the panel that went dark, and later frames show that dark panel. Start a new recording after a fold. - Open gap: on the folded home screen, XCTest reports the page icons with the inner panel's layout in both panels' windows. The Dock and the page indicator report the outer panel correctly. From 8ee8d47b16272e2cb999181bc26c12b2969aa3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 09:48:51 +0100 Subject: [PATCH 22/29] test(ios): drive the lit-panel booking for a partial window; scope the walk cap The rotation-basis case now books the launched app's window under a lit panel taller than it and checks the synthesized reference and turn box, so a wiring regression fails it. The one-caller screen-frame helper is inlined. The window cap's doc names the booking walk it bounds; the element-search copy stays uncapped so every lit window is searched. Co-Authored-By: Claude Opus 5.5 --- .../RunnerRequestPanel.swift | 9 ---- .../RunnerTests+SynthesizedInteraction.swift | 5 ++- .../RunnerXCTestEventBridge.h | 9 ++-- .../UnitTests/RunnerRequestPanelTests.swift | 42 +++++++------------ 4 files changed, 25 insertions(+), 40 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 6439973bad..4e6bd224cf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -55,15 +55,6 @@ extension RunnerTests { return nil } - /// The lit panel's screen in `interfaceOrientation`, at the screen origin: the box a synthesized - /// turn swaps sides about, whatever the booked window's origin or shape. - static func litPanelScreenFrame(_ panel: RunnerLitPanel, interfaceOrientation: Int) -> CGRect? { - soleScreenReferenceFrame( - nativePortraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), - interfaceOrientation: interfaceOrientation - ) - } - /// The roots of every element read: the app, or the windows proven on the lit display. A walk that /// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`. func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 3296d7c64c..965f7ab377 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -521,7 +521,10 @@ extension RunnerTests { @MainActor func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { if let panel = requestLitPanel { - return Self.litPanelScreenFrame(panel, interfaceOrientation: interfaceOrientation) ?? fallback + return Self.soleScreenReferenceFrame( + portraitSize: CGSize(width: panel.widthPt, height: panel.heightPt), + interfaceOrientation: interfaceOrientation + ) ?? fallback } guard XCUIScreen.screens.count == 1 else { return fallback } switch interfaceOrientation { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index d977ca2717..df32b82604 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -40,8 +40,8 @@ typedef struct { CGFloat heightPt; } RunnerLitPanel; -// The walk reads at most this many windows: each read of a window that has vanished records an -// XCTest issue (#2995), and SpringBoard lists about forty. +// The booking walk (`RunnerResolveLitPanelWindow`) reads at most this many windows: a vanished +// window's read records an XCTest issue (#2995), and SpringBoard lists about forty. #define AGENT_DEVICE_MAX_LIT_PANEL_WINDOWS 8 // Resolves the window on `litPanel`: the first window on that display whose frame is the panel's @@ -78,8 +78,9 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( RunnerApplicationScreenFailure *failure ); -// The application's windows that report `litPanel`'s display, in list order; a window that is gone -// or names no display is left out. Nil when the window list itself cannot be read. +// Every application window that reports `litPanel`'s display, in list order, uncapped so element +// search sees all of them; the caller holds the walk's XCTest issues. A window that is gone or names +// no display is left out. Nil when the window list itself cannot be read. FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); // Reads the display ID off an already-resolved window, keeping a synthesized record's display and diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index da964512a2..6eda5aac1f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -49,18 +49,6 @@ extension RunnerTests { XCTAssertEqual(queuedLater, 3) } - /// The box a lit-panel turn swaps sides about is the panel's screen in the interface orientation, - /// at the screen origin; it takes no window input. - func testLitPanelScreenFrameIsTheScreenInTheInterfaceOrientation() { - let inner = RunnerLitPanel(displayID: 3, widthPt: 669, heightPt: 951) - let landscape = CGRect(x: 0, y: 0, width: 951, height: 669) - let portrait = CGRect(x: 0, y: 0, width: 669, height: 951) - XCTAssertEqual(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft), landscape) - XCTAssertEqual(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight), landscape) - XCTAssertEqual(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.portrait), portrait) - XCTAssertNil(Self.litPanelScreenFrame(inner, interfaceOrientation: RunnerInterfaceOrientation.unknown), "an unknown orientation books nothing") - } - func testLitPanelSnapshotRefusesAPinToABackendThatCannotProveThePanel() throws { func refusal(_ fields: String) throws -> ErrorPayload? { try Self.litPanelBackendPinRefusal(runnerCommandFixture(#"{"command":"snapshot"\#(fields)}"#))?.error @@ -96,25 +84,27 @@ extension RunnerTests { } #if os(iOS) - /// The Duo's 80 pt status strip puts a portrait app's window at (0,80) on the inner panel. Points - /// stay in the window's translated space, and the turn is about the whole lit screen, not the - /// window's shape. - func testLitPanelRotationIsAboutTheScreenForAWindowAtANonZeroOrigin() throws { - defer { setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"tap"}"#)) } + /// A window that leaves part of the lit panel uncovered, like an app beside the Duo's 80 pt status + /// strip, books its own frame as the reference, and a synthesized turn is about the whole panel. + @MainActor + func testLitPanelBookingTurnsAboutTheWholePanelForAPartialWindow() throws { + let (_, frame) = launchAppOnTheLitPanel() + defer { tearDownLitPanelApp() } + let window = app.windows.firstMatch + var displayID: UInt = 0 + XCTAssertNil(RunnerResolveWindowDisplayID(window, &displayID)) setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + #"{"command":"tap","litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height + 80)}}"# )) - let window = CGRect(x: 0, y: 80, width: 669, height: 871) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let context = try XCTUnwrap(synthesizedCoordinateContext(app: app)) + XCTAssertEqual(context.referenceFrame, frame, "the booked reference is the window's own frame") XCTAssertEqual( - synthesizedRotationFrame(fallback: window, interfaceOrientation: RunnerInterfaceOrientation.portrait), - CGRect(x: 0, y: 0, width: 669, height: 951) - ) - XCTAssertEqual( - synthesizedRotationFrame(fallback: CGRect(x: 0, y: 0, width: 871, height: 669), interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft), - CGRect(x: 0, y: 0, width: 951, height: 669) + synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: RunnerInterfaceOrientation.portrait), + CGRect(x: 0, y: 0, width: frame.width, height: frame.height + 80), + "the turn is about the whole lit panel, not the window" ) } - /// Launches the app and names its own window's display and box as the lit panel: the /// `litPanel` JSON member, and the window frame. @MainActor From 24be5ee9c852c700f23cb01eca0dd6fb33648788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 10:09:03 +0100 Subject: [PATCH 23/29] docs(adr): map each capture refusal to its cases and scope the window cap Co-Authored-By: Claude Opus 5.5 --- docs/adr/0025-foldable-apple-panels.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 425324e5fd..1869c9e1bb 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -154,8 +154,13 @@ SpringBoard owns the home-screen window. Thus, a session without an app, or with a stopped app, can still resolve a window's display. The result always names a display occupied by a window. It never defaults to `XCUIScreen.main`, which is dark on a Duo when the app uses the inner panel. -A capture fails closed only in four cases. Neither window resolves. A resolved window names no display. A resolved display yields an unrenderable image. On a multi-panel device, every window of the target app is on the dark panel. -The typed reasons are `unresolvedWindow`, `unresolvedScreen`, `unrenderableImage`, and `offLitPanel` (`APP_SCREEN_OFF_LIT_PANEL`) (`RunnerAppScreenCapture.swift`). +A capture fails closed with one of four typed reasons (`RunnerAppScreenCapture.swift`). + +- `unresolvedWindow` (`APP_SCREEN_WINDOW_UNRESOLVED`): neither window resolves. +- `unresolvedScreen` (`APP_SCREEN_UNRESOLVED`): a resolved window names no display. On a multi-panel device, it also covers a window list that cannot be read and a window the lit-panel walk cannot place. +- `unrenderableImage` (`APP_SCREEN_CAPTURE_UNRENDERABLE`): a resolved display yields an unrenderable image. +- `offLitPanel` (`APP_SCREEN_OFF_LIT_PANEL`): on a multi-panel device, every window of the target app is on the dark panel. + The query uses both `exists` and frame evidence to refuse an app without a window. Such an app returns `(0,0 0x0)` from `frame` without an exception. Geometry alone cannot prove that no window exists. @@ -717,7 +722,7 @@ SpringBoard keeps windows on both panels. Its first window is the outer panel's Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. -With a named panel, the runner walks at most eight windows. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. +With a named panel, the runner walks at most eight windows to book one; element search reads every window that reports the lit display. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search owns this rule: if the window list cannot be read, the search throws `APP_SCREEN_UNRESOLVED`, which answers the request instead of no match. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. From eaf63107d339ab5b1ffaededaa588f57dd168b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 10:32:47 +0100 Subject: [PATCH 24/29] fix(ios): preserve screen points across lit-panel coordinate fallbacks --- .../RunnerRequestPanel.swift | 14 +- .../RunnerTests+Alert.swift | 19 +- .../RunnerTests+CommandExecution.swift | 43 ++-- .../RunnerTests+GestureExecution.swift | 16 +- .../RunnerTests+Interaction.swift | 54 ++--- .../RunnerTests+Navigation.swift | 24 +-- .../RunnerTests+ScrollDragExecution.swift | 22 +- .../RunnerTests+SequenceExecution.swift | 22 +- .../RunnerTests+TextEntry.swift | 25 +-- .../RunnerTests+TextEntryFocus.swift | 2 +- .../RunnerTests+TextEntryReadiness.swift | 8 +- .../RunnerTests+TextTyping.swift | 5 +- .../RunnerTests+TvRemote.swift | 4 +- .../UnitTests/RunnerRequestPanelTests.swift | 196 +++++++++++++++--- .../RunnerTests+AlertButtonChoiceTests.swift | 4 +- .../RunnerTests+AlertObservationTests.swift | 127 +++++------- .../UnitTests/RunnerTests+AlertTests.swift | 2 +- .../RunnerTests+SequenceExecutionTests.swift | 18 +- .../RunnerTests+TextEntryLatencyTests.swift | 2 +- .../RunnerTests+TextEntryReadinessTests.swift | 6 +- .../RunnerTests+XCTestEventBridgeTests.swift | 16 +- docs/adr/0025-foldable-apple-panels.md | 2 +- .../src/foldable/runner-panel.test.ts | 16 +- 23 files changed, 406 insertions(+), 241 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 4e6bd224cf..fe9573cd39 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -13,7 +13,6 @@ struct LitPanelSurface { final class RunnerLitPanelState { var panel: RunnerLitPanel? var surface: LitPanelSurface? - var windows: (app: XCUIApplication, elements: [XCUIElement])? } extension RunnerTests { @@ -28,7 +27,6 @@ extension RunnerTests { let write = { state.panel = panel state.surface = nil - state.windows = nil } if Thread.isMainThread { write() @@ -55,11 +53,18 @@ extension RunnerTests { return nil } + func requireRequestLitPanelSurface(app: XCUIApplication) throws -> LitPanelSurface { + if let refusal = bookRequestLitPanelSurface(app: app) { throw refusal } + guard let surface = litPanelState.surface, surface.app === app else { + throw RunnerAppScreenCaptureFailure.unresolvedScreen + } + return surface + } + /// The roots of every element read: the app, or the windows proven on the lit display. A walk that /// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`. func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] { guard let litPanel = requestLitPanel else { return [app] } - if let cached = litPanelState.windows, cached.app === app { return cached.elements } var windows: [XCUIElement]? let walk = holdingRecordedIssues { windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap { $0 as? XCUIElement } @@ -69,10 +74,9 @@ extension RunnerTests { } #endif } - guard walk.issues.isEmpty, walk.exceptionMessage == nil, let windows else { + guard walk.issues.isEmpty, walk.exceptionMessage == nil, let windows, !windows.isEmpty else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } - litPanelState.windows = (app, windows) return windows } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index f7dd07b92f..0d26d8fc13 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -95,7 +95,7 @@ extension RunnerTests { } @MainActor - func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) -> Response { + func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) throws -> Response { if action == "accept" || action == "dismiss" { guard let button = chooseAlertButton(alert, action: action) else { // The choice refused before any activation, so this discloses dispatched: no like the @@ -141,7 +141,7 @@ extension RunnerTests { // (#2546). The post-tap settle stays, because the verification below reads the alert this tap // replaces; an alert that dismisses and presents an identical replacement passes through a // window with no alert, and a first read landing there reports a dismissal nothing proved. - guard let outcome = activateAlertButton(alert, button: button, action: action, frame: buttonFrame, deadline: deadline) else { + guard let outcome = try activateAlertButton(alert, button: button, action: action, frame: buttonFrame, deadline: deadline) else { return alertVerificationResponse(.timedOut, action: action, activated: false) } if let response = unsupportedResponse(for: outcome) { @@ -190,8 +190,9 @@ extension RunnerTests { action: String, frame: CGRect, deadline: Date - ) -> RunnerInteractionOutcome? { + ) throws -> RunnerInteractionOutcome? { var outcome: RunnerInteractionOutcome? + var activationError: Error? withUIInterruptionHandlingDisabledIfSupported(alert.ownerApp) { withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { guard Date() < deadline else { return } @@ -247,15 +248,15 @@ extension RunnerTests { break } #else - outcome = activateElement( - app: alert.ownerApp, - element: button, - action: "alert \(action)", - resolvedFrame: frame - ) + do { + outcome = try activateElement(app: alert.ownerApp, element: button, action: "alert \(action)", resolvedFrame: frame) + } catch { + activationError = error + } #endif } } + if let activationError { throw activationError } return outcome } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 40d0d1792c..5a6e017300 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -2,6 +2,11 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { + private func refusedTapResponse(_ message: String, hint: String?) -> Response { + clearRememberedTextEntryTap() + return unsupportedResponse(message: message, hint: hint) + } + /// Runs `command` against its prepared app; a thrown lit-panel refusal answers it (ADR 0025). @MainActor func executeOnMainPrepared( @@ -112,7 +117,7 @@ extension RunnerTests { }) { case .performed(let timing): if isTextEntry { - waitForTextEntryReadinessAfterTap(app: activeApp, element: element) + try waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } rememberTextEntryTap(isTextEntry ? element : nil) return gestureResponse( @@ -127,28 +132,27 @@ extension RunnerTests { : nil ) case .xctestFallback(let message, let hint): - fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + fallback = .coordinateTap(message, hint: hint) case .refused(_, let message, let hint): - clearRememberedTextEntryTap() - return unsupportedResponse(message: message, hint: hint) + return refusedTapResponse(message, hint: hint) } } - let (timing, outcome) = performGesture(activeApp) { + let (timing, outcome) = try performGesture(activeApp) { if expectedPoint != nil || match.usedNonHittableFallback { // Maestro compatibility: RN E2E backdoor controls can be 1x1 and // reported non-hittable by XCTest. Snapshot-resolved targets also // need coordinate delivery because XCUIElement.activate() does not // invoke every React Native accessibility wrapper. - return tapAt(app: activeApp, x: touchPoint.x, y: touchPoint.y) + return try tapAt(app: activeApp, x: touchPoint.x, y: touchPoint.y) } - return activateElement(app: activeApp, element: element, action: "tap by selector") + return try activateElement(app: activeApp, element: element, action: "tap by selector") } if let response = unsupportedResponse(for: outcome) { clearRememberedTextEntryTap() return response } if isTextEntry { - waitForTextEntryReadinessAfterTap(app: activeApp, element: element) + try waitForTextEntryReadinessAfterTap(app: activeApp, element: element) } rememberTextEntryTap(isTextEntry ? element : nil) return gestureResponse( @@ -199,14 +203,13 @@ extension RunnerTests { rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) case .xctestFallback(let message, let hint): - fallback = GestureFallback(strategy: "xctest-coordinate-tap", message: message, hint: hint) + fallback = .coordinateTap(message, hint: hint) case .refused(_, let message, let hint): - clearRememberedTextEntryTap() - return unsupportedResponse(message: message, hint: hint) + return refusedTapResponse(message, hint: hint) } } let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) - let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) } + let (timing, outcome) = try performGesture(activeApp) { try tapAt(app: activeApp, x: x, y: y) } if let response = unsupportedResponse(for: outcome) { clearRememberedTextEntryTap() return response @@ -268,8 +271,8 @@ extension RunnerTests { fallback = nil #endif let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) - let (timing, outcome) = performGesture(activeApp) { - longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) + let (timing, outcome) = try performGesture(activeApp) { + try longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) } if let response = unsupportedResponse(for: outcome) { return response @@ -285,7 +288,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2")) } let defaults = runnerDragCommandDefaults(command) - return executeDragGesture( + return try executeDragGesture( activeApp: activeApp, x: x, y: y, @@ -340,7 +343,7 @@ extension RunnerTests { return invalidScrollDurationResponse(commandName: "scroll") } return gesture.attachingEvidence( - to: executeScrollDragGesture( + to: try executeScrollDragGesture( activeApp: activeApp, x: gesture.planFrame.minX + gesture.plan.x1, y: gesture.planFrame.minY + gesture.plan.y1, @@ -647,7 +650,7 @@ extension RunnerTests { ) switch resolveAlert(app: activeApp, deadline: deadline) { case .resolved(let alert): - return handleAlert(alert, action: action, deadline: deadline) + return try handleAlert(alert, action: action, deadline: deadline) case .absent: // Typed so the host retries on absence alone: a transport or runner failure carries no // code and must not be mistaken for "no alert yet" (ALERT_NOT_FOUND_RUNNER_CODE). @@ -684,7 +687,7 @@ extension RunnerTests { let first = plan.pointers[0].samples.first!.point let last = plan.pointers[0].samples.last!.point return canonicalPlannedGestureResponse( - executeDragGesture( + try executeDragGesture( activeApp: activeApp, x: first.x, y: first.y, @@ -696,7 +699,7 @@ extension RunnerTests { ) ) case .sampled: - let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + let (timing, outcome) = try performGesture(activeApp, idleTimeout: false) { sampledPlannedGesture(app: activeApp, plan: plan) } return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) @@ -708,7 +711,7 @@ extension RunnerTests { } return Response(ok: true, data: DataPayload(message: "gestureViewport", x: frame.minX, y: frame.minY, x2: frame.width, y2: frame.height)) case .sequence: - return executeSequence(command: command, activeApp: activeApp) + return try executeSequence(command: command, activeApp: activeApp) } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift index 2d9e4b2395..14642eac1c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -47,6 +47,10 @@ extension RunnerTests { let strategy: String let message: String let hint: String? + + static func coordinateTap(_ message: String, hint: String?) -> Self { + Self(strategy: "xctest-coordinate-tap", message: message, hint: hint) + } } @@ -61,19 +65,19 @@ extension RunnerTests { func performGesture( _ app: XCUIApplication, idleTimeout: Bool = true, - _ action: () -> RunnerInteractionOutcome - ) -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) { - var outcome = RunnerInteractionOutcome.performed + _ action: () throws -> RunnerInteractionOutcome + ) throws -> (timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), outcome: RunnerInteractionOutcome) { + var outcome: Result = .success(.performed) let timing = measureGesture { if idleTimeout { withBoundedInteractionIdleTimeoutIfSupported(app, waits: .bothSkipped) { - outcome = action() + outcome = Result { try action() } } } else { - outcome = action() + outcome = Result { try action() } } } - return (timing, outcome) + return (timing, try outcome.get()) } /// Single factory for the success payload every gesture returns (message + gesture timing + diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 1ce69bcf25..bbcfed5dc9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -203,10 +203,10 @@ extension RunnerTests { let matches = try elementQueries(app: app, matching: .any) .flatMap { $0.matching(predicate).allElementsBoundByIndex } .filter(\.exists) - let facts = matches.map { element in + let facts = try matches.map { element in SelectorCandidateFacts( isHittable: element.isHittable, - hasTappableFrame: hasTappableFrame(app: app, element: element), + hasTappableFrame: try hasTappableFrame(app: app, element: element), containsExpectedPoint: expectedPoint.map(element.frame.contains) ?? true ) } @@ -236,12 +236,12 @@ extension RunnerTests { // resolved bounds Maestro-style, so the union frame is intentional. A // lit-panel request uses its booked window, because app.frame names one // panel (ADR 0025). - private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { + private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) throws -> Bool { + let windowFrame = try requestLitPanel == nil ? app.frame : requireRequestLitPanelSurface(app: app).frame let frame = element.frame if frame.isEmpty { return false } - let windowFrame = requestLitPanel == nil ? app.frame : onScreenWindowFrame(app: app) return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: windowFrame) } @@ -418,16 +418,16 @@ extension RunnerTests { } } - func tapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + func tapAt(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "tap") { return outcome } - return performCoordinateTap(app: app, x: x, y: y) + return try performCoordinateTap(app: app, x: x, y: y) } func mouseClickAt(app: XCUIApplication, x: Double, y: Double, button: String) throws { #if os(macOS) - let coordinate = interactionCoordinate(app: app, x: x, y: y) + let coordinate = try interactionCoordinate(app: app, x: x, y: y) switch button { case "primary": coordinate.tap() @@ -460,7 +460,7 @@ extension RunnerTests { durationMs: durationMs ) - let coordinate = interactionCoordinate(app: app, x: x, y: y) + let coordinate = try interactionCoordinate(app: app, x: x, y: y) let interval = desktopScrollEventIntervalSeconds(durationMs: durationMs, eventCount: events.count) for (index, deltas) in events.enumerated() { // Keep desktop scrolling on XCTest's coordinate API so macOS owns wheel synthesis, natural @@ -534,21 +534,21 @@ extension RunnerTests { } } - func doubleTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + func doubleTapAt(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "double tap") { guard case .performed = outcome else { return outcome } sleepFor(0.1) _ = pressTvRemote(.select) return .performed } - return performCoordinateDoubleTap(app: app, x: x, y: y) + return try performCoordinateDoubleTap(app: app, x: x, y: y) } - func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome { + func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) throws -> RunnerInteractionOutcome { if let outcome = longSelectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), duration: duration) { return outcome } - return performCoordinateLongPress(app: app, x: x, y: y, duration: duration) + return try performCoordinateLongPress(app: app, x: x, y: y, duration: duration) } func dragAt( @@ -558,7 +558,7 @@ extension RunnerTests { x2: Double, y2: Double, holdDuration: TimeInterval - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { // tvOS has no coordinate drag. Preserve the direction as a focus move. let dx = x2 - x let dy = y2 - y @@ -568,41 +568,41 @@ extension RunnerTests { if pressTvRemote(button) { return .performed } - return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration) + return try performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration) } - private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." ) #else - interactionCoordinate(app: app, x: x, y: y).tap() + try interactionCoordinate(app: app, x: x, y: y).tap() return .performed #endif } - private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element", hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it." ) #else - interactionCoordinate(app: app, x: x, y: y).doubleTap() + try interactionCoordinate(app: app, x: x, y: y).doubleTap() return .performed #endif } - private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome { + private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element", hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then long-select it." ) #else - interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration) + try interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration) return .performed #endif } @@ -614,22 +614,22 @@ extension RunnerTests { x2: Double, y2: Double, holdDuration: TimeInterval - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { #if os(tvOS) return .unsupported( message: "coordinate drag is not supported on tvOS", hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead." ) #else - let start = interactionCoordinate(app: app, x: x, y: y) - let end = interactionCoordinate(app: app, x: x2, y: y2) + let start = try interactionCoordinate(app: app, x: x, y: y) + let end = try interactionCoordinate(app: app, x: x2, y: y2) start.press(forDuration: holdDuration, thenDragTo: end) return .performed #endif } #if !os(tvOS) - private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { + private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) throws -> XCUICoordinate { #if os(iOS) // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the // resolved window: callers pass screen points, and `app.frame` starts at the screen origin even @@ -643,13 +643,17 @@ extension RunnerTests { let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) } -#endif + let booked = try requireRequestLitPanelSurface(app: app) + let origin = booked.window.coordinate(withNormalizedOffset: .zero) + return origin.withOffset(CGVector(dx: x - booked.frame.minX, dy: y - booked.frame.minY)) +#else let resolved = resolveRunnerWindow(app: app) let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) let offsetX = x - Double(resolved.frame.origin.x) let offsetY = y - Double(resolved.frame.origin.y) return origin.withOffset(CGVector(dx: offsetX, dy: offsetY)) +#endif } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index c40d4c7562..3576fb2258 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -37,7 +37,7 @@ extension RunnerTests { func tapInAppBackControl(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) if let back = macOSNavigationBackElement(app: app) { - tapElementCenter(app: app, element: back) + try tapElementCenter(app: app, element: back) return .performed } return .unavailable @@ -58,19 +58,19 @@ extension RunnerTests { { NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", mainOwned.bundleId ?? "") } else if let frame = try topNavigationBackFrame(app: app) { - return tapResolvedNavigationBackControl(app: app, frame: frame) + return try tapResolvedNavigationBackControl(app: app, frame: frame) } - return tapTopLeadingNavigationFallback( + return try tapTopLeadingNavigationFallback( app: app, windowQueryRejected: navigationRejectedQueryCount() != rejectedQueriesBefore ) #endif } - private func tapElementCenter(app: XCUIApplication, element: XCUIElement) { + private func tapElementCenter(app: XCUIApplication, element: XCUIElement) throws { let frame = element.frame if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = try tapAt(app: app, x: frame.midX, y: frame.midY) return } #if !os(tvOS) @@ -79,10 +79,10 @@ extension RunnerTests { } @MainActor - private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) -> InAppBackOutcome { + private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) throws -> InAppBackOutcome { guard SnapshotGeometry.isPositiveFinite(frame) else { return .unavailable } let context = synthesizedCoordinateContext(app: app) - guard dispatchNavigationTap(app: app, point: CGPoint(x: frame.midX, y: frame.midY), context: context) else { + guard try dispatchNavigationTap(app: app, point: CGPoint(x: frame.midX, y: frame.midY), context: context) else { return .unavailable } return .performed @@ -95,7 +95,7 @@ extension RunnerTests { app: XCUIApplication, point: CGPoint, context: SynthesizedCoordinateContext? - ) -> Bool { + ) throws -> Bool { switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) }) { @@ -104,8 +104,8 @@ extension RunnerTests { case .refused: return false case .xctestFallback: - let fallback = performGesture(app) { - tapAt(app: app, x: point.x, y: point.y) + let fallback = try performGesture(app) { + try tapAt(app: app, x: point.x, y: point.y) } if case .performed = fallback.outcome { return true } return false @@ -221,7 +221,7 @@ extension RunnerTests { private func tapTopLeadingNavigationFallback( app: XCUIApplication, windowQueryRejected: Bool - ) -> InAppBackOutcome { + ) throws -> InAppBackOutcome { #if os(iOS) guard let context = synthesizedCoordinateContext( app: app, @@ -233,7 +233,7 @@ extension RunnerTests { return .unavailable } let before = captureNavigationFallbackVisualState(app: app, context: context) - guard dispatchNavigationTap(app: app, point: point, context: context) else { + guard try dispatchNavigationTap(app: app, point: point, context: context) else { return .unavailable } return verifyNavigationFallbackOutcome(app: app, before: before, context: context) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 340e3278bf..4de569032c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -38,9 +38,9 @@ extension RunnerTests { message: String, context: SynthesizedCoordinateContext, releaseBehavior: ScrollReleaseBehavior? - ) -> Response { + ) throws -> Response { #if os(iOS) - return executeDragGesture( + return try executeDragGesture( activeApp: activeApp, x: x, y: y, @@ -52,7 +52,7 @@ extension RunnerTests { synthesized: (profile: scrollDragProfile(releaseBehavior: releaseBehavior), policyKind: .scroll) ) #else - return executeDragGesture( + return try executeDragGesture( activeApp: activeApp, x: x, y: y, @@ -77,7 +77,7 @@ extension RunnerTests { message: String, synthesizedContext: SynthesizedCoordinateContext? = nil, synthesized: (profile: SynthesizedDragProfile, policyKind: SynthesizedGesturePolicyKind)? = nil - ) -> Response { + ) throws -> Response { let durationMs = durationMs ?? runnerDefaultDragDurationMs let commandName = dragCommandName(message: message) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { @@ -86,7 +86,7 @@ extension RunnerTests { error: ErrorPayload(code: "INVALID_ARGS", message: "\(commandName) requires finite coordinates") ) } - if let synthesized, let synthesizedResponse = executeSynthesizedDragGesture( + if let synthesized, let synthesizedResponse = try executeSynthesizedDragGesture( activeApp: activeApp, x: x, y: y, @@ -100,7 +100,7 @@ extension RunnerTests { ) { return synthesizedResponse } - return executeCoordinateDrag( + return try executeCoordinateDrag( activeApp: activeApp, x: x, y: y, @@ -124,7 +124,7 @@ extension RunnerTests { holdDuration: TimeInterval, message: String, fallback: GestureFallback? = nil - ) -> Response { + ) throws -> Response { let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) let dragFrame = resolvedDragVisualizationFrame( app: activeApp, @@ -133,8 +133,8 @@ extension RunnerTests { x2: dragPoints.x2, y2: dragPoints.y2 ) - let (timing, outcome) = performGesture(activeApp) { - dragAt( + let (timing, outcome) = try performGesture(activeApp) { + try dragAt( app: activeApp, x: dragPoints.x, y: dragPoints.y, @@ -161,7 +161,7 @@ extension RunnerTests { context: SynthesizedCoordinateContext?, policyKind: SynthesizedGesturePolicyKind, profile: SynthesizedDragProfile - ) -> Response? { + ) throws -> Response? { #if os(iOS) let context = context ?? synthesizedCoordinateContext(app: activeApp) guard let plan = axFreeSynthesizedDragPlan( @@ -206,7 +206,7 @@ extension RunnerTests { case .performed(let timing): return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) case .xctestFallback(let fallbackMessage, let hint): - return executeCoordinateDrag( + return try executeCoordinateDrag( activeApp: activeApp, x: plan.points.x, y: plan.points.y, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 586cdc038c..e8935a77cd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -20,7 +20,7 @@ extension RunnerTests { } @MainActor - func executeSequence(command: Command, activeApp: XCUIApplication) -> Response { + func executeSequence(command: Command, activeApp: XCUIApplication) throws -> Response { guard let steps = command.steps, !steps.isEmpty else { return sequenceInvalidArgs("sequence requires at least one step") } @@ -45,8 +45,8 @@ extension RunnerTests { ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil - let execution = assembleSequenceExecution(steps: steps) { _, step in - performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) + let execution = try assembleSequenceExecution(steps: steps) { _, step in + try performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } return sequenceResponse(execution: execution, touchFrame: firstFrame) } @@ -58,8 +58,8 @@ extension RunnerTests { /// results.count == completedSteps + (failedStepIndex != nil ? 1 : 0). func assembleSequenceExecution( steps: [SequenceStep], - perform: (Int, SequenceStep) -> SequenceStepOutcome - ) -> SequenceExecutionResult { + perform: (Int, SequenceStep) throws -> SequenceStepOutcome + ) rethrows -> SequenceExecutionResult { var results: [SequenceStepResult] = [] var completedSteps = 0 var failedStepIndex: Int? @@ -67,7 +67,7 @@ extension RunnerTests { var gestureEndUptimeMs: Double? for (index, step) in steps.enumerated() { - let stepOutcome = perform(index, step) + let stepOutcome = try perform(index, step) if gestureStartUptimeMs == nil { gestureStartUptimeMs = stepOutcome.gestureStartUptimeMs } @@ -144,7 +144,7 @@ extension RunnerTests { _ step: SequenceStep, activeApp: XCUIApplication, synthesizedContext: SynthesizedCoordinateContext? = nil - ) -> SequenceStepOutcome { + ) throws -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 let longPressDurationMs = min(max(step.durationMs ?? 800, 16), 10000) @@ -171,15 +171,15 @@ extension RunnerTests { break } } - let (timing, outcome) = performGesture(activeApp) { + let (timing, outcome) = try performGesture(activeApp) { switch step.kind { case "doubleTap": // doubleTapAt per step, matching the behavior of the retired tapSeries doubleTap path. - return doubleTapAt(app: activeApp, x: x, y: y) + return try doubleTapAt(app: activeApp, x: x, y: y) case "longPress": - return longPressAt(app: activeApp, x: x, y: y, duration: longPressDurationMs / 1000.0) + return try longPressAt(app: activeApp, x: x, y: y, duration: longPressDurationMs / 1000.0) default: - return tapAt(app: activeApp, x: x, y: y) + return try tapAt(app: activeApp, x: x, y: y) } } return finishedSequenceStep(step, timing: timing, outcome: outcome) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 76e8c4ac03..4bc51d69cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -305,9 +305,9 @@ extension RunnerTests { } /// Runs `dispatch`, which posts to the app or pauses, so the next `input()` resolves again. - mutating func dispatch(_ dispatch: () -> T) -> T { + mutating func dispatch(_ dispatch: () throws -> T) rethrows -> T { lastRead = nil - return dispatch() + return try dispatch() } } @@ -415,7 +415,7 @@ extension RunnerTests { var valueLength = normalizedElementText(input.attributes.value).count // The tap also focuses the field, which a clear can reach with focus unconfirmed. #if !os(tvOS) - reader.dispatch { moveCaretToEnd(app: app, input: input) } + try reader.dispatch { try moveCaretToEnd(app: app, input: input) } #endif // An input that stops answering once focused (#3060) cannot be read back, so it gets one burst. guard try reader.input() != nil else { @@ -555,18 +555,13 @@ extension RunnerTests { /// Whether a bound target's input is proven gone: XCTest found no match for it, or a different /// input answers in its place, and nothing at its point carries its identity. A failed probe - /// proves nothing, and neither does a lit-panel refusal. Unbound targets and unidentified inputs + /// proves nothing; a lit-panel refusal propagates. Unbound targets and unidentified inputs /// without an element cannot be proven gone. - func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) -> Bool { + func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) throws -> Bool { guard let boundIdentity = target.boundIdentity else { return false } - let byIdentifier: XCUIElement? - do { - byIdentifier = try boundTextEntryInputQuery(app: app, identity: boundIdentity) - } catch { - return false - } + let byIdentifier = try boundTextEntryInputQuery(app: app, identity: boundIdentity) guard let query = byIdentifier ?? target.element else { return false } @@ -581,12 +576,12 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch try? probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { + switch try probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { case .absent: return true case .matches(let candidates): return candidates.allSatisfy { $0.identity != boundIdentity } - case .unavailable, nil: + case .unavailable: return false } } @@ -634,7 +629,7 @@ extension RunnerTests { return focusedTextInput(app: app).flatMap(read) } - private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { + private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) throws { #if os(tvOS) return #else @@ -653,7 +648,7 @@ extension RunnerTests { guard let frame = input.frameOnScreen, !frame.isEmpty else { return } - _ = tapAt( + _ = try tapAt( app: app, x: frame.minX + max(2, frame.width - 4), y: frame.minY + max(2, frame.height / 2) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index f0e6cbee74..fca641cc12 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -141,7 +141,7 @@ extension RunnerTests { let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint - _ = tapAt(app: app, x: tapPoint.x, y: tapPoint.y) + _ = try tapAt(app: app, x: tapPoint.x, y: tapPoint.y) // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index 97b3beb889..fa7f9da807 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -85,7 +85,7 @@ extension RunnerTests { /// `ResolvedTextEntryInput` already reads elements this way. It is read again after the /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. - func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) throws { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return @@ -95,18 +95,18 @@ extension RunnerTests { }) else { return } - retapAt(read: settled, app: app, element: element) + try retapAt(read: settled, app: app, element: element) #endif } /// The readiness repeat, at the read's screen box; a read that names none taps nothing (#3899). - func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { + func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) throws { guard let frame = read.frameOnScreen, !frame.isEmpty else { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=%@", read.frameOnScreen == nil ? "no-screen-frame" : "empty-frame") return } NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", frame.midX, frame.midY) - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = try tapAt(app: app, x: frame.midX, y: frame.midY) _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 0adae35fb3..7adcbac550 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -398,7 +398,8 @@ extension RunnerTests { while true { guard let input = try reader.input() else { let target = reader.target - let failure: TextEntryFailure? = target.boundIdentity != nil && !boundTextEntryInputIsGone(app: app, target: target) + let inputIsGone = try boundTextEntryInputIsGone(app: app, target: target) + let failure: TextEntryFailure? = target.boundIdentity != nil && !inputIsGone ? .commitNotObserved : nil if target.boundIdentity != nil && failure == nil { NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_INPUT_REMOVED_AFTER_DELIVERY") @@ -424,7 +425,7 @@ extension RunnerTests { expectedText: String? ) throws -> TextEntryResult { if try reader.input() == nil, reader.target.boundIdentity != nil { - guard boundTextEntryInputIsGone(app: app, target: reader.target) else { + guard try boundTextEntryInputIsGone(app: app, target: reader.target) else { return TextEntryResult( verified: nil, repaired: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift index b768656c87..17a05d9f6a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift @@ -60,7 +60,7 @@ extension RunnerTests { element: XCUIElement, action: String, resolvedFrame: CGRect? = nil - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { if let outcome = selectFocusedTvElement(app: app, element: element, action: action) { return outcome } @@ -72,7 +72,7 @@ extension RunnerTests { // XCUIElement.tap() can fail the whole XCTest after navigation because it // re-resolves the tapped element even after the app removed it. Keep the // selector target semantic, then activate its resolved stable screen point. - return tapAt(app: app, x: frame.midX, y: frame.midY) + return try tapAt(app: app, x: frame.midX, y: frame.midY) } return performElementTap(element) #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 6eda5aac1f..eab38e985b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -1,6 +1,151 @@ import AgentDeviceSnapshotPresentation import XCTest +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private final class LitPanelBookingScreen: NSObject { + @objc let displayID: UInt = 3 +} + +@MainActor +private final class LitPanelBookingProbe: NSObject { + static var originalValueForKey: IMP? + static let query = NSObject() + static var window: XCUIElement? + static var coordinate: XCUICoordinate? + static var windowFrame = CGRect.zero + static var orientation = RunnerInterfaceOrientation.landscapeLeft + static var anchor: NSObject? + static var point = CGPoint.zero + static var tappedPoint: CGPoint? + + @objc class func interfaceOrientation(forApplication application: AnyObject) -> Int { orientation } + + nonisolated override func value(forKey key: String) -> Any? { + MainActor.assumeIsolated { + switch key { + case "windows": return Self.query + case "allElementsBoundByIndex": return [Self.window!] + case "screen": return LitPanelBookingScreen() + case "exists": return true + case "frame": return NSValue(cgRect: Self.windowFrame) + default: break + } + let original = unsafeBitCast( + Self.originalValueForKey!, + to: (@convention(c) (AnyObject, Selector, NSString) -> AnyObject?).self + ) + return original(self, #selector(NSObject.value(forKey:)), key as NSString) + } + } + + @objc(coordinateWithNormalizedOffset:) + func coordinateWithNormalizedOffset(_ offset: CGVector) -> XCUICoordinate { + Self.anchor = self + Self.point = CGPoint( + x: Self.windowFrame.minX + offset.dx * Self.windowFrame.width, + y: Self.windowFrame.minY + offset.dy * Self.windowFrame.height + ) + return Self.coordinate! + } + + @objc(coordinateWithOffset:) + func coordinateWithOffset(_ offset: CGVector) -> XCUICoordinate { + Self.point.x += offset.dx + Self.point.y += offset.dy + return Self.coordinate! + } + + @objc func tap() { Self.tappedPoint = Self.point } +} + +extension RunnerTests { + @MainActor + func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { + app.launch() + defer { app.terminate() } + let query = app.windows + let window = query.firstMatch + LitPanelBookingProbe.window = window + LitPanelBookingProbe.coordinate = window.coordinate(withNormalizedOffset: .zero) + let windowClass = try XCTUnwrap(object_getClass(window)) + let coordinateClass = try XCTUnwrap(object_getClass(LitPanelBookingProbe.coordinate!)) + let valueForKey = try XCTUnwrap(class_getInstanceMethod(windowClass, #selector(NSObject.value(forKey:)))) + LitPanelBookingProbe.originalValueForKey = method_getImplementation(valueForKey) + let swaps: [(Method?, Method?)] = [ + (valueForKey, + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(NSObject.value(forKey:)))), + (class_getClassMethod(RunnerSynthesizedGesture.self, #selector(RunnerSynthesizedGesture.interfaceOrientation(forApplication:))), + class_getClassMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.interfaceOrientation(forApplication:)))), + (class_getInstanceMethod(windowClass, #selector(XCUIElement.coordinate(withNormalizedOffset:))), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.coordinateWithNormalizedOffset(_:)))), + (class_getInstanceMethod(coordinateClass, #selector(XCUICoordinate.withOffset(_:))), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.coordinateWithOffset(_:)))), + (class_getInstanceMethod(coordinateClass, #selector(XCUICoordinate.tap)), + class_getInstanceMethod(LitPanelBookingProbe.self, #selector(LitPanelBookingProbe.tap))), + ] + var originals: [(Method, IMP)] = [] + defer { + for (method, original) in originals.reversed() { method_setImplementation(method, original) } + LitPanelBookingProbe.window = nil + LitPanelBookingProbe.coordinate = nil + LitPanelBookingProbe.anchor = nil + LitPanelBookingProbe.originalValueForKey = nil + litPanelState.panel = nil + litPanelState.surface = nil + } + for (method, stub) in swaps { + let method = try XCTUnwrap(method) + let stub = try XCTUnwrap(stub) + originals.append((method, method_setImplementation(method, method_getImplementation(stub)))) + } + for (frame, orientation, expected) in [ + ( + CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, + CGRect(x: 0, y: 0, width: 951, height: 669) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, + CGRect(x: 0, y: 0, width: 669, height: 951) + ), + ( + CGRect(x: 80, y: 0, width: 180, height: 669), RunnerInterfaceOrientation.landscapeRight, + CGRect(x: 0, y: 0, width: 951, height: 669) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, + CGRect(x: 0, y: 0, width: 669, height: 951) + ), + ] { + LitPanelBookingProbe.windowFrame = frame + LitPanelBookingProbe.orientation = orientation + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.frame, frame) + XCTAssertEqual(synthesizedRotationFrame(fallback: booked.frame, interfaceOrientation: orientation), expected) + if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } + XCTAssertTrue(LitPanelBookingProbe.anchor === window) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + } + XCTAssertNil(LitPanelBookingProbe.tappedPoint) + + } +} +#endif + #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { /// One runner process can serve a foldable's `tap` and then a single-panel request; the second @@ -206,7 +351,6 @@ extension RunnerTests { let command = try runnerCommandFixture(#"{"command":"type",\#(panel)}"#) setRequestLitPanel(from: command) XCTAssertNotNil(try resolveTextEntryInput(app: app, target: target), "the walk answers") - setRequestLitPanel(from: command) litPanelWindowWalkIssueForTesting = true XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) @@ -214,14 +358,15 @@ extension RunnerTests { XCTAssertThrowsError(try clearedFieldRead(app: app, target: target), "a clear pass read propagates it too") { XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) } - XCTAssertFalse(boundTextEntryInputIsGone(app: app, target: target)) + XCTAssertThrowsError(try boundTextEntryInputIsGone(app: app, target: target)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } } - /// The walk answers but finds no input at the refresh point, so resolution reaches the - /// identifier query: on a lit-panel request it searches only the lit windows, so a field the - /// app-wide query finds, standing in for the dark panel's twin, is not resolved. + /// A bound input read refuses when every window is off the named panel, even if the app-wide + /// query can resolve the same input. @MainActor - func testBoundInputIdentifierQuerySearchesOnlyTheLitWindows() throws { + func testBoundInputIdentifierReadRefusesWhenAllWindowsAreOffTheLitPanel() throws { app.launchArguments = ["--agent-device-text-entry-regression"] app.launch() mainOwned.app = app @@ -235,41 +380,38 @@ extension RunnerTests { setRequestLitPanel(from: try runnerCommandFixture( #"{"command":"type","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# )) - guard let point = target.refreshPoint, - case .absent = try probeTextInputs(app: app, point: point, contentTranslation: .none) - else { - return XCTFail("the walk answers with no input at the point") + let point = try XCTUnwrap(target.refreshPoint) + XCTAssertThrowsError(try probeTextInputs(app: app, point: point, contentTranslation: .none)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertThrowsError(try resolveTextEntryInput(app: app, target: target)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) } - XCTAssertNil(try resolveTextEntryInput(app: app, target: target)) } /// With a lit panel named, selector and point reads search only the windows on that display; the /// launched app's windows are all on another one, so neither read may answer. @MainActor func testElementReadsSearchOnlyTheLitDisplaysWindows() throws { - var found: (unnamed: Bool, named: Bool, text: String?, namedText: String?) = (false, false, nil, nil) _ = try responseWithTheAppOffTheLitPanel( #"{"command":"tap","x":1,"y":1,"litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# ) { _ in - found.named = try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") - .element != nil - let unnamed = try runnerCommandFixture(#"{"command":"tap"}"#) - setRequestLitPanel(from: unnamed) + XCTAssertThrowsError(try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner")) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"tap"}"#)) let match = try findElement(app: app, selectorKey: "label", selectorValue: "Agent Device Runner") - found.unnamed = match.element != nil - if let frame = match.element?.frame { - found.text = try readTextAt(app: app, x: frame.midX, y: frame.midY) - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# - )) - found.namedText = try readTextAt(app: app, x: frame.midX, y: frame.midY) + let element = try XCTUnwrap(match.element) + let frame = element.frame + XCTAssertNotNil(try readTextAt(app: app, x: frame.midX, y: frame.midY)) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try readTextAt(app: app, x: frame.midX, y: frame.midY)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) } return Response(ok: true) } - XCTAssertTrue(found.unnamed, "the label exists in the app's own window") - XCTAssertNotNil(found.text) - XCTAssertFalse(found.named, "a selector read must not search off the lit display") - XCTAssertNil(found.namedText, "a point read must not search off the lit display") } /// A refused pin is a request refusal: it never reaches the capture plan, so it does not diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index b741c7d931..380c8c9199 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -170,7 +170,7 @@ extension RunnerTests { let alert = try requireResolvedAlert(within: RunnerTests.alertResolutionAllowance) XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last") - let response = handleAlert( + let response = try handleAlert( alert, action: "dismiss", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline) @@ -194,7 +194,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try requireResolvedAlert(within: RunnerTests.alertResolutionAllowance) - let response = handleAlert( + let response = try handleAlert( alert, action: "dismiss", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index f764124a0a..de2db7af2c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -45,15 +45,9 @@ extension RunnerTests { @MainActor func testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let alert = try resolveAlertBeforeTheCommand() - let response = handleAlert(alert, action: "accept", deadline: .distantPast) + defer { tearDownReplacementAlert() } + let alert = try launchReplacementAlert() + let response = try handleAlert(alert, action: "accept", deadline: .distantPast) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") XCTAssertTrue(app.alerts.firstMatch.exists) @@ -77,7 +71,7 @@ extension RunnerTests { } return true } - let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(1)) + let response = try handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(1)) XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.code, "ALERT_DEADLINE_EXCEEDED") XCTAssertTrue(app.alerts.firstMatch.exists) @@ -86,34 +80,22 @@ extension RunnerTests { @MainActor func testAlertActivationAfterDeadlineDoesNotTapTheOriginal() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let alert = try resolveAlertBeforeTheCommand() + defer { tearDownReplacementAlert() } + let alert = try launchReplacementAlert() let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame - let outcome = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantPast) + let outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantPast) XCTAssertNil(outcome, "an expired command must not synthesize a tap") - XCTAssertNil(activateAlertButton(alert, button: button, action: "accept", frame: .zero, deadline: .distantFuture)) + XCTAssertNil(try activateAlertButton(alert, button: button, action: "accept", frame: .zero, deadline: .distantFuture)) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") } @MainActor func testAlertActivationRefusesAnUnreadableHostWindowWithoutTapping() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let resolved = try resolveAlertBeforeTheCommand() + defer { tearDownReplacementAlert() } + let resolved = try launchReplacementAlert() let alert = RunnerAlert( root: resolved.root, ownerApp: resolved.ownerApp, @@ -127,8 +109,8 @@ extension RunnerTests { let options = XCTExpectedFailure.Options() options.issueMatcher = { $0.compactDescription.contains("agent-device-missing-window") } var outcome: RunnerInteractionOutcome? - XCTExpectFailure("The unreadable window is the condition under test", options: options) { - outcome = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) + try XCTExpectFailure("The unreadable window is the condition under test", options: options) { + outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) } guard case .unsupported? = outcome else { @@ -139,39 +121,24 @@ extension RunnerTests { @MainActor func testAlertDeadlineDuringSynthesisPreparationLeavesTheOriginalUntouched() throws { - app.launchArguments = ["--agent-device-alert-replacement-regression"] - app.launch() - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) - let alert = try resolveAlertBeforeTheCommand() + defer { tearDownReplacementAlert() } + let alert = try launchReplacementAlert() let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame - let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") - let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) - let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) - let original = method_getImplementation(method) - method_setImplementation(method, method_getImplementation(stub)) - defer { - AlertTapDeadlineStub.status = nil - AlertTapDeadlineStub.interruptionHandlingDisabled = nil - method_setImplementation(method, original) - } + try withAlertTapStub { + let outcome = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) - let outcome = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) + XCTAssertNil(outcome) + XCTAssertTrue(app.alerts.firstMatch.exists) + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") - XCTAssertNil(outcome) - XCTAssertTrue(app.alerts.firstMatch.exists) - XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") - - AlertTapDeadlineStub.status = RunnerTapSynthesisStatus(rawValue: 999) - let unknown = activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) - guard case .unsupported? = unknown else { - return XCTFail("an unknown native status must report failure rather than deadline expiry") + AlertTapDeadlineStub.status = RunnerTapSynthesisStatus(rawValue: 999) + let unknown = try activateAlertButton(alert, button: button, action: "accept", frame: frame, deadline: .distantFuture) + guard case .unsupported? = unknown else { + return XCTFail("an unknown native status must report failure rather than deadline expiry") } XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") + } } @MainActor @@ -189,7 +156,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) let alert = try resolveAlertBeforeTheCommand() - let response = handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) + let response = try handleAlert(alert, action: "accept", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) // The fixture keeps an animation in flight until a button is answered or its backstop, which // outlasts resolution plus activation, stops it, and an in-flight animation is what XCTest waits @@ -212,7 +179,7 @@ extension RunnerTests { try assertAlertActivationDisablesInterruptionHandling(alert) let deadline = Date().addingTimeInterval(RunnerTests.alertActivationDeadline) - let response = handleAlert(alert, action: "accept", deadline: deadline) + let response = try handleAlert(alert, action: "accept", deadline: deadline) XCTAssertEqual(consultedInterruptions, [], "alert activation consulted XCTest's interruption handling") XCTAssertTrue(response.ok, String(describing: response.error)) @@ -225,24 +192,42 @@ extension RunnerTests { private func assertAlertActivationDisablesInterruptionHandling(_ alert: RunnerAlert) throws { let button = try XCTUnwrap(alert.buttons.first { $0.label == "OK" }) let frame = button.frame + try withAlertTapStub { + AlertTapDeadlineStub.status = .deadlineExceeded + let outcome = try activateAlertButton( + alert, button: button, action: "accept", frame: frame, deadline: .distantFuture + ) + XCTAssertNil(outcome) + XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + } + } + + @MainActor + private func launchReplacementAlert() throws -> RunnerAlert { + app.launchArguments = ["--agent-device-alert-replacement-regression"] + app.launch() + XCTAssertTrue(app.alerts.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout)) + return try resolveAlertBeforeTheCommand() + } + + @MainActor + private func tearDownReplacementAlert() { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + @MainActor + private func withAlertTapStub(_ body: () throws -> Void) throws { let selector = NSSelectorFromString("synthesizeTapWithApplication:resolvedWindow:x:y:deadline:errorMessage:") let method = try XCTUnwrap(class_getClassMethod(RunnerSynthesizedGesture.self, selector)) let stub = try XCTUnwrap(class_getClassMethod(AlertTapDeadlineStub.self, selector)) - let original = method_getImplementation(method) - AlertTapDeadlineStub.status = .deadlineExceeded - AlertTapDeadlineStub.interruptionHandlingDisabled = nil - method_setImplementation(method, method_getImplementation(stub)) + let original = method_setImplementation(method, method_getImplementation(stub)) defer { AlertTapDeadlineStub.status = nil AlertTapDeadlineStub.interruptionHandlingDisabled = nil method_setImplementation(method, original) } - - let outcome = activateAlertButton( - alert, button: button, action: "accept", frame: frame, deadline: .distantFuture - ) - XCTAssertNil(outcome) - XCTAssertEqual(AlertTapDeadlineStub.interruptionHandlingDisabled, true) + try body() } @MainActor @@ -273,7 +258,7 @@ extension RunnerTests { while Date() < deadline, !fixtureAlert.exists { if allow.exists { let authorization = try requireResolvedAlert(deadline: deadline) - let response = handleAlert(authorization, action: "accept", deadline: deadline) + let response = try handleAlert(authorization, action: "accept", deadline: deadline) XCTAssertTrue(response.ok, String(describing: response.error)) } else { Thread.sleep(forTimeInterval: 0.25) @@ -322,7 +307,7 @@ extension RunnerTests { XCTAssertTrue(app.alerts.firstMatch.exists, "the replacement must remain visible") XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") let current = try resolveAlertBeforeTheCommand() - let inspection = handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) + let inspection = try handleAlert(current, action: "get", deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline)) XCTAssertTrue(inspection.ok) XCTAssertEqual(inspection.data?.items?.sorted(), ["Cancel", "OK"]) XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift index 85591434c9..cbdefc8023 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -93,7 +93,7 @@ extension RunnerTests { defer { terminateCrowdedScreen() } let alert = try requireResolvedAlert(within: RunnerTests.defaultAlertCommandTimeout) - let inspection = handleAlert(alert, action: "get", deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + let inspection = try handleAlert(alert, action: "get", deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) XCTAssertTrue(inspection.ok) XCTAssertEqual(inspection.data?.message, "Relaunch to continue") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift index 5e103b00c4..2e91292030 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SequenceExecutionTests.swift @@ -21,10 +21,10 @@ extension RunnerTests { } @MainActor - func testSequenceAcceptsDoubleTapKind() { + func testSequenceAcceptsDoubleTapKind() throws { // A doubleTap step missing coords must fail on the coords check, not the kind allowlist — // proving "doubleTap" passes validateSequenceStep without needing a device to execute on. - let response = executeSequenceForTest(steps: [ + let response = try executeSequenceForTest(steps: [ sequenceStep(kind: "doubleTap", x: nil) ]) XCTAssertEqual(response.ok, false) @@ -35,7 +35,7 @@ extension RunnerTests { @MainActor func testSequenceRejectsUnknownKind() throws { - let response = executeSequenceForTest(steps: [ + let response = try executeSequenceForTest(steps: [ sequenceStep(kind: "tap", x: 1, y: 2), sequenceStep(kind: "pinch", x: 3, y: 4), ]) @@ -46,16 +46,16 @@ extension RunnerTests { } @MainActor - func testSequenceRejectsEmpty() { - let response = executeSequenceForTest(steps: []) + func testSequenceRejectsEmpty() throws { + let response = try executeSequenceForTest(steps: []) XCTAssertEqual(response.ok, false) XCTAssertEqual(response.error?.code, "INVALID_ARGS") } @MainActor - func testSequenceRejectsTooManySteps() { + func testSequenceRejectsTooManySteps() throws { let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) } - let response = executeSequenceForTest(steps: steps) + let response = try executeSequenceForTest(steps: steps) XCTAssertEqual(response.ok, false) XCTAssertEqual(response.error?.code, "INVALID_ARGS") XCTAssertTrue(response.error?.message.contains("at most 20") ?? false) @@ -156,9 +156,9 @@ extension RunnerTests { /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without /// reaching the device executor (which is never invoked when validation rejects). @MainActor - private func executeSequenceForTest(steps: [SequenceStep]) -> Response { + private func executeSequenceForTest(steps: [SequenceStep]) throws -> Response { let command = makeSequenceCommand(steps: steps) - return executeSequence(command: command, activeApp: app) + return try executeSequence(command: command, activeApp: app) } /// Build a sequence Command via JSON so the test does not depend on the memberwise init's diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 19e4cdd194..8b2fc7aad9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) let failures = currentXCTestFailureCount() let startedAt = Date() - waitForTextEntryReadinessAfterTap(app: app, element: field) + try waitForTextEntryReadinessAfterTap(app: app, element: field) let elapsed = Date().timeIntervalSince(startedAt) XCTAssertFalse(didRecordXCTestFailure(since: failures)) XCTAssertTrue(keyboardFocusConfirmed(app: app, element: field)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift index f42559ae14..44e05347f0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -23,14 +23,14 @@ extension RunnerTests { let unknown = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .unknown)) XCTAssertNil(unknown.frameOnScreen) - retapAt(read: unknown, app: app, element: field) + try retapAt(read: unknown, app: app, element: field) XCTAssertEqual(presses.label, "presses=0", "an unknown content space dispatched a point") - retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)), app: app, element: field) + try retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)), app: app, element: field) XCTAssertEqual(presses.label, "presses=0", "a full-screen read dispatched a point off the field") let vector = CGSize(width: witnessFrame.midX - fieldFrame.midX, height: witnessFrame.midY - fieldFrame.midY) - retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))), app: app, element: field) + try retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))), app: app, element: field) XCTAssertEqual(presses.label, "presses=1", "the retap dispatched the raw snapshot center, not the screen-space one") } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 1918d62b50..f1e641560f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -352,10 +352,24 @@ extension RunnerTests { /// #2995: a `frame` read on a window the query says is gone records an XCTest issue. func testLitPanelWindowNeverAsksAnAbsentWindowForItsFrame() { let absent = DisplayWindowFixture(frame: Self.innerHome, displayID: 3, present: false) - XCTAssertEqual(resolveLitPanelWindow([absent]).failure, .unresolvedWindow) + let failure = resolveLitPanelWindow([absent]).failure + XCTAssertEqual(failure, .unresolvedWindow) + XCTAssertEqual(RunnerAppScreenCaptureFailure(failure).response.error?.code, "APP_SCREEN_WINDOW_UNRESOLVED") XCTAssertFalse(absent.resolved) } + func testLitPanelElementSearchIncludesWindowsPastTheBookingCap() { + let early = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let later = DisplayWindowFixture(frame: Self.innerHome, displayID: 3) + let windows: [NSObject] = [early] + (0.. { + const device = simulator(); // Both panels lit: panel power settles no single panel, so the runner keeps its historic route. answering({ result: { @@ -117,10 +118,18 @@ test('a multi-panel device whose panel power decides nothing sends no panel', as })), }, }); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + answering(IPHONE_DUO_OPEN); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3); + assert.equal( + mockPanelTable.mock.calls.length, + 2, + 'ambiguous power does not cache a single-panel verdict', + ); }); test('a multi-panel device whose panel table is incomplete sends no panel', async () => { + const device = simulator(); // The parser read one of two integrated panels, so the unread one may be the lit one. const [outer, inner] = IPHONE_DUO_CLOSED.result.displays; answering({ @@ -128,7 +137,10 @@ test('a multi-panel device whose panel table is incomplete sends no panel', asyn displays: [outer, { ...inner, nativeSize: undefined }], }, }); - assert.equal(await resolveRunnerRequestLitPanel(simulator(), {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + answering(IPHONE_DUO_OPEN); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3); + assert.equal(mockPanelTable.mock.calls.length, 2, 'an incomplete table remains multi-panel'); }); test('a device the fold fact refuses never costs a panel table read', async () => { From 7b7e0df86c7bae8539e59189b40b309ae923be2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 11:23:11 +0100 Subject: [PATCH 25/29] fix(ios): key lit-panel bookings by app and request identity --- .../RunnerRequestPanel.swift | 47 +++++- .../RunnerTests+CommandDispatch.swift | 3 +- .../RunnerTests+Interaction.swift | 14 +- .../RunnerTests+Navigation.swift | 3 +- .../UnitTests/RunnerRequestPanelTests.swift | 157 +++++++++++++----- .../RunnerTests+NavigationTests.swift | 16 ++ docs/adr/0025-foldable-apple-panels.md | 4 +- .../src/foldable/runner-panel.test.ts | 24 ++- .../src/foldable/runner-panel.ts | 10 +- 9 files changed, 211 insertions(+), 67 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index fe9573cd39..3767d7e09f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -2,8 +2,13 @@ import XCTest // The lit panel of the command in flight (ADR 0025), main-owned and written behind main work a // previous command left queued, so that work still reads its own command's panel. +struct LitPanelBookingKey: Equatable { + let bundleID: String + let requestID: UUID +} + struct LitPanelSurface { - let app: XCUIApplication + let key: LitPanelBookingKey let displayID: UInt let window: XCUIElement let frame: CGRect @@ -11,12 +16,15 @@ struct LitPanelSurface { /// The command's lit panel and what it booked on it. Read and written only on the main thread. final class RunnerLitPanelState { + var requestID: UUID? var panel: RunnerLitPanel? var surface: LitPanelSurface? } extension RunnerTests { - func setRequestLitPanel(from command: Command) { + @discardableResult + func setRequestLitPanel(from command: Command) -> UUID { + let requestID = UUID() var panel: RunnerLitPanel? if let fact = command.litPanel, fact.displayId != 0, fact.widthPt.isFinite, fact.heightPt.isFinite, fact.widthPt > 0, fact.heightPt > 0 @@ -25,6 +33,7 @@ extension RunnerTests { } let state = litPanelState let write = { + state.requestID = requestID state.panel = panel state.surface = nil } @@ -33,6 +42,22 @@ extension RunnerTests { } else { DispatchQueue.main.async(execute: write) } + return requestID + } + + func endRequestLitPanel(_ requestID: UUID) { + let state = litPanelState + let clear = { + guard state.requestID == requestID else { return } + state.requestID = nil + state.panel = nil + state.surface = nil + } + if Thread.isMainThread { + clear() + } else { + DispatchQueue.main.async(execute: clear) + } } /// The lit panel the current request named, or nil for a single-panel request. Main only. @@ -41,7 +66,12 @@ extension RunnerTests { /// Books `app`'s lit-panel window for the request, or returns why it cannot. func bookRequestLitPanelSurface(app: XCUIApplication) -> RunnerAppScreenCaptureFailure? { guard let litPanel = requestLitPanel else { return nil } - if let booked = litPanelState.surface, booked.app === app { return nil } + guard let requestID = litPanelState.requestID, + let bundleID = safely("LIT_PANEL_APP_IDENTITY", { app.value(forKey: "bundleID") as? String })?.trimmedNonEmpty + else { return .unresolvedScreen } + let key = LitPanelBookingKey(bundleID: bundleID, requestID: requestID) + if litPanelState.surface?.key == key { return nil } + litPanelState.surface = nil var window: AnyObject? var frame = CGRect.null var screen: AnyObject? @@ -49,13 +79,13 @@ extension RunnerTests { guard failure == .none, let element = window as? XCUIElement else { return RunnerAppScreenCaptureFailure(failure == .none ? .unresolvedWindow : failure) } - litPanelState.surface = LitPanelSurface(app: app, displayID: litPanel.displayID, window: element, frame: frame) + litPanelState.surface = LitPanelSurface(key: key, displayID: litPanel.displayID, window: element, frame: frame) return nil } func requireRequestLitPanelSurface(app: XCUIApplication) throws -> LitPanelSurface { if let refusal = bookRequestLitPanelSurface(app: app) { throw refusal } - guard let surface = litPanelState.surface, surface.app === app else { + guard let surface = litPanelState.surface else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } return surface @@ -75,6 +105,13 @@ extension RunnerTests { #endif } guard walk.issues.isEmpty, walk.exceptionMessage == nil, let windows, !windows.isEmpty else { + for issue in walk.issues { + NSLog("AGENT_DEVICE_RUNNER_LIT_PANEL_WALK_ISSUE=%@", issue.compactDescription) + } + NSLog( + "AGENT_DEVICE_RUNNER_LIT_PANEL_WALK_REFUSED issues=%ld exception=%@ windows=%ld", + walk.issues.count, walk.exceptionMessage ?? "none", windows?.count ?? -1 + ) throw RunnerAppScreenCaptureFailure.unresolvedScreen } return windows diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index 2af1c4faf0..c3edc1ab95 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -7,7 +7,8 @@ extension RunnerTests { func executeAccepted(command: Command) throws -> Response { commandJournal.start(command: command) pendingTargetActivation = nil - setRequestLitPanel(from: command) + let panelRequestID = setRequestLitPanel(from: command) + defer { endRequestLitPanel(panelRequestID) } do { let response = try executeDispatched(command: command) commandJournal.finish(command: command, response: response) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index bbcfed5dc9..fc006a6b2e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -204,10 +204,13 @@ extension RunnerTests { .flatMap { $0.matching(predicate).allElementsBoundByIndex } .filter(\.exists) let facts = try matches.map { element in - SelectorCandidateFacts( - isHittable: element.isHittable, - hasTappableFrame: try hasTappableFrame(app: app, element: element), - containsExpectedPoint: expectedPoint.map(element.frame.contains) ?? true + let isHittable = element.isHittable + let containsExpectedPoint = expectedPoint.map(element.frame.contains) ?? true + return SelectorCandidateFacts( + isHittable: isHittable, + hasTappableFrame: allowNonHittableFallback && !isHittable && containsExpectedPoint + ? try hasTappableFrame(app: app, element: element) : false, + containsExpectedPoint: containsExpectedPoint ) } switch classifyDirectSelectorCandidates( @@ -277,8 +280,7 @@ extension RunnerTests { /// tree, that read fails the same way after XCTest's own retries (about 2s on Bluesky's feed). func resolveRunnerWindowElement(app: XCUIApplication) -> (window: XCUIElement, frame: CGRect)? { if requestLitPanel != nil { - _ = bookRequestLitPanelSurface(app: app) - guard let booked = litPanelState.surface, booked.app === app else { return nil } + guard bookRequestLitPanelSurface(app: app) == nil, let booked = litPanelState.surface else { return nil } return (booked.window, booked.frame) } for window in app.windows.allElementsBoundByIndex where window.exists { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 3576fb2258..78b2d0c62c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -46,7 +46,8 @@ extension RunnerTests { return .performed #else let rejectedQueriesBefore = navigationRejectedQueryCount() - let buttons = app.navigationBars.buttons.allElementsBoundByIndex + let buttons = try elementQueries(app: app, matching: .navigationBar) + .flatMap { $0.buttons.allElementsBoundByIndex } if let back = buttons.first(where: { $0.isHittable }) { // `isHittable` promises a hittable point, not a hittable center: a bar button partly under an // overlay keeps the element tap, which aims at the point XCTest computed. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index eab38e985b..9af9d71548 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -12,6 +12,8 @@ private final class LitPanelBookingScreen: NSObject { private final class LitPanelBookingProbe: NSObject { static var originalValueForKey: IMP? static let query = NSObject() + static var windowListReads = 0 + static var maxWindowListReads: Int? static var window: XCUIElement? static var coordinate: XCUICoordinate? static var windowFrame = CGRect.zero @@ -26,7 +28,10 @@ private final class LitPanelBookingProbe: NSObject { MainActor.assumeIsolated { switch key { case "windows": return Self.query - case "allElementsBoundByIndex": return [Self.window!] + case "allElementsBoundByIndex": + Self.windowListReads += 1 + if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] } + return [Self.window!] case "screen": return LitPanelBookingScreen() case "exists": return true case "frame": return NSValue(cgRect: Self.windowFrame) @@ -63,6 +68,111 @@ private final class LitPanelBookingProbe: NSObject { extension RunnerTests { @MainActor func testLitPanelBookingAndCoordinateFallbackShareScreenPointsForPartialWindows() throws { + try withLitPanelBookingProbe { window in + for (frame, orientation, expected) in [ + ( + CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, + CGRect(x: 0, y: 0, width: 951, height: 669) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, + CGRect(x: 0, y: 0, width: 669, height: 951) + ), + ( + CGRect(x: 80, y: 0, width: 180, height: 669), RunnerInterfaceOrientation.landscapeRight, + CGRect(x: 0, y: 0, width: 951, height: 669) + ), + ( + CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, + CGRect(x: 0, y: 0, width: 669, height: 951) + ), + ] { + LitPanelBookingProbe.windowFrame = frame + LitPanelBookingProbe.orientation = orientation + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let booked = try XCTUnwrap(litPanelState.surface) + XCTAssertTrue(booked.window === window) + XCTAssertEqual(booked.frame, frame) + XCTAssertEqual(synthesizedRotationFrame(fallback: booked.frame, interfaceOrientation: orientation), expected) + if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } + XCTAssertTrue(LitPanelBookingProbe.anchor === window) + XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) + } + LitPanelBookingProbe.tappedPoint = nil + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + } + XCTAssertNil(LitPanelBookingProbe.tappedPoint) + + } + } + + @MainActor + func testLitPanelBookingUsesBundleIdentityAcrossProxiesAndRequests() throws { + try withLitPanelBookingProbe { window in + let bundleID = try XCTUnwrap(app.value(forKey: "bundleID") as? String) + let sameApp = XCUIApplication(bundleIdentifier: bundleID) + let otherApp = XCUIApplication(bundleIdentifier: "com.apple.springboard") + let secondWindow = otherApp.windows.firstMatch + let command = try runnerCommandFixture(#"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"#) + LitPanelBookingProbe.windowFrame = CGRect(x: 80, y: 0, width: 871, height: 669) + let firstRequest = setRequestLitPanel(from: command) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let firstReads = LitPanelBookingProbe.windowListReads + XCTAssertNil(bookRequestLitPanelSurface(app: sameApp)) + XCTAssertEqual(LitPanelBookingProbe.windowListReads, firstReads, "a second proxy of the same bundle reuses this request's booking") + LitPanelBookingProbe.window = secondWindow + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 80, width: 669, height: 871) + XCTAssertNil(bookRequestLitPanelSurface(app: otherApp)) + XCTAssertTrue(try requireRequestLitPanelSurface(app: otherApp).window === secondWindow) + XCTAssertGreaterThan(LitPanelBookingProbe.windowListReads, firstReads) + LitPanelBookingProbe.window = window + let nextRequest = setRequestLitPanel(from: command) + endRequestLitPanel(firstRequest) + XCTAssertNil(bookRequestLitPanelSurface(app: sameApp)) + XCTAssertTrue(try requireRequestLitPanelSurface(app: app).window === window) + XCTAssertEqual(litPanelState.surface?.key.requestID, nextRequest) + endRequestLitPanel(nextRequest) + XCTAssertNil(litPanelState.surface) + XCTAssertNil(requestLitPanel) + } + } + + @MainActor + func testAcceptedRequestEndsItsLitPanelBooking() throws { + try withLitPanelBookingProbe { _ in + let command = try runnerCommandFixture(#"{"command":"uptime","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"#) + _ = try executeAccepted(command: command) + XCTAssertNil(requestLitPanel, "finished commands cannot leave a panel for unrelated main work") + XCTAssertNil(litPanelState.surface) + } + } + + @MainActor + func testDuplicateSelectorRefusesWithoutTryingToBookATapPoint() throws { + app.launchArguments = ["--agent-device-selector-read-regression"] + try withLitPanelBookingProbe { _ in + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951) + LitPanelBookingProbe.maxWindowListReads = 1 + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + let match = try findElement(app: app, selectorKey: "id", selectorValue: "agent-device-selector-read-duplicate") + XCTAssertTrue(match.isAmbiguous) + XCTAssertNil(match.element) + XCTAssertEqual(LitPanelBookingProbe.windowListReads, 1, "ambiguity does not require a coordinate booking") + } + } + + @MainActor + private func withLitPanelBookingProbe(_ body: (XCUIElement) throws -> Void) throws { app.launch() defer { app.terminate() } let query = app.windows @@ -92,6 +202,7 @@ extension RunnerTests { LitPanelBookingProbe.coordinate = nil LitPanelBookingProbe.anchor = nil LitPanelBookingProbe.originalValueForKey = nil + LitPanelBookingProbe.maxWindowListReads = nil litPanelState.panel = nil litPanelState.surface = nil } @@ -100,48 +211,8 @@ extension RunnerTests { let stub = try XCTUnwrap(stub) originals.append((method, method_setImplementation(method, method_getImplementation(stub)))) } - for (frame, orientation, expected) in [ - ( - CGRect(x: 80, y: 0, width: 871, height: 669), RunnerInterfaceOrientation.landscapeLeft, - CGRect(x: 0, y: 0, width: 951, height: 669) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portrait, - CGRect(x: 0, y: 0, width: 669, height: 951) - ), - ( - CGRect(x: 80, y: 0, width: 180, height: 669), RunnerInterfaceOrientation.landscapeRight, - CGRect(x: 0, y: 0, width: 951, height: 669) - ), - ( - CGRect(x: 0, y: 80, width: 669, height: 871), RunnerInterfaceOrientation.portraitUpsideDown, - CGRect(x: 0, y: 0, width: 669, height: 951) - ), - ] { - LitPanelBookingProbe.windowFrame = frame - LitPanelBookingProbe.orientation = orientation - LitPanelBookingProbe.tappedPoint = nil - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# - )) - XCTAssertNil(bookRequestLitPanelSurface(app: app)) - let booked = try XCTUnwrap(litPanelState.surface) - XCTAssertTrue(booked.window === window) - XCTAssertEqual(booked.frame, frame) - XCTAssertEqual(synthesizedRotationFrame(fallback: booked.frame, interfaceOrientation: orientation), expected) - if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } - XCTAssertTrue(LitPanelBookingProbe.anchor === window) - XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) - } - LitPanelBookingProbe.tappedPoint = nil - setRequestLitPanel(from: try runnerCommandFixture( - #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# - )) - XCTAssertThrowsError(try tapAt(app: app, x: 200, y: 150)) { - XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) - } - XCTAssertNil(LitPanelBookingProbe.tappedPoint) - + LitPanelBookingProbe.windowListReads = 0 + try body(window) } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 1f72efc51d..2b987ae36a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -11,6 +11,22 @@ extension RunnerTests { try assertInAppBackDispatch(navigationBar: false) } + @MainActor + func testNavigationBarBackDoesNotChooseAControlOffTheLitPanel() throws { + let status = try launchBackFixture(["--agent-device-back-regression"], navigationBar: true) + defer { + setRequestLitPanel(from: try! runnerCommandFixture(#"{"command":"uptime"}"#)) + endBackFixture() + } + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"backInApp","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertThrowsError(try tapInAppBackControl(app: app)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + XCTAssertNotEqual(status.label, "Back performed") + } + /// A navigation bar button keeps XCTest's element tap, which aims at its computed hittable point. @MainActor func testNavigationBarBackKeepsTheElementTap() throws { diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 8bd7dcc0d6..c806000bc7 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -720,11 +720,11 @@ Android reports the app window on the next snapshot instead of a panel. SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. +Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. A table read that succeeds but spends the request budget refuses with `COMMAND_FAILED`, `reason: panel-probe-budget-exhausted` and `dispatched: no`, without a runner error code. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. With a named panel, the runner walks at most eight windows to book one; element search reads every window that reports the lit display. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. -The request books the window before it acts. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search rereads the complete window list for each read, including windows beyond the booking cap, because mutations can open another window. If the list cannot be read or contains no proven lit windows, the search throws `APP_SCREEN_UNRESOLVED` instead of answering no match. A lit-panel refusal during text-entry confirmation propagates with its typed code. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. +The request books the window by bundle identity and request identity before it acts, and releases the booking when the request ends. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search rereads the complete window list for each read, including windows beyond the booking cap, because mutations can open another window. If the list cannot be read or contains no proven lit windows, the search throws `APP_SCREEN_UNRESOLVED` instead of answering no match. A lit-panel refusal during text-entry confirmation propagates with its typed code. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. A `customActions` request with no explicit backend pin implies the private AX pin, so its refusal names `customActions`. diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index 821414a491..10e1845ca2 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -88,12 +88,21 @@ test('a device with one panel sends no panel and is never asked again', async () assert.equal(mockPanelTable.mock.calls.length, reads, 'one panel is hardware, probed once'); }); -test('a panel table that would not answer is not asked again by the next request', async () => { - const device = simulator(); - mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); - assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); - assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); - assert.equal(mockPanelTable.mock.calls.length, 1); +test('a failed first panel read backs off temporarily and probes again after expiry', async () => { + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000); + try { + const device = simulator(); + mockPanelTable.mockRejectedValue(new Error('devicectl does not report displays')); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(await resolveRunnerRequestLitPanel(device, {}), undefined); + assert.equal(mockPanelTable.mock.calls.length, 1); + now.mockReturnValue(6_001); + answering(IPHONE_DUO_OPEN); + assert.equal((await resolveRunnerRequestLitPanel(device, {}))?.displayId, 3); + assert.equal(mockPanelTable.mock.calls.length, 2); + } finally { + now.mockRestore(); + } }); test('a cancelled panel-table read leaves the next request free to ask', async () => { @@ -196,7 +205,8 @@ test('a panel read that spends the whole request budget refuses instead of sendi runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 20 }), ), (error: AppError) => { - assert.equal(error.details?.reason, 'panel-table-unreadable'); + assert.equal(error.details?.reason, 'panel-probe-budget-exhausted'); + assert.equal(error.details?.runnerErrorCode, undefined); assert.equal(error.details?.dispatched, 'no'); return true; }, diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 53c47dd433..45f8a31310 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -63,9 +63,15 @@ export async function nameRunnerRequestLitPanel( if (command.timeoutMs === undefined) return { ...command, litPanel }; const timeoutMs = command.timeoutMs - (Date.now() - startedAt); if (timeoutMs < 1) { - throw panelTableRefusal( - device, + throw new AppError( + 'COMMAND_FAILED', `Reading the panel table of ${device.name} spent the request's ${command.timeoutMs} ms, so nothing was sent`, + { + deviceId: device.id, + reason: 'panel-probe-budget-exhausted', + dispatched: 'no', + hint: 'Increase the request timeout to leave time for the command after reading the panel table.', + }, ); } return { ...command, litPanel, timeoutMs }; From c1ad1398334b79274e227f28ae898752943970f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 11:35:58 +0100 Subject: [PATCH 26/29] docs(adr): scope the panel-probe budget refusal to a named lit panel --- docs/adr/0025-foldable-apple-panels.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index c806000bc7..7ff6864875 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -720,7 +720,7 @@ Android reports the app window on the next snapshot instead of a panel. SpringBoard keeps windows on both panels. Its first window is the outer panel's status bar, whichever panel is lit. The runner booked gestures, the synthesized reference frame, the snapshot viewport, and the capture display against that window. A home-screen gesture could therefore land on a dark panel. -Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. A table read that succeeds but spends the request budget refuses with `COMMAND_FAILED`, `reason: panel-probe-budget-exhausted` and `dispatched: no`, without a runner error code. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. +Decision: the host names the lit panel in each runner request that resolves a screen. `RUNNER_COMMAND_RESOLVES_SURFACE` in `packages/apple-runner/src/runner-command-traits.ts` is the one list of these commands: `tap`, `longPress`, `drag`, `type`, `swipe`, `scroll`, `findText`, `querySelector`, `readText`, `snapshot`, `screenshot`, `backInApp`, `backSystem`, `gesture`, `gestureViewport`, `appSwitcher`, `keyboardDismiss`, `sequence`, and `recordStart`. The request field is `litPanel` (`displayId`, `widthPt`, `heightPt`). The panel table is read once per request because a fold changes the answer. The read never outlasts the request's `timeoutMs`. A multi-panel table read that names the lit panel but spends the request budget refuses with `COMMAND_FAILED`, `reason: panel-probe-budget-exhausted` and `dispatched: no`, without a runner error code. The host sends the field only for a multi-panel simulator whose panel power decides exactly one panel. The runner gets what the read left of `timeoutMs`. A device known to be multi-panel whose table does not answer is refused with `COMMAND_FAILED`, `reason: panel-table-unreadable` and `runnerErrorCode: APP_SCREEN_UNRESOLVED`, before anything is sent. Only a device whose panel count was never read keeps the historic first-window route unchanged. With a named panel, the runner walks at most eight windows to book one; element search reads every window that reports the lit display. It books the first window on the lit display whose frame is the panel's native box, upright or turned. If no window fills the panel, it books the largest window on the lit display. A window that the walk cannot place refuses the request with `APP_SCREEN_UNRESOLVED`, even if a full-panel window follows it. A list past the cap with no full-panel window refuses the same way. An app whose windows are all on another display is `APP_SCREEN_OFF_LIT_PANEL`. An app with no window at all is `APP_SCREEN_WINDOW_UNRESOLVED`. A window list that cannot be read is `APP_SCREEN_UNRESOLVED`, so it never reaches the system-surface fallback. From 294c7c6acbc0764661b91bde8c85f0dfbf07bfde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 12:10:02 +0100 Subject: [PATCH 27/29] fix(ios): refuse unbooked lit-panel reference frames --- .../RunnerScrollViewportPolicy.swift | 4 +- .../RunnerTests+AXSnapshotFallback.swift | 2 +- .../RunnerTests+Alert.swift | 21 ++++-- .../RunnerTests+CommandExecution.swift | 53 ++++++--------- .../RunnerTests+GestureExecution.swift | 4 +- .../RunnerTests+Interaction.swift | 35 +++++----- .../RunnerTests+Navigation.swift | 10 +-- .../RunnerTests+ScrollDragExecution.swift | 12 ++-- .../RunnerTests+SequenceExecution.swift | 14 ++-- .../RunnerTests+Snapshot.swift | 4 +- .../RunnerTests+SnapshotAcquisition.swift | 25 ++++--- .../RunnerTests+SnapshotCapturePlan.swift | 2 +- ...RunnerTests+SynthesizedGesturePolicy.swift | 23 +++++-- .../RunnerTests+SynthesizedInteraction.swift | 53 ++++++++------- .../RunnerTests+TypeExecution.swift | 7 +- .../UnitTests/RunnerRequestPanelTests.swift | 40 ++++++++++- .../RunnerTests+InteractionTests.swift | 2 +- ...rTests+SynthesizedGesturePolicyTests.swift | 14 ++-- ...nerTests+SynthesizedInteractionTests.swift | 10 +-- .../src/foldable/runner-panel.test.ts | 68 ++++++++++++++++++- .../src/foldable/runner-panel.ts | 1 + 21 files changed, 260 insertions(+), 144 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index d6921c5269..955f9389fd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -190,7 +190,7 @@ extension RunnerTests { func resolvedScrollViewport( app: XCUIApplication, context: SynthesizedCoordinateContext - ) -> RunnerScrollViewport { + ) throws -> RunnerScrollViewport { #if os(iOS) // Every scroll reports its decision, including the two ways it avoids reading the keyboard at // all: unavailable accessibility health, and a probe that finds no keyboard. @@ -223,7 +223,7 @@ extension RunnerTests { } return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip) #else - let fallbackFrame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + let fallbackFrame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed) #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index d212491567..ad05f5c884 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -290,7 +290,7 @@ extension RunnerTests { timeout: 1, timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport") ) { - self.safeSnapshotViewport(app: app, readingOrientation: true) + try self.safeSnapshotViewport(app: app, readingOrientation: true) } if case .missing = reported { return fallback diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 0d26d8fc13..16d6ca5d77 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -224,13 +224,20 @@ extension RunnerTests { } hostContext = context } - switch synthesizedTapAt( - app: alert.ownerApp, - x: frame.midX, - y: frame.midY, - context: hostContext, - deadline: deadline - ) { + let attempt: RunnerInteractionOutcome? + do { + attempt = try synthesizedTapAt( + app: alert.ownerApp, + x: frame.midX, + y: frame.midY, + context: hostContext, + deadline: deadline + ) + } catch { + activationError = error + return + } + switch attempt { case .performed?: typealias WaitForQuiescence = @convention(c) (NSObject, Selector, Bool) -> Void let waitForQuiescence = unsafeBitCast( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 5a6e017300..4bc64ae5f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -86,7 +86,7 @@ extension RunnerTests { // negative x), so it happily "contains" unreachable coordinates. // The DECISION lives in TapPointPolicy (golden parity table with the // TS twin); onScreenWindowFrame only supplies the frame. - if !match.usedNonHittableFallback + if try !match.usedNonHittableFallback && !TapPointPolicy.isAllowed( elementFrame: frame, windowFrame: onScreenWindowFrame(app: activeApp) @@ -98,23 +98,14 @@ extension RunnerTests { } let isTextEntry = isTextEntryElement(element) let touchPoint = expectedPoint ?? CGPoint(x: frame.midX, y: frame.midY) - let touchFrame = resolvedTouchVisualizationFrame( + let touchFrame = try resolvedTouchVisualizationFrame( app: activeApp, x: touchPoint.x, y: touchPoint.y ) var fallback: GestureFallback? if command.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedTapAt( - app: activeApp, - x: touchPoint.x, - y: touchPoint.y, - context: context - ) - }) { + switch try performSynthesizedTapGesture(app: activeApp, x: touchPoint.x, y: touchPoint.y) { case .performed(let timing): if isTextEntry { try waitForTextEntryReadinessAfterTap(app: activeApp, element: element) @@ -194,11 +185,7 @@ extension RunnerTests { } var fallback: GestureFallback? if command.synthesized == true { - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - }) { + switch try performSynthesizedTapGesture(app: activeApp, x: x, y: y) { case .performed(let timing): rememberTextEntryTap(textInput) return gestureResponse(message: "tapped", timing: timing) @@ -208,7 +195,7 @@ extension RunnerTests { return refusedTapResponse(message, hint: hint) } } - let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) + let touchFrame = try resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = try performGesture(activeApp) { try tapAt(app: activeApp, x: x, y: y) } if let response = unsupportedResponse(for: outcome) { clearRememberedTextEntryTap() @@ -228,7 +215,7 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "mouseClick requires x and y")) } - let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) + let touchFrame = try resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) do { // mouseClick throws (it has no RunnerInteractionOutcome), so it keeps raw measureGesture // and only routes the success payload through gestureResponse. @@ -255,9 +242,9 @@ extension RunnerTests { let fallback: GestureFallback? #if os(iOS) let policyKind = SynthesizedGesturePolicyKind.coordinateLongPress - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) + let context = try synthesizedCoordinateContext(app: activeApp) + switch try performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { + try synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) }) { case .performed(let timing): let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) } @@ -270,7 +257,7 @@ extension RunnerTests { #else fallback = nil #endif - let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) + let touchFrame = try resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = try performGesture(activeApp) { try longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) } @@ -306,13 +293,13 @@ extension RunnerTests { else { return invalidScrollDirectionResponse(commandName: "scroll") } - guard let scrollContext = synthesizedCoordinateContext(app: activeApp) else { + guard let scrollContext = try synthesizedCoordinateContext(app: activeApp) else { return Response( ok: false, error: ErrorPayload(message: "scroll could not resolve a usable interaction frame") ) } - let viewport = resolvedScrollViewport(app: activeApp, context: scrollContext) + let viewport = try resolvedScrollViewport(app: activeApp, context: scrollContext) let defaults = runnerDragCommandDefaults(command) switch viewport.gestureDispatch( direction: direction, @@ -363,7 +350,7 @@ extension RunnerTests { return invalidScrollDirectionResponse(commandName: "desktopScroll") } let appFrame = activeApp.frame - let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: appFrame) + let frame = try resolvedTouchReferenceFrame(app: activeApp, appFrame: appFrame) guard frame.width > 0, frame.height > 0 else { return Response( ok: false, @@ -392,7 +379,7 @@ extension RunnerTests { guard scrollDurationIsValid(command.durationMs) else { return invalidScrollDurationResponse(commandName: "desktopScroll") } - let touchFrame = resolvedTouchVisualizationFrame( + let touchFrame = try resolvedTouchVisualizationFrame( app: activeApp, x: localX, y: localY @@ -444,13 +431,13 @@ extension RunnerTests { } // swipe returns an optional frame (tvOS-only) rather than a RunnerInteractionOutcome, so it // keeps raw measureGesture and only routes the success payload through gestureResponse. - var executedFrame: DragVisualizationFrame? + var executedFrame: Result = .success(nil) let timing = measureGesture { withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { - executedFrame = swipe(app: activeApp, direction: direction) + executedFrame = Result { try swipe(app: activeApp, direction: direction) } } } - guard let dragFrame = executedFrame else { + guard let dragFrame = try executedFrame.get() else { return Response(ok: false, error: ErrorPayload(message: "swipe is only supported on tvOS")) } return gestureResponse(message: "swiped", timing: timing, frame: .drag(dragFrame)) @@ -565,7 +552,7 @@ extension RunnerTests { return Response(ok: false, error: error) } case .backSystem: - if performSystemBackAction(app: activeApp) { + if try performSystemBackAction(app: activeApp) { return Response(ok: true, data: DataPayload(message: "backSystem")) } return Response(ok: false, error: ErrorPayload(message: "system back is not available")) @@ -575,7 +562,7 @@ extension RunnerTests { case .rotate: return executeRotateCommand(command) case .appSwitcher: - performAppSwitcherGesture(app: activeApp) + try performAppSwitcherGesture(app: activeApp) return Response(ok: true, data: DataPayload(message: "appSwitcher")) case .actionButton: guard pressActionButton() else { @@ -705,7 +692,7 @@ extension RunnerTests { return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } case .gestureViewport: - let frame = gestureViewportFrame(app: activeApp) + let frame = try gestureViewportFrame(app: activeApp) guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { return Response(ok: false, error: ErrorPayload(code: "COMMAND_FAILED", message: "Active app interaction viewport is unavailable")) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift index 14642eac1c..b148a21c88 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift @@ -6,9 +6,9 @@ extension RunnerTests { ProcessInfo.processInfo.systemUptime * 1000 } - func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) { + func measureGesture(_ action: () throws -> Void) rethrows -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) { let gestureStartUptimeMs = currentUptimeMs() - action() + try action() return (gestureStartUptimeMs, currentUptimeMs()) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index fc006a6b2e..dd469d9c6a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -66,46 +66,46 @@ extension RunnerTests { let usedNonHittableFallback: Bool } - func performBackGesture(app: XCUIApplication) { + func performBackGesture(app: XCUIApplication) throws { if pressTvRemote(.menu) { return } - performCoordinateBackGesture(app: app) + try performCoordinateBackGesture(app: app) } - private func performCoordinateBackGesture(app: XCUIApplication) { + private func performCoordinateBackGesture(app: XCUIApplication) throws { #if !os(tvOS) - let target = resolveRunnerWindow(app: app).window ?? app + let target = try resolveRunnerWindow(app: app).window ?? app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5)) let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5)) start.press(forDuration: 0.05, thenDragTo: end) #endif } - func performSystemBackAction(app: XCUIApplication) -> Bool { + func performSystemBackAction(app: XCUIApplication) throws -> Bool { #if os(macOS) return false #else if pressTvRemote(.menu) { return true } - performBackGesture(app: app) + try performBackGesture(app: app) return true #endif } - func performAppSwitcherGesture(app: XCUIApplication) { + func performAppSwitcherGesture(app: XCUIApplication) throws { if pressTvRemote(.home) { sleepFor(resolveTvRemoteDoublePressDelay()) _ = pressTvRemote(.home) return } - performCoordinateAppSwitcherGesture(app: app) + try performCoordinateAppSwitcherGesture(app: app) } - private func performCoordinateAppSwitcherGesture(app: XCUIApplication) { + private func performCoordinateAppSwitcherGesture(app: XCUIApplication) throws { #if !os(tvOS) - let target = resolveRunnerWindow(app: app).window ?? app + let target = try resolveRunnerWindow(app: app).window ?? app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99)) let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7)) start.press(forDuration: 0.6, thenDragTo: end) @@ -267,11 +267,12 @@ extension RunnerTests { /// The read that remains is not atomic: a window vanishing while it is the one being resolved still /// records, and the chosen window is read again on its way to a display ID /// (`RunnerResolveWindowDisplayID`). - func resolveRunnerWindow(app: XCUIApplication) -> (window: XCUIElement?, frame: CGRect) { - guard let resolved = resolveRunnerWindowElement(app: app) else { - // A lit-panel request never falls back to the application, whose frame names another panel. - return (nil, requestLitPanel == nil ? app.frame : .null) + func resolveRunnerWindow(app: XCUIApplication) throws -> (window: XCUIElement?, frame: CGRect) { + if requestLitPanel != nil { + let booked = try requireRequestLitPanelSurface(app: app) + return (booked.window, booked.frame) } + guard let resolved = resolveRunnerWindowElement(app: app) else { return (nil, app.frame) } return (resolved.window, resolved.frame) } @@ -292,8 +293,8 @@ extension RunnerTests { return nil } - func onScreenWindowFrame(app: XCUIApplication) -> CGRect { - resolveRunnerWindow(app: app).frame + func onScreenWindowFrame(app: XCUIApplication) throws -> CGRect { + try resolveRunnerWindow(app: app).frame } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) throws -> Response { @@ -649,7 +650,7 @@ extension RunnerTests { let origin = booked.window.coordinate(withNormalizedOffset: .zero) return origin.withOffset(CGVector(dx: x - booked.frame.minX, dy: y - booked.frame.minY)) #else - let resolved = resolveRunnerWindow(app: app) + let resolved = try resolveRunnerWindow(app: app) let root = resolved.window ?? app let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) let offsetX = x - Double(resolved.frame.origin.x) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 78b2d0c62c..f35f0d16f6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -82,7 +82,7 @@ extension RunnerTests { @MainActor private func tapResolvedNavigationBackControl(app: XCUIApplication, frame: CGRect) throws -> InAppBackOutcome { guard SnapshotGeometry.isPositiveFinite(frame) else { return .unavailable } - let context = synthesizedCoordinateContext(app: app) + let context = try synthesizedCoordinateContext(app: app) guard try dispatchNavigationTap(app: app, point: CGPoint(x: frame.midX, y: frame.midY), context: context) else { return .unavailable } @@ -97,8 +97,8 @@ extension RunnerTests { point: CGPoint, context: SynthesizedCoordinateContext? ) throws -> Bool { - switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { - synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) + switch try performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { + try synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) }) { case .performed: return true @@ -115,7 +115,7 @@ extension RunnerTests { private func topNavigationBackFrame(app: XCUIApplication) throws -> CGRect? { #if os(iOS) - let window = onScreenWindowFrame(app: app) + let window = try onScreenWindowFrame(app: app) let candidates = try elementQueries(app: app, matching: .button).flatMap { $0.matching(Self.navigationBackPredicate()).allElementsBoundByIndex }.compactMap { @@ -224,7 +224,7 @@ extension RunnerTests { windowQueryRejected: Bool ) throws -> InAppBackOutcome { #if os(iOS) - guard let context = synthesizedCoordinateContext( + guard let context = try synthesizedCoordinateContext( app: app, windowQueryRejected: windowQueryRejected ), let point = Self.topLeadingNavigationFallbackPoint( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift index 4de569032c..4b1ae239cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift @@ -125,8 +125,8 @@ extension RunnerTests { message: String, fallback: GestureFallback? = nil ) throws -> Response { - let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) - let dragFrame = resolvedDragVisualizationFrame( + let dragPoints = try keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) + let dragFrame = try resolvedDragVisualizationFrame( app: activeApp, x: dragPoints.x, y: dragPoints.y, @@ -163,8 +163,8 @@ extension RunnerTests { profile: SynthesizedDragProfile ) throws -> Response? { #if os(iOS) - let context = context ?? synthesizedCoordinateContext(app: activeApp) - guard let plan = axFreeSynthesizedDragPlan( + let context = try context ?? synthesizedCoordinateContext(app: activeApp) + guard let plan = try axFreeSynthesizedDragPlan( app: activeApp, x: x, y: y, @@ -191,8 +191,8 @@ extension RunnerTests { referenceWidth: Double(plan.referenceFrame.width), referenceHeight: Double(plan.referenceFrame.height) ) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { - synthesizedDragAt( + switch try performSynthesizedGesture(activeApp, kind: policyKind, context: plan.context, synthesize: { + try synthesizedDragAt( app: activeApp, x: plan.points.x, y: plan.points.y, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index e8935a77cd..11b714522d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -35,12 +35,12 @@ extension RunnerTests { } } - let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) + let synthesizedContext = try synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) // Touch frame resolves from the first step's coords so recording-gestures works unchanged. A // synthesized context already names the frame, so the app tree is not read for it again. let firstStep = steps[0] - let firstFrame = (firstStep.x != nil && firstStep.y != nil) + let firstFrame = try (firstStep.x != nil && firstStep.y != nil) ? synthesizedContext.map { touchVisualizationFrame(x: firstStep.x!, y: firstStep.y!, context: $0) } ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil @@ -149,8 +149,8 @@ extension RunnerTests { let y = step.y ?? 0 let longPressDurationMs = min(max(step.durationMs ?? 800, 16), 10000) if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { - switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { - synthesizedSequenceStep( + switch try performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { + try synthesizedSequenceStep( policyKind, app: activeApp, x: x, @@ -193,11 +193,11 @@ extension RunnerTests { y: Double, longPressDurationMs: Double, context: SynthesizedCoordinateContext? - ) -> RunnerInteractionOutcome? { + ) throws -> RunnerInteractionOutcome? { switch policyKind { #if os(iOS) case .coordinateLongPress: - return synthesizedLongPressAt( + return try synthesizedLongPressAt( app: app, x: x, y: y, @@ -206,7 +206,7 @@ extension RunnerTests { ) #endif default: - return synthesizedTapAt(app: app, x: x, y: y, context: context) + return try synthesizedTapAt(app: app, x: x, y: y, context: context) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index d6466341ad..81bb6793d1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -501,7 +501,7 @@ extension RunnerTests { app: XCUIApplication, hint: CaptureHint, sliceDeadline deadline: Date - ) -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { + ) throws -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var nodes: [RawAXNode] = [ interactiveRootNode(rect: .zero) ] @@ -518,7 +518,7 @@ extension RunnerTests { ) } - let viewport = safeSnapshotViewport(app: app, readingOrientation: false) + let viewport = try safeSnapshotViewport(app: app, readingOrientation: false) var seen = Set() var candidates: [RawAXNode] = [] let flatElements = flatInteractiveElements(app: app, deadline: deadline) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 509bc23a45..977ee617cf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -55,7 +55,7 @@ extension RunnerTests { timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)), timeoutError: snapshotMainThreadTimeoutError("reading snapshot viewport") ) { - self.safeSnapshotViewport(app: app, readingOrientation: true) + try self.safeSnapshotViewport(app: app, readingOrientation: true) } // Use the same tree for keyboard coverage so a later dismissal cannot invalidate the band. let keyboardBand = captureKeyboardBandFact(snapshot: rootSnapshot, viewport: viewport, deadline: captureDeadline) @@ -138,15 +138,18 @@ extension RunnerTests { /// The viewport as a declared fact; a read that raises is `.missing(reason: .notProvided)` (#2891). /// `readingOrientation` reads the interface orientation in the same hop, for tiers whose frames can /// arrive in the device's native space; without it the viewport cannot anchor a rotation. - func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) -> SnapshotViewport { - safely("SNAPSHOT_VIEWPORT", .missing(reason: .notProvided)) { - .reported( - box: snapshotAppFrame(app: app), - interfaceOrientation: readingOrientation - ? capturedInterfaceOrientation(app: app) - : RunnerInterfaceOrientation.unknown - ) + func safeSnapshotViewport(app: XCUIApplication, readingOrientation: Bool) throws -> SnapshotViewport { + let result: Result = safely("SNAPSHOT_VIEWPORT", .success(.missing(reason: .notProvided))) { + Result { + .reported( + box: try snapshotAppFrame(app: app), + interfaceOrientation: readingOrientation + ? capturedInterfaceOrientation(app: app) + : RunnerInterfaceOrientation.unknown + ) + } } + return try result.get() } private func describeSnapshotError(_ error: Error) -> String { @@ -259,9 +262,9 @@ extension RunnerTests { return placeholderText(element.placeholderValue) } - private func snapshotAppFrame(app: XCUIApplication) -> CGRect { + private func snapshotAppFrame(app: XCUIApplication) throws -> CGRect { #if os(iOS) - return onScreenWindowFrame(app: app) + return try onScreenWindowFrame(app: app) #else return app.frame #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index f2325fa606..dadf4b0128 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -377,7 +377,7 @@ extension RunnerTests { timeout: max(0.1, sliceDeadline.timeIntervalSinceNow), timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot") ) { - self.querySweepSnapshotAcquisition( + try self.querySweepSnapshotAcquisition( app: app, hint: hint, sliceDeadline: sliceDeadline diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 99a1a4838c..df29a9509d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -115,11 +115,24 @@ extension RunnerTests { func synthesizedSequenceCoordinateContext( steps: [SequenceStep], app: XCUIApplication - ) -> SynthesizedCoordinateContext? { + ) throws -> SynthesizedCoordinateContext? { guard steps.lazy.contains(where: { synthesizedPolicyKind(forSequenceStep: $0) != nil }) else { return nil } - return synthesizedCoordinateContext(app: app) + return try synthesizedCoordinateContext(app: app) + } + + @MainActor + func performSynthesizedTapGesture( + app: XCUIApplication, + x: Double, + y: Double, + context: SynthesizedCoordinateContext? = nil + ) throws -> SynthesizedGestureAttempt { + let context = try context ?? synthesizedCoordinateContext(app: app) + return try performSynthesizedGesture(app, kind: .coordinateTap, context: context) { + try synthesizedTapAt(app: app, x: x, y: y, context: context) + } } /// `context` is nil when no window frame resolved; `kind`'s fallback decision then reads the @@ -129,10 +142,10 @@ extension RunnerTests { _ app: XCUIApplication, kind: SynthesizedGesturePolicyKind, context: SynthesizedCoordinateContext?, - synthesize: () -> RunnerInteractionOutcome? - ) -> SynthesizedGestureAttempt { + synthesize: () throws -> RunnerInteractionOutcome? + ) throws -> SynthesizedGestureAttempt { var outcome: RunnerInteractionOutcome? - let timing = measureGesture { outcome = synthesize() } + let timing = try measureGesture { outcome = try synthesize() } guard let outcome else { logSynthesizedGesturePolicyDecision(kind: kind, context: context, fallbackAttempted: false) return .refused(timing: timing, message: "synthesized gesture deadline expired before dispatch", hint: nil) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 965f7ab377..4b11d38245 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -12,7 +12,7 @@ extension RunnerTests { durationMs: Double, profile: SynthesizedDragProfile, context: SynthesizedCoordinateContext? = nil - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { #if os(iOS) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { return .unsupported( @@ -21,7 +21,7 @@ extension RunnerTests { ) } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let context = context ?? synthesizedCoordinateContext(app: app) else { + guard let context = try context ?? synthesizedCoordinateContext(app: app) else { return .unsupported( message: "synthesized coordinate drag could not resolve an app window with a finite screen frame", hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." @@ -99,7 +99,7 @@ extension RunnerTests { y: Double, context: SynthesizedCoordinateContext? = nil, deadline: Date? = nil - ) -> RunnerInteractionOutcome? { + ) throws -> RunnerInteractionOutcome? { #if os(iOS) guard x.isFinite, y.isFinite else { return .unsupported( @@ -113,7 +113,7 @@ extension RunnerTests { x: x, y: y, orientation: orientation, - context: context ?? synthesizedCoordinateContext(app: app), + context: try context ?? synthesizedCoordinateContext(app: app), gesture: .tap ) guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { @@ -174,7 +174,7 @@ extension RunnerTests { y: Double, durationMs: Double, context: SynthesizedCoordinateContext? - ) -> RunnerInteractionOutcome { + ) throws -> RunnerInteractionOutcome { guard x.isFinite, y.isFinite, durationMs.isFinite, durationMs > 0 else { return .unsupported( message: "synthesized coordinate long press requires finite coordinates and a positive duration", @@ -227,7 +227,7 @@ extension RunnerTests { y: Double, x2: Double, y2: Double - ) -> DragPoints { + ) throws -> DragPoints { let original = DragPoints(x: x, y: y, x2: x2, y2: y2) #if os(iOS) guard let keyboardFrame = visibleKeyboardFrame(app: app) else { @@ -245,7 +245,7 @@ extension RunnerTests { return original } - let appFrame = onScreenWindowFrame(app: app) + let appFrame = try onScreenWindowFrame(app: app) guard !appFrame.isEmpty else { return original } @@ -282,9 +282,9 @@ extension RunnerTests { #endif } - func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame { + func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) throws -> TouchVisualizationFrame { let appFrame = app.frame - let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame) + let referenceFrame = try resolvedTouchReferenceFrame(app: app, appFrame: appFrame) let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY return TouchVisualizationFrame( @@ -316,9 +316,9 @@ extension RunnerTests { y: Double, x2: Double, y2: Double - ) -> DragVisualizationFrame { - let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y) - let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2) + ) throws -> DragVisualizationFrame { + let start = try resolvedTouchVisualizationFrame(app: app, x: x, y: y) + let end = try resolvedTouchVisualizationFrame(app: app, x: x2, y: y2) return DragVisualizationFrame( x: start.x, y: start.y, @@ -329,8 +329,8 @@ extension RunnerTests { ) } - func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect { - let resolved = resolveRunnerWindow(app: app) + func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) throws -> CGRect { + let resolved = try resolveRunnerWindow(app: app) if resolved.window != nil { return frameAvoidingKeyboard(app: app, frame: resolved.frame) } @@ -344,9 +344,9 @@ extension RunnerTests { /// context its record is routed through, so a tree with no window whose app provably fills its /// sole screen plans on that screen, the same route a synthesized tap or scroll takes. @MainActor - func gestureViewportFrame(app: XCUIApplication) -> CGRect { + func gestureViewportFrame(app: XCUIApplication) throws -> CGRect { #if os(iOS) - guard let context = synthesizedCoordinateContext(app: app) else { + guard let context = try synthesizedCoordinateContext(app: app) else { return frameAvoidingKeyboard(app: app, frame: app.frame) } // The sole-screen route means the windows query just failed, so a keyboard probe would too. @@ -354,7 +354,7 @@ extension RunnerTests { ? context.referenceFrame : frameAvoidingKeyboard(app: app, frame: context.referenceFrame) #else - return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + return try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) #endif } @@ -389,16 +389,16 @@ extension RunnerTests { x2: Double, y2: Double, context: SynthesizedCoordinateContext? = nil - ) -> SynthesizedDragPlan? { + ) throws -> SynthesizedDragPlan? { #if os(iOS) - let context = context ?? synthesizedCoordinateContext(app: app) + let context = try context ?? synthesizedCoordinateContext(app: app) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, let context else { return nil } let points = context.allowsKeyboardProbe - ? keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) + ? try keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) : DragPoints(x: x, y: y, x2: x2, y2: y2) return SynthesizedDragPlan( points: points, @@ -449,11 +449,18 @@ extension RunnerTests { func synthesizedCoordinateContext( app: XCUIApplication, windowQueryRejected: Bool = false - ) -> SynthesizedCoordinateContext? { + ) throws -> SynthesizedCoordinateContext? { #if os(iOS) var health = mainOwned.accessibilityHealth let window: XCUIElement? let referenceFrame: CGRect + if requestLitPanel != nil { + let booked = try requireRequestLitPanelSurface(app: app) + guard let context = SynthesizedCoordinateContext( + validating: booked.frame, resolvedWindow: booked.window, accessibilityHealth: health + ) else { throw RunnerAppScreenCaptureFailure.unresolvedScreen } + return context + } // A tree that just rejected a query skips the windows query only where the sole screen answers // without it; anywhere else that query is still the only route. let rejectedTreeSoleScreenFrame = windowQueryRejected ? soleScreenReferenceFrame(app: app) : nil @@ -663,9 +670,9 @@ extension RunnerTests { } } - func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? { + func swipe(app: XCUIApplication, direction: String) throws -> DragVisualizationFrame? { if performTvRemoteSwipeIfAvailable(direction: direction) { - let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + let frame = try resolvedTouchReferenceFrame(app: app, appFrame: app.frame) let midX = frame.midX let midY = frame.midY return DragVisualizationFrame( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 25e47ffcd6..7299948cef 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -41,11 +41,8 @@ extension RunnerTests { case .unavailable: return textInputProbeUnavailableResponse() } - let policyKind = SynthesizedGesturePolicyKind.coordinateTap - let context = synthesizedCoordinateContext(app: activeApp) - switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { - synthesizedTapAt(app: activeApp, x: x, y: y, context: context) - }) { + let context = try synthesizedCoordinateContext(app: activeApp) + switch try performSynthesizedTapGesture(app: activeApp, x: x, y: y, context: context) { case .performed: resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 9af9d71548..705f9583e5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -97,7 +97,7 @@ extension RunnerTests { let booked = try XCTUnwrap(litPanelState.surface) XCTAssertTrue(booked.window === window) XCTAssertEqual(booked.frame, frame) - XCTAssertEqual(synthesizedRotationFrame(fallback: booked.frame, interfaceOrientation: orientation), expected) + XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: booked.frame, interfaceOrientation: orientation), expected) if case .unsupported(let message, _) = try tapAt(app: app, x: 200, y: 150) { XCTFail(message) } XCTAssertTrue(LitPanelBookingProbe.anchor === window) XCTAssertEqual(LitPanelBookingProbe.tappedPoint, CGPoint(x: 200, y: 150)) @@ -171,6 +171,40 @@ extension RunnerTests { } } + @MainActor + func testFailedLitPanelBookingRefusesEveryReferenceFrameConsumer() throws { + try withLitPanelBookingProbe { _ in + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"tap","litPanel":{"displayId":999,"widthPt":669,"heightPt":951}}"# + )) + func refuses(_ read: () throws -> T) { + XCTAssertThrowsError(try read()) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .offLitPanel) + } + } + refuses { try resolveRunnerWindow(app: app) } + refuses { try onScreenWindowFrame(app: app) } + refuses { try resolvedTouchReferenceFrame(app: app, appFrame: CGRect(x: 0, y: 0, width: 466, height: 678)) } + refuses { try synthesizedCoordinateContext(app: app) } + refuses { try gestureViewportFrame(app: app) } + refuses { try resolvedTouchVisualizationFrame(app: app, x: 20, y: 30) } + refuses { try resolvedDragVisualizationFrame(app: app, x: 20, y: 30, x2: 40, y2: 50) } + refuses { try performSynthesizedTapGesture(app: app, x: 20, y: 30) } + var alertRefusal: Error? + let held = holdingRecordedIssues { + do { + _ = try activateAlertButton( + RunnerAlert(root: app, ownerApp: app, buttons: [app], source: .appAlert, window: nil), + button: app, action: "accept", frame: CGRect(x: 10, y: 10, width: 40, height: 40), deadline: .distantFuture + ) + } catch { alertRefusal = error } + } + XCTAssertEqual(alertRefusal as? RunnerAppScreenCaptureFailure, .offLitPanel) + XCTAssertTrue(held.issues.isEmpty, "the typed refusal must survive the idle/interruption wrappers") + } + } + @MainActor private func withLitPanelBookingProbe(_ body: (XCUIElement) throws -> Void) throws { app.launch() @@ -313,10 +347,10 @@ extension RunnerTests { #"{"command":"tap","litPanel":{"displayId":\#(displayID),"widthPt":\#(frame.width),"heightPt":\#(frame.height + 80)}}"# )) XCTAssertNil(bookRequestLitPanelSurface(app: app)) - let context = try XCTUnwrap(synthesizedCoordinateContext(app: app)) + let context = try XCTUnwrap(try synthesizedCoordinateContext(app: app)) XCTAssertEqual(context.referenceFrame, frame, "the booked reference is the window's own frame") XCTAssertEqual( - synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: RunnerInterfaceOrientation.portrait), + synthesizedRotationFrame(app: app, fallback: context.referenceFrame, interfaceOrientation: RunnerInterfaceOrientation.portrait), CGRect(x: 0, y: 0, width: frame.width, height: frame.height + 80), "the turn is about the whole lit panel, not the window" ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index b73dde9bf2..95627b9b7c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -109,7 +109,7 @@ extension RunnerTests { ] for testCase in cases { RunnerWindowFrameReads.begin(answering: testCase.frames) - let resolved = resolveRunnerWindow(app: app) + let resolved = try resolveRunnerWindow(app: app) XCTAssertEqual(resolved.window != nil, testCase.hasWindow, testCase.name) XCTAssertEqual(RunnerWindowFrameReads.recorded(), testCase.reads, testCase.name) XCTAssertEqual( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index e5cf577f63..63880cb5da 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -89,23 +89,23 @@ extension RunnerTests { } @MainActor - func testCoordinateTapThatWasNotDispatchedDoesNotFallBackToXCTest() { + func testCoordinateTapThatWasNotDispatchedDoesNotFallBackToXCTest() throws { for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { mainOwned.accessibilityHealth = health for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { - let attempt = performSynthesizedGesture(app, kind: .coordinateTap, context: context) { nil } + let attempt = try performSynthesizedGesture(app, kind: .coordinateTap, context: context) { nil } XCTAssertEqual(synthesizedGestureRoute(attempt), "refused", "axHealth=\(health.rawValue)") } } } @MainActor - func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() { + func testFailedCoordinateTapSynthesisFallsBackToXCTestAtEveryAccessibilityHealth() throws { for health: RunnerAccessibilityHealth in [.unknown, .healthy, .unavailable] { mainOwned.accessibilityHealth = health for context in [nil, synthesizedGestureTestContext(accessibilityHealth: health)] { let label = "axHealth=\(health.rawValue) context=\(context == nil ? "unresolved" : "resolved")" - let attempt = performSynthesizedGesture( + let attempt = try performSynthesizedGesture( app, kind: .coordinateTap, context: context, @@ -117,7 +117,7 @@ extension RunnerTests { } @MainActor - func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() { + func testSynthesizedGestureFallbackFollowsItsKindAndTheResolvedAccessibilityHealth() throws { mainOwned.accessibilityHealth = .healthy let unavailable = synthesizedGestureTestContext(accessibilityHealth: .unavailable) let unknown = synthesizedGestureTestContext(accessibilityHealth: .unknown) @@ -130,7 +130,7 @@ extension RunnerTests { (.coordinateLongPress, unknown, "xctestFallback"), ] for (kind, context, expected) in cases { - let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) + let attempt = try performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) XCTAssertEqual(synthesizedGestureRoute(attempt), expected, "kind=\(kind.rawValue)") switch attempt { case .xctestFallback(let message, let hint), .refused(_, let message, let hint): @@ -139,7 +139,7 @@ extension RunnerTests { case .performed: break } - let performed = performSynthesizedGesture(app, kind: kind, context: context) { .performed } + let performed = try performSynthesizedGesture(app, kind: kind, context: context) { .performed } XCTAssertEqual(synthesizedGestureRoute(performed), "performed", "kind=\(kind.rawValue)") } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 9ca703caee..7df11aa361 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -407,7 +407,7 @@ extension RunnerTests { RunnerInterfaceOrientation.portrait ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .performed? = outcome else { return XCTFail("expected a portrait tap to be dispatched, got \(String(describing: outcome))") } @@ -430,7 +430,7 @@ extension RunnerTests { RunnerInterfaceOrientation.portrait ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40) guard case .performed? = outcome else { return XCTFail("expected a window-less portrait tap to be dispatched, got \(String(describing: outcome))") } @@ -445,7 +445,7 @@ extension RunnerTests { let (_, cleanup) = try launchWithUnreadableDisplay(landscape: true, windowQueryRejected: true) defer { cleanup() } - let outcome = synthesizedTapAt(app: app, x: 40, y: 40) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40) guard case .unsupported(let message, _)? = outcome else { return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") } @@ -471,7 +471,7 @@ extension RunnerTests { synthesizedRotationFrame(app: app, fallback: windowFrame, interfaceOrientation: orientation), windowFrame ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .performed? = outcome else { return XCTFail("expected the tap to be dispatched about the window, got \(String(describing: outcome))") } @@ -499,7 +499,7 @@ extension RunnerTests { placedWindow, "portrait skips the box read (only landscape and upside-down read it), so it must not refuse" ) - let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + let outcome = try synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .unsupported(let message, _)? = outcome else { return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") } diff --git a/packages/platform-apple/src/foldable/runner-panel.test.ts b/packages/platform-apple/src/foldable/runner-panel.test.ts index 10e1845ca2..f1461b60bb 100644 --- a/packages/platform-apple/src/foldable/runner-panel.test.ts +++ b/packages/platform-apple/src/foldable/runner-panel.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { beforeEach, test, vi } from 'vitest'; import { bindTestCommandHost } from '@agent-device/test-kit'; -import { withAppleRunnerProvider } from '@agent-device/apple-runner'; +import { withAppleRunnerProvider, type AppleRunnerProvider } from '@agent-device/apple-runner'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { AppError } from '@agent-device/kernel/errors'; @@ -18,6 +18,7 @@ vi.mock('../core/devicectl.ts', async (importOriginal) => ({ import { runIosDevicectlJsonRequest } from '../core/devicectl.ts'; import { runAppleRunnerCommand } from '../core/runner-client.ts'; +import { iosRunnerOverrides } from '../interactions.ts'; import { recordingRunnerProvider, type RecordedRunnerCall, @@ -253,3 +254,68 @@ test('a slow failed read does not back off over a newer read that resolved', asy 'the stale failure must not hold the next request on the historic route', ); }); + +test('every press-series chunk uses the lit-panel naming path', async () => { + answering(IPHONE_DUO_OPEN); + const device = simulator(); + const calls: RecordedRunnerCall[] = []; + const base = recordingRunnerProvider(calls); + const provider: AppleRunnerProvider = { + ...base, + runCommand: async (target, command, options) => { + await base.runCommand(target, command, options); + return { + completedSteps: command.steps?.length ?? 0, + sequenceResults: command.steps?.map(({ kind }) => ({ ok: true, kind })) ?? [], + }; + }, + }; + // A long interval forces separate sequence chunks; each dispatch must name its current panel. + mockPanelTable.mockResolvedValueOnce({ ok: true, payload: IPHONE_DUO_OPEN }); + mockPanelTable.mockResolvedValueOnce({ ok: true, payload: IPHONE_DUO_CLOSED }); + await withAppleRunnerProvider(provider, { deviceId: device.id }, async () => { + const { overrides } = iosRunnerOverrides( + device, + { appBundleId: 'com.example.app' }, + process.env, + ); + await overrides.pressPoint!( + { x: 100, y: 200 }, + { + button: 'primary', + count: 4, + intervalMs: 10_000, + holdMs: 0, + jitterPx: 0, + doubleTap: false, + }, + ); + }); + assert.ok(calls.length > 1, 'the regression must exercise more than one chunk'); + assert.ok(calls.every(({ command }) => command.command === 'sequence' && command.litPanel)); + assert.equal(calls[0]?.command.litPanel?.displayId, 3); + assert.equal(calls[1]?.command.litPanel?.displayId, 1); + assert.equal(mockPanelTable.mock.calls.length, calls.length); +}); + +test('an unnamed panel keeps the historic wire timeout and caller signal', async () => { + let time = 1_000; + const clock = vi.spyOn(Date, 'now').mockImplementation(() => time); + try { + mockPanelTable.mockImplementation(async () => { + time += 50; + return { ok: true, payload: IPHONE_17_SINGLE_PANEL }; + }); + const device = simulator(); + const calls: RecordedRunnerCall[] = []; + const signal = new AbortController().signal; + await withAppleRunnerProvider(recordingRunnerProvider(calls), { deviceId: device.id }, () => + runAppleRunnerCommand(device, { command: 'tap', x: 10, y: 20, timeoutMs: 1_200 }, { signal }), + ); + assert.equal(calls[0]?.command.timeoutMs, 1_200); + assert.equal(calls[0]?.command.litPanel, undefined); + assert.equal(calls[0]?.options.signal, signal); + } finally { + clock.mockRestore(); + } +}); diff --git a/packages/platform-apple/src/foldable/runner-panel.ts b/packages/platform-apple/src/foldable/runner-panel.ts index 45f8a31310..0e8010172d 100644 --- a/packages/platform-apple/src/foldable/runner-panel.ts +++ b/packages/platform-apple/src/foldable/runner-panel.ts @@ -48,6 +48,7 @@ export async function resolveRunnerRequestLitPanel( /** * `command` with the lit panel named and the request budget the read left, measured on one clock. * A read that leaves no budget is refused before anything is sent. + * Only a named panel charges probe time to the wire timeout; unnamed-panel requests keep the historic timeout and share the caller's AbortSignal. */ export async function nameRunnerRequestLitPanel( device: DeviceInfo, From 4d6f5231c3ee6d4c6eb646bfe6eeadc10386747c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 13:29:36 +0100 Subject: [PATCH 28/29] fix(ios): scope collapsed-tab candidates to the captured lit display --- .../RunnerRequestPanel.swift | 16 ++- .../RunnerTests+Snapshot.swift | 8 +- .../RunnerTests+SnapshotAcquisition.swift | 52 +++++---- .../RunnerXCTestEventBridge.h | 2 +- .../RunnerXCTestEventBridge.m | 4 +- .../UnitTests/RunnerRequestPanelTests.swift | 108 +++++++++++++++++- ...RunnerTests+SnapshotAcquisitionTests.swift | 4 +- .../RunnerTests+XCTestEventBridgeTests.swift | 6 +- docs/adr/0025-foldable-apple-panels.md | 2 +- 9 files changed, 160 insertions(+), 42 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift index 3767d7e09f..fa17a6bd0b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerRequestPanel.swift @@ -93,11 +93,11 @@ extension RunnerTests { /// The roots of every element read: the app, or the windows proven on the lit display. A walk that /// cannot read the list, or that records an issue (#2995), throws `APP_SCREEN_UNRESOLVED`. - func elementSearchRoots(app: XCUIApplication) throws -> [XCUIElement] { - guard let litPanel = requestLitPanel else { return [app] } + func elementSearchRoots(root: XCUIElement, litDisplayID: UInt?) throws -> [XCUIElement] { + guard let litDisplayID else { return [root] } var windows: [XCUIElement]? let walk = holdingRecordedIssues { - windows = RunnerCopyLitPanelWindows(app, litPanel)?.compactMap { $0 as? XCUIElement } + windows = RunnerCopyLitPanelWindows(root, litDisplayID)?.compactMap { $0 as? XCUIElement } #if AGENT_DEVICE_RUNNER_UNIT_TESTS if litPanelWindowWalkIssueForTesting { record(XCTIssue(type: .assertionFailure, compactDescription: "Window vanished during the lit-panel walk")) @@ -118,6 +118,14 @@ extension RunnerTests { } func elementQueries(app: XCUIApplication, matching type: XCUIElement.ElementType) throws -> [XCUIElementQuery] { - try elementSearchRoots(app: app).map { $0.descendants(matching: type) } + try elementQueries(root: app, matching: type, litDisplayID: requestLitPanel?.displayID) + } + + func elementQueries( + root: XCUIElement, + matching type: XCUIElement.ElementType, + litDisplayID: UInt? + ) throws -> [XCUIElementQuery] { + try elementSearchRoots(root: root, litDisplayID: litDisplayID).map { $0.descendants(matching: type) } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 81bb6793d1..82f6d386cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -193,11 +193,11 @@ extension RunnerTests { ) throws -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { var outcome = SnapshotTierOutcome.completed var cachedCollapsedTabCandidates: [CollapsedTabCandidate]? - func collapsedTabCandidatesOnce() -> [CollapsedTabCandidate] { + func collapsedTabCandidatesOnce() throws -> [CollapsedTabCandidate] { if let cachedCollapsedTabCandidates { return cachedCollapsedTabCandidates } - let candidates = collapsedTabCandidates(context: context, processingDeadline: processingDeadline) + let candidates = try collapsedTabCandidates(context: context, processingDeadline: processingDeadline) if candidates == nil { outcome = .deadlineExhausted } cachedCollapsedTabCandidates = candidates ?? [] return candidates ?? [] @@ -217,7 +217,7 @@ extension RunnerTests { let rootChildren = try Self.litPanelRootChildren(context.rootSnapshot, surface: context.litPanelSurface) var nodes: [RawAXNode] = [root] if Self.canDescendAtRawDepth(0, hint: hint) { - appendCollapsedTabFallbackNodes( + try appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: context.rootSnapshot, resolveCandidates: collapsedTabCandidatesOnce, @@ -274,7 +274,7 @@ extension RunnerTests { throw regularSnapshotTooLargeFailure(nodeCount: nodes.count) } if Self.canDescendAtRawDepth(depth, hint: hint) { - appendCollapsedTabFallbackNodes( + try appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: snapshot, resolveCandidates: collapsedTabCandidatesOnce, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 977ee617cf..17f409cd26 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -299,11 +299,11 @@ extension RunnerTests { func appendCollapsedTabFallbackNodes( to nodes: inout [RawAXNode], containerSnapshot: XCUIElementSnapshot, - resolveCandidates: () -> [CollapsedTabCandidate], + resolveCandidates: () throws -> [CollapsedTabCandidate], depth: Int, parentIndex: Int - ) { - let fallbackNodes = collapsedTabFallbackNodes( + ) throws { + let fallbackNodes = try collapsedTabFallbackNodes( for: containerSnapshot, resolveCandidates: resolveCandidates, startingIndex: nodes.count, @@ -336,15 +336,14 @@ extension RunnerTests { let selected: Bool } - /// The live elements any collapsed tab container in this capture may adopt, read in one XCTest - /// query: `collapsedTabCandidatePredicate` copies each match's attributes while XCTest evaluates - /// it, so no match costs a further round trip. The query starts only while the time left before - /// `processingDeadline` covers what this capture's root tree read took; nil means it did not start. + /// Live controls collapsed tab containers may adopt, queried under the capture's proven-lit roots. + /// The predicate copies attributes during each query without further round trips. Queries start + /// only while the remaining slice covers the root read's duration; nil means a query could not start. @MainActor func collapsedTabCandidates( context: SnapshotTraversalContext, processingDeadline: Date - ) -> [CollapsedTabCandidate]? { + ) throws -> [CollapsedTabCandidate]? { let containerFrames = Self.snapshotDescendants(of: context.rootSnapshot) .filter(isCollapsedTabContainer) .map(\.frame) @@ -352,19 +351,26 @@ extension RunnerTests { let predicate = Self.collapsedTabCandidatePredicate(containerFrames: containerFrames) { attributes in candidates.append(self.collapsedTabCandidate(attributes)) } - guard - Self.xctestQueryCanStart( - deadline: processingDeadline, - now: Date(), - expectedCost: context.rootReadDuration - ) - else { - NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED") - return nil + func canStartQuery() -> Bool { + guard Self.xctestQueryCanStart(deadline: processingDeadline, now: Date(), expectedCost: context.rootReadDuration) else { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED") + return false + } + return true + } + guard canStartQuery() else { return nil } + let queries = try elementQueries( + root: context.queryRoot, + matching: .any, + litDisplayID: context.litPanelSurface?.displayID + ) + var matches: [XCUIElement] = [] + for query in queries { + guard canStartQuery() else { return nil } + matches += snapshotElementsQuery { + query.matching(predicate).allElementsBoundByIndex + }.elements } - let matches = snapshotElementsQuery { - context.queryRoot.descendants(matching: .any).matching(predicate).allElementsBoundByIndex - }.elements return matches.isEmpty ? [] : candidates } @@ -434,18 +440,18 @@ extension RunnerTests { private func collapsedTabFallbackNodes( for containerSnapshot: XCUIElementSnapshot, - resolveCandidates: () -> [CollapsedTabCandidate], + resolveCandidates: () throws -> [CollapsedTabCandidate], startingIndex: Int, depth: Int, parentIndex: Int - ) -> [RawAXNode] { + ) throws -> [RawAXNode] { guard isCollapsedTabContainer(containerSnapshot) else { return [] } // Candidates are read from live elements, which XCTest reports in the app's own space, and a // collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` keys // rotation on ancestry, so these nodes never enter a declared native space and need no special // case here; `isCollapsedTabCandidate` compares reported frames to each other. let containerFrame = containerSnapshot.frame - let candidates = resolveCandidates().compactMap { candidate in + let candidates = try resolveCandidates().compactMap { candidate in collapsedTabCandidateNode( candidate: candidate, containerSnapshot: containerSnapshot, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index df32b82604..dadac43409 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -81,7 +81,7 @@ FOUNDATION_EXPORT BOOL RunnerResolveApplicationScreen( // Every application window that reports `litPanel`'s display, in list order, uncapped so element // search sees all of them; the caller holds the walk's XCTest issues. A window that is gone or names // no display is left out. Nil when the window list itself cannot be read. -FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel); +FOUNDATION_EXPORT NSArray * _Nullable RunnerCopyLitPanelWindows(id application, NSUInteger litDisplayID); // Reads the display ID off an already-resolved window, keeping a synthesized record's display and // the caller's booked reference frame on the same window. The frame is still read before the diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 4f43e177dd..dbc169a5e9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -316,7 +316,7 @@ RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( return outcome; } -NSArray *RunnerCopyLitPanelWindows(id application, RunnerLitPanel litPanel) { +NSArray *RunnerCopyLitPanelWindows(id application, NSUInteger litDisplayID) { NSArray *windows = RunnerCopyWindowList(application); if (windows == nil) { return nil; @@ -328,7 +328,7 @@ RunnerApplicationScreenFailure RunnerResolveLitPanelWindow( NSUInteger displayID = 0; if (RunnerReadLitPanelCandidate(candidate, &frame, &screen, &displayID) == RunnerApplicationScreenFailureNone && - displayID == litPanel.displayID) { + displayID == litDisplayID) { [lit addObject:candidate]; } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift index 705f9583e5..4e832de878 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerRequestPanelTests.swift @@ -5,7 +5,9 @@ import XCTest import ObjectiveC.runtime private final class LitPanelBookingScreen: NSObject { - @objc let displayID: UInt = 3 + @objc let displayID: UInt + + init(_ displayID: UInt = 3) { self.displayID = displayID } } @MainActor @@ -15,6 +17,7 @@ private final class LitPanelBookingProbe: NSObject { static var windowListReads = 0 static var maxWindowListReads: Int? static var window: XCUIElement? + static var darkWindow: XCUIElement? static var coordinate: XCUICoordinate? static var windowFrame = CGRect.zero static var orientation = RunnerInterfaceOrientation.landscapeLeft @@ -31,8 +34,8 @@ private final class LitPanelBookingProbe: NSObject { case "allElementsBoundByIndex": Self.windowListReads += 1 if let max = Self.maxWindowListReads, Self.windowListReads > max { return [] } - return [Self.window!] - case "screen": return LitPanelBookingScreen() + return [Self.window!, Self.darkWindow].compactMap { $0 } + case "screen": return LitPanelBookingScreen(self === Self.darkWindow ? 1 : 3) case "exists": return true case "frame": return NSValue(cgRect: Self.windowFrame) default: break @@ -114,6 +117,80 @@ extension RunnerTests { } } + @MainActor + func testCollapsedTabCandidatesExcludeOverlappingDarkWindowsAndKeepTheCaptureScope() throws { + try withLitPanelBookingProbe { window in + let darkWindow = XCUIApplication(bundleIdentifier: "com.apple.springboard").windows.firstMatch + LitPanelBookingProbe.darkWindow = darkWindow + LitPanelBookingProbe.windowFrame = CGRect(x: 0, y: 0, width: 669, height: 951) + setRequestLitPanel(from: try runnerCommandFixture( + #"{"command":"snapshot","litPanel":{"displayId":3,"widthPt":669,"heightPt":951}}"# + )) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let container = ElementAttributesFixture(.tabBar, CGRect(x: 0, y: 800, width: 669, height: 31)) + let context = SnapshotTraversalContext( + queryRoot: app, + rootSnapshot: ElementAttributesFixture(.application, LitPanelBookingProbe.windowFrame, children: [ + ElementAttributesFixture(.window, LitPanelBookingProbe.windowFrame, children: [container], displayID: 3), + ]), + rootReadDuration: 0, + viewport: .missing(reason: .notProvided), + keyboardBand: nil, + litPanelSurface: try XCTUnwrap(litPanelState.surface) + ) + let lit = ElementAttributesFixture(.button, CGRect(x: 30, y: 800, width: 100, height: 30), label: "Lit") + let second = ElementAttributesFixture(.button, CGRect(x: 170, y: 800, width: 100, height: 30), label: "Second") + let dark = ElementAttributesFixture(.button, lit.frame, label: "Dark") + let appQuery = app.descendants(matching: .any) + let litQuery = window.descendants(matching: .any) + let darkQuery = darkWindow.descendants(matching: .any) + var predicates: [ObjectIdentifier: NSPredicate] = [:] + var restores: [() -> Void] = [] + defer { restores.reversed().forEach { $0() } } + func swap(_ type: AnyClass, _ name: String, _ block: Any) throws { + let method = try XCTUnwrap(class_getInstanceMethod(type, NSSelectorFromString(name))) + let replacement = imp_implementationWithBlock(block) + let original = method_setImplementation(method, replacement) + restores.append { + method_setImplementation(method, original) + imp_removeBlock(replacement) + } + } + try swap(XCUIElement.self, "descendantsMatchingType:", { (root: AnyObject, _: UInt) -> XCUIElementQuery in + root === window ? litQuery : root === darkWindow ? darkQuery : appQuery + } as @convention(block) (AnyObject, UInt) -> XCUIElementQuery) + try swap(XCUIElementQuery.self, "matchingPredicate:", { (query: AnyObject, predicate: NSPredicate) -> XCUIElementQuery in + predicates[ObjectIdentifier(query)] = predicate + return query as! XCUIElementQuery + } as @convention(block) (AnyObject, NSPredicate) -> XCUIElementQuery) + try swap(XCUIElementQuery.self, "allElementsBoundByIndex", { (query: AnyObject) -> NSArray in + let attributes = query === litQuery ? [lit, second] : query === darkQuery ? [dark] : [lit, second, dark] + let matches = attributes.filter { predicates[ObjectIdentifier(query)]?.evaluate(with: $0) == true } + return matches.isEmpty ? [] : [window] + } as @convention(block) (AnyObject) -> NSArray) + let candidates = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) + XCTAssertEqual(candidates.map(\.label), ["Lit", "Second"], "an overlapping control on a dark window must not enter the capture") + let hint = CaptureHint(projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false) + let capture = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: .distantFuture) + let labels = capture.acquisition.nodes.compactMap(\.label) + XCTAssertTrue(labels.contains("Lit")) + XCTAssertTrue(labels.contains("Second")) + XCTAssertFalse(labels.contains("Dark"), "the serialized capture must exclude the overlapping dark control") + endRequestLitPanel(try XCTUnwrap(litPanelState.requestID)) + let pinned = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) + XCTAssertEqual(pinned.map(\.label), ["Lit", "Second"], "the capture's display scope survives a cleared current request") + var unscoped = context + unscoped.litPanelSurface = nil + let all = try XCTUnwrap(try collapsedTabCandidates(context: unscoped, processingDeadline: .distantFuture)) + XCTAssertEqual(all.map(\.label), ["Lit", "Second", "Dark"], "an unscoped capture keeps application-wide candidates") + litPanelWindowWalkIssueForTesting = true + defer { litPanelWindowWalkIssueForTesting = false } + XCTAssertThrowsError(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) { + XCTAssertEqual($0 as? RunnerAppScreenCaptureFailure, .unresolvedScreen) + } + } + } + @MainActor func testLitPanelBookingUsesBundleIdentityAcrossProxiesAndRequests() throws { try withLitPanelBookingProbe { window in @@ -232,6 +309,7 @@ extension RunnerTests { var originals: [(Method, IMP)] = [] defer { for (method, original) in originals.reversed() { method_setImplementation(method, original) } + LitPanelBookingProbe.darkWindow = nil LitPanelBookingProbe.window = nil LitPanelBookingProbe.coordinate = nil LitPanelBookingProbe.anchor = nil @@ -519,6 +597,30 @@ extension RunnerTests { } } + @MainActor + func testCollapsedTabSnapshotReadsProvenLitLiveWindows() throws { + app.launchArguments = ["--agent-device-collapsed-tab-bar", "--agent-device-wide-slider"] + let (panel, _) = launchAppOnTheLitPanel() + defer { tearDownLitPanelApp() } + setRequestLitPanel(from: try runnerCommandFixture(#"{"command":"snapshot",\#(panel)}"#)) + XCTAssertNil(bookRequestLitPanelSurface(app: app)) + let hint = CaptureHint(projection: .regular, depth: nil, regularPresentedDepth: nil, interactiveOnly: false, customActions: false) + let context = try XCTUnwrap(try makeSnapshotTraversalContext( + app: app, + hint: hint, + litPanelSurface: try XCTUnwrap(litPanelState.surface) + )) + let container = try XCTUnwrap(Self.snapshotDescendants(of: context.rootSnapshot).first { $0.elementType == .tabBar }) + XCTAssertTrue(container.children.isEmpty, "the live fixture must reach collapsed-tab recovery") + let candidates = try XCTUnwrap(try collapsedTabCandidates(context: context, processingDeadline: .distantFuture)) + XCTAssertTrue(candidates.contains { $0.label == "Left" }) + XCTAssertTrue(candidates.contains { $0.label == "Right" }) + let capture = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: .distantFuture) + let labels = capture.acquisition.nodes.compactMap(\.label) + XCTAssertTrue(labels.contains("Left")) + XCTAssertTrue(labels.contains("Right")) + } + /// A refused pin is a request refusal: it never reaches the capture plan, so it does not /// invalidate the cached target the next command relies on. @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift index 7d872aa45b..b884ebf27b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift @@ -10,13 +10,15 @@ final class ElementAttributesFixture: NSObject, XCUIElementSnapshot { let label: String let children: [XCUIElementSnapshot] let identifier: String + @objc let displayID: NSNumber? - init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "", identifier: String = "", children: [XCUIElementSnapshot] = []) { + init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "", identifier: String = "", children: [XCUIElementSnapshot] = [], displayID: NSNumber? = nil) { self.elementType = elementType self.frame = frame self.label = label self.children = children self.identifier = identifier + self.displayID = displayID } var value: Any? { nil } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index f1e641560f..210153bb3d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -365,7 +365,7 @@ extension RunnerTests { DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1) } + [later] let lit = RunnerCopyLitPanelWindows( - DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel) + DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel.displayID) XCTAssertEqual(lit?.count, 2) XCTAssertTrue((lit?.last as AnyObject) === later) } @@ -380,13 +380,13 @@ extension RunnerTests { DisplayWindowFixture(frame: Self.innerHome, displayID: 3, present: false), dock, ] let lit = RunnerCopyLitPanelWindows( - DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel) + DisplayApplicationFixture(windows: WindowListFixture(windows)), Self.innerPanel.displayID) XCTAssertEqual(lit?.count, 2) XCTAssertTrue((lit?.first as AnyObject) === home) XCTAssertTrue((lit?.last as AnyObject) === dock) XCTAssertNil( RunnerCopyLitPanelWindows( - DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel), + DisplayApplicationFixture(windows: UnanswerableWindowsFixture()), Self.innerPanel.displayID), "an unreadable window list is not an empty lit display" ) } diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 7ff6864875..297c18dc5e 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -726,7 +726,7 @@ With a named panel, the runner walks at most eight windows to book one; element The request books the window by bundle identity and request identity before it acts, and releases the booking when the request ends. A refusal is returned before any gesture or snapshot. Dispatch, the coordinate fallback, and the snapshot viewport reuse the booked window of the same app, so they never fall back to the application's own frame. Every element read of such a request (`findText`, `querySelector`, `readText`, a selector `tap`, and the text-input lookup of `tap` and `type`) searches only under the application's windows that report the lit display. The element search rereads the complete window list for each read, including windows beyond the booking cap, because mutations can open another window. If the list cannot be read or contains no proven lit windows, the search throws `APP_SCREEN_UNRESOLVED` instead of answering no match. A lit-panel refusal during text-entry confirmation propagates with its typed code. The panel is written onto the main queue behind work already queued there, so work that a previous command abandoned still reads that command's panel. Capture resolves its own window with the same rule. It asks the system surface only when the app has no window, and it refuses an app on the dark panel. -A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. A `customActions` request with no explicit backend pin implies the private AX pin, so its refusal names `customActions`. +A snapshot runs only the tree tier, because only that tier can prove which display a window is on. A pin to another backend is refused with `UNSUPPORTED_OPERATION`. The snapshot traverses the application's tree, not only the booked window, and keeps every window whose element snapshot reports the lit display. A window that reports no display is dropped. If no window is proven, the snapshot refuses with `APP_SCREEN_UNRESOLVED`. The root takes the booked window's frame. `--raw` applies the same window filter and the same root frame, so raw and regular coordinates share one space. The snapshot reads the booked window only from its capture target. Collapsed-tab fallback candidates use the shared proven-lit element-query owner with that capture target's pinned display ID. A failed root walk propagates `APP_SCREEN_UNRESOLVED`. A `customActions` request with no explicit backend pin implies the private AX pin, so its refusal names `customActions`. Rejected alternatives: From ec80eecb1e0251d51d93aa68071c88577f1ea7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 14:16:52 +0100 Subject: [PATCH 29/29] fix(ios): refuse unread lit-panel window frames --- .../RunnerXCTestEventBridge.m | 5 ++++- .../RunnerTests+XCTestEventBridgeTests.swift | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index dbc169a5e9..f5f0583f18 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -218,7 +218,10 @@ static RunnerApplicationScreenFailure RunnerReadLitPanelCandidate( } }]; *frame = readFrame; - if (exception != nil || !present || !AgentDeviceForegroundFrameIsMeasurable(readFrame)) { + if (exception != nil) { + return RunnerApplicationScreenFailureUnresolvedScreen; + } + if (!present || !AgentDeviceForegroundFrameIsMeasurable(readFrame)) { return RunnerApplicationScreenFailureUnresolvedWindow; } return RunnerReadResolvedWindowScreen(candidate, readFrame, screen, displayID); diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift index 210153bb3d..3439148f11 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+XCTestEventBridgeTests.swift @@ -325,6 +325,20 @@ extension RunnerTests { XCTAssertNil(walk.window) } + func testLitPanelWindowRefusesAFrameReadExceptionBeforeAnyLaterWindow() { + let laterWindows: [[NSObject]] = [ + [], + [DisplayWindowFixture(frame: Self.innerHome, displayID: 3)], + [DisplayWindowFixture(frame: Self.outerStatusBar, displayID: 1)], + ] + for later in laterWindows { + let walk = resolveLitPanelWindow([FramelessWindowFixture()] + later) + XCTAssertEqual(walk.failure, .unresolvedScreen) + XCTAssertNil(walk.window) + XCTAssertTrue(walk.frame.isNull) + } + } + /// An app that occupies part of the lit panel has no full-panel box; being on the lit display is /// still the fact the first-window route could not establish. func testLitPanelWindowBooksTheLargestWindowOnTheLitDisplay() {