From eb706bd8a1850dc0d77f5a697a117666ac9e5345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 08:41:29 +0100 Subject: [PATCH 01/18] fix(ios): size the sole display from the system app, not the runner's UIScreen The generated AgentDeviceRunnerUITests-Runner.app has no launch screen, so the runner process can read UIScreen.main in compatibility mode (768x1024 on an iPad, a letterbox on an iPhone). The synthesized rotation then turned landscape touches about that box, and every press landed a fixed offset away while reporting ok. Read the display from the AXRuntime system application frame, the same screen the fills-screen proof already compares against. Fixes #4091 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/xctest-nightly.yml | 6 ++- .../RunnerTests+SynthesizedInteraction.swift | 9 ++-- .../RunnerXCTestEventBridge.h | 8 ++++ .../RunnerXCTestEventBridge.m | 17 ++++++++ ...nerTests+SynthesizedInteractionTests.swift | 41 ++++++++++++++++++- 5 files changed, 73 insertions(+), 8 deletions(-) diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index cdecf78c38..44c3dfd232 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -150,8 +150,9 @@ jobs: ' # The iPad leg runs a named selection: the smallest set that can execute the off-phone half of - # the sole-screen rule — the admission boundary itself, one hostile-tree command that must - # reach synthesis, and `back`'s route witness. + # the sole-screen rule — the admission boundary itself, the display box a synthesized turn + # swaps (#4091 reported it off on iPads), one hostile-tree command that must reach synthesis, + # and `back`'s route witness. # Geometry and error cases live in the host-runnable evidence matrix instead. `xcodebuild` # exits 0 when a selection matches nothing, so the reporter below asserts the executed count # against this lane's reach (`XCTEST_LANE=nightly-ipad`, derived by check:xctest-selection). @@ -169,6 +170,7 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=$UDID" \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSoleScreenDisplayIDIsNamedOnlyForAnAppThatFillsItsScreen \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryTapsThroughTheRouteThatOpens \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryWithUnprovableForegroundTapsThroughTheWindow \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index b0598958e5..cb0712065c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -510,13 +510,12 @@ extension RunnerTests { private func soleDisplayFrame(interfaceOrientation: Int) -> CGRect? { guard XCUIScreen.screens.count == 1 else { return nil } - let screen = UIScreen.main - let scale = screen.nativeScale - guard scale > 0 else { return nil } + // Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). + let screen = RunnerSystemApplicationScreenSize() return Self.soleScreenReferenceFrame( nativePortraitSize: CGSize( - width: screen.nativeBounds.width / scale, - height: screen.nativeBounds.height / scale + width: min(screen.width, screen.height), + height: max(screen.width, screen.height) ), interfaceOrientation: interfaceOrientation ) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index cf357347d9..db56d8e17d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -71,6 +71,14 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreen( NSUInteger *displayID ); +// The device screen's size in points, read off the AXRuntime system application, which always +// fills the screen. It does not depend on the runner process's own `UIScreen`, which the generated +// runner app, having no launch screen, can read in compatibility mode (768x1024 on an iPad). Any +// failed read answers `CGSizeZero`. +#if TARGET_OS_IOS +FOUNDATION_EXPORT CGSize RunnerSystemApplicationScreenSize(void); +#endif + // What the full-screen proof concluded. The decision is a function of these values, so a test // supplies them directly instead of restating the device rule. typedef NS_ENUM(NSUInteger, RunnerFullScreenEvidence) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index f16705845a..d6c7a6a7ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -251,6 +251,23 @@ RunnerFullScreenEvidence RunnerFullScreenEvidenceFromForeground( return cached; } +CGSize RunnerSystemApplicationScreenSize(void) { + Class element = RunnerForegroundElementClass(); + if (element == Nil || ![element respondsToSelector:@selector(systemApplication)]) { + return CGSizeZero; + } + __block CGRect screen = CGRectNull; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + id systemApp = [element systemApplication]; + if (systemApp == nil || ![systemApp respondsToSelector:@selector(frame)]) return; + screen = [systemApp frame]; + }]; + if (exception != nil || !AgentDeviceForegroundFrameIsMeasurable(screen)) { + return CGSizeZero; + } + return screen.size; +} + // One contained probe: it checks each selector before sending it and reads the target pid and both // frames inside one exception scope, so any failed read answers Unknown. The target pid comes from // the existing runner owner for that read (RunnerAXSnapshotBridge). diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 7ee8056f4c..9028e17f94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -26,7 +26,32 @@ private final class RunnerCoordinatePressCounter: NSObject { } } +/// Answers like the runner process's `UIScreen` in iPad compatibility mode, which the generated runner +/// app (no launch screen) can run in: 768x1024 whatever the device (#4091). +private final class RunnerCompatibilityModeScreenStub: NSObject { + @objc var bounds: CGRect { CGRect(x: 0, y: 0, width: 768, height: 1024) } + @objc var nativeBounds: CGRect { CGRect(x: 0, y: 0, width: 1536, height: 2048) } + @objc var nativeScale: CGFloat { 2 } +} + extension RunnerTests { + /// Installs `RunnerCompatibilityModeScreenStub` until the returned closure restores UIKit. + private func simulateCompatibilityModeScreen() throws -> () -> Void { + let getters = [ + #selector(getter: UIScreen.bounds), + #selector(getter: UIScreen.nativeBounds), + #selector(getter: UIScreen.nativeScale), + ] + let originals = try getters.map { getter in + let target = try XCTUnwrap(class_getInstanceMethod(UIScreen.self, getter)) + let stub = try XCTUnwrap(class_getInstanceMethod(RunnerCompatibilityModeScreenStub.self, getter)) + return (target, method_setImplementation(target, method_getImplementation(stub))) + } + return { + for (method, original) in originals.reversed() { method_setImplementation(method, original) } + } + } + /// Installs `RunnerUnserializableTreeStub` until the returned closure restores XCTest. private func simulateUnserializableAppTree() throws -> () -> Void { let swaps: [(Method?, Method?)] = [ @@ -181,10 +206,24 @@ extension RunnerTests { } /// #3899: a synthesized turn swaps the display's side lengths, never a placed window's smaller box. + /// #4091: the display is the one the launched host app fills, not the runner process's own + /// `UIScreen`, which the generated runner app (no launch screen) reads in compatibility mode. + @MainActor func testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + app.launch() + let appFrame = app.frame + let restoreUIKit = try simulateCompatibilityModeScreen() + defer { + restoreUIKit() + app.terminate() + } let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let screen = UIScreen.main.fixedCoordinateSpace.bounds.size + let screen = CGSize( + width: min(appFrame.width, appFrame.height), + height: max(appFrame.width, appFrame.height) + ) + XCTAssertNotEqual(screen, CGSize(width: 768, height: 1024), "the host app must not fill a 768x1024 screen") XCTAssertEqual( synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), CGRect(origin: .zero, size: screen) From f1e452f1b91e759e4a74c8cc78637185d3e7015a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 09:21:31 +0100 Subject: [PATCH 02/18] fix(ios): log a failed system-app screen read and harden its test Log AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app when the AXRuntime read fails, rename the reference-frame size to portraitSize (it is points), resolve every UIScreen getter before swapping any, and skip the display test unless the host app fills the screen. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 4 +-- .../RunnerXCTestEventBridge.m | 8 +++++ ...nerTests+SynthesizedInteractionTests.swift | 30 +++++++++++-------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index cb0712065c..a5d2d69e06 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -513,7 +513,7 @@ extension RunnerTests { // Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). let screen = RunnerSystemApplicationScreenSize() return Self.soleScreenReferenceFrame( - nativePortraitSize: CGSize( + portraitSize: CGSize( width: min(screen.width, screen.height), height: max(screen.width, screen.height) ), @@ -525,7 +525,7 @@ extension RunnerTests { /// The screen in the app's interface orientation, anchored at the origin like a window frame. An /// unknown orientation cannot say which way to lay the screen, so it names no frame. static func soleScreenReferenceFrame( - nativePortraitSize size: CGSize, + portraitSize size: CGSize, interfaceOrientation: Int ) -> CGRect? { guard size.width > 0, size.height > 0 else { return nil } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index d6c7a6a7ad..ee005d1c24 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -263,6 +263,14 @@ CGSize RunnerSystemApplicationScreenSize(void) { screen = [systemApp frame]; }]; if (exception != nil || !AgentDeviceForegroundFrameIsMeasurable(screen)) { + NSLog( + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app frame=(%g,%g %gx%g) raised=%@", + screen.origin.x, + screen.origin.y, + screen.size.width, + screen.size.height, + exception != nil ? @"yes" : @"no" + ); return CGSizeZero; } return screen.size; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 9028e17f94..41b5e94bca 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -27,7 +27,8 @@ private final class RunnerCoordinatePressCounter: NSObject { } /// Answers like the runner process's `UIScreen` in iPad compatibility mode, which the generated runner -/// app (no launch screen) can run in: 768x1024 whatever the device (#4091). +/// app (no launch screen) can run in: 768x1024 whatever the device (#4091). It replaces the getters a +/// runner-process screen read takes: `bounds`, and `nativeBounds` over `nativeScale`. private final class RunnerCompatibilityModeScreenStub: NSObject { @objc var bounds: CGRect { CGRect(x: 0, y: 0, width: 768, height: 1024) } @objc var nativeBounds: CGRect { CGRect(x: 0, y: 0, width: 1536, height: 2048) } @@ -42,10 +43,14 @@ extension RunnerTests { #selector(getter: UIScreen.nativeBounds), #selector(getter: UIScreen.nativeScale), ] - let originals = try getters.map { getter in - let target = try XCTUnwrap(class_getInstanceMethod(UIScreen.self, getter)) - let stub = try XCTUnwrap(class_getInstanceMethod(RunnerCompatibilityModeScreenStub.self, getter)) - return (target, method_setImplementation(target, method_getImplementation(stub))) + let resolved = try getters.map { getter in + ( + try XCTUnwrap(class_getInstanceMethod(UIScreen.self, getter)), + try XCTUnwrap(class_getInstanceMethod(RunnerCompatibilityModeScreenStub.self, getter)) + ) + } + let originals = resolved.map { target, stub in + (target, method_setImplementation(target, method_getImplementation(stub))) } return { for (method, original) in originals.reversed() { method_setImplementation(method, original) } @@ -162,13 +167,13 @@ extension RunnerTests { let portrait = CGSize(width: 402, height: 874) for orientation in [RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown] { XCTAssertEqual( - Self.soleScreenReferenceFrame(nativePortraitSize: portrait, interfaceOrientation: orientation), + Self.soleScreenReferenceFrame(portraitSize: portrait, interfaceOrientation: orientation), CGRect(x: 0, y: 0, width: 402, height: 874) ) } for orientation in [RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight] { XCTAssertEqual( - Self.soleScreenReferenceFrame(nativePortraitSize: portrait, interfaceOrientation: orientation), + Self.soleScreenReferenceFrame(portraitSize: portrait, interfaceOrientation: orientation), CGRect(x: 0, y: 0, width: 874, height: 402) ) } @@ -212,12 +217,11 @@ extension RunnerTests { func testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") app.launch() + defer { app.terminate() } + try XCTSkipUnless(soleScreenRouteIsOpen(for: app), "the host app's frame is the display only while it fills it") let appFrame = app.frame let restoreUIKit = try simulateCompatibilityModeScreen() - defer { - restoreUIKit() - app.terminate() - } + defer { restoreUIKit() } let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) let screen = CGSize( width: min(appFrame.width, appFrame.height), @@ -238,13 +242,13 @@ extension RunnerTests { func testSoleScreenReferenceFrameRefusesAnUnknownOrientationOrAnEmptyScreen() { XCTAssertNil( Self.soleScreenReferenceFrame( - nativePortraitSize: CGSize(width: 402, height: 874), + portraitSize: CGSize(width: 402, height: 874), interfaceOrientation: RunnerInterfaceOrientation.unknown ) ) XCTAssertNil( Self.soleScreenReferenceFrame( - nativePortraitSize: .zero, + portraitSize: .zero, interfaceOrientation: RunnerInterfaceOrientation.portrait ) ) From b2bfca5808cd0049debf8f9f59b80836dfaf78eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 09:34:58 +0100 Subject: [PATCH 03/18] test(ios): read the expected screen from the host app, within the fill tolerance The gesture-viewport test now expects the host app's frame instead of the runner's UIScreen, and both display tests compare within the 0.5 pt the fills-screen proof allows. The sole-screen frame doc names its unit. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 4 +-- ...nerTests+SynthesizedInteractionTests.swift | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index a5d2d69e06..c0563beb80 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -522,8 +522,8 @@ extension RunnerTests { } #endif - /// The screen in the app's interface orientation, anchored at the origin like a window frame. An - /// unknown orientation cannot say which way to lay the screen, so it names no frame. + /// The screen, in points, in the app's interface orientation, anchored at the origin like a window + /// frame. An unknown orientation cannot say which way to lay the screen, so it names no frame. static func soleScreenReferenceFrame( portraitSize size: CGSize, interfaceOrientation: Int diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 41b5e94bca..0923a26a42 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -134,6 +134,7 @@ extension RunnerTests { func testGestureViewportOnAnUnserializableTreeIsTheSoleScreen() throws { app.launch() mainOwned.app = app + let appFrame = app.frame let restoreXCTest = try simulateUnserializableAppTree() defer { restoreXCTest() @@ -148,11 +149,10 @@ extension RunnerTests { if soleScreenRouteIsOpen(for: app) { XCTAssertTrue(response.ok, String(describing: response.error)) - let screen = UIScreen.main.bounds XCTAssertEqual(response.data?.x, 0) XCTAssertEqual(response.data?.y, 0) - XCTAssertEqual(response.data?.x2, Double(screen.width)) - XCTAssertEqual(response.data?.y2, Double(screen.height)) + XCTAssertEqual(try XCTUnwrap(response.data?.x2), Double(appFrame.width), accuracy: 0.5) + XCTAssertEqual(try XCTUnwrap(response.data?.y2), Double(appFrame.height), accuracy: 0.5) } else { XCTAssertFalse(response.ok) XCTAssertEqual(response.error?.message, "Active app interaction viewport is unavailable") @@ -228,14 +228,17 @@ extension RunnerTests { height: max(appFrame.width, appFrame.height) ) XCTAssertNotEqual(screen, CGSize(width: 768, height: 1024), "the host app must not fill a 768x1024 screen") - XCTAssertEqual( - synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), - CGRect(origin: .zero, size: screen) - ) - XCTAssertEqual( - synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight), - CGRect(x: 0, y: 0, width: screen.height, height: screen.width) - ) + // Two AX readers of one screen; the fills-screen proof ties them within 0.5 pt. + let expectations = [ + (RunnerInterfaceOrientation.portrait, screen), + (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), + ] + for (orientation, expected) in expectations { + let frame = synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: orientation) + XCTAssertEqual(frame.origin, .zero) + XCTAssertEqual(frame.width, expected.width, accuracy: 0.5) + XCTAssertEqual(frame.height, expected.height, accuracy: 0.5) + } } #endif From 0fcd727090b772147ba61b74c76955f295c875d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 09:47:49 +0100 Subject: [PATCH 04/18] fix(ios): refuse a synthesized gesture when the sole display cannot be measured A sole display whose size AXRuntime would not report fell back to the resolved window's box, which on a placed iPad window is the #3899 basis. synthesizedRotationFrame now answers nil only for that case, and tap, long press, drag and planned gestures refuse with UNSUPPORTED_OPERATION before they log or dispatch. A multi-screen host keeps the window route. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 64 +++++++++++++------ ...nerTests+SynthesizedInteractionTests.swift | 53 ++++++++++++++- 2 files changed, 97 insertions(+), 20 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index c0563beb80..55892ce6bc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -28,7 +28,9 @@ extension RunnerTests { ) } let frame = context.referenceFrame - let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) + guard let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) else { + return Self.unmeasuredSoleDisplayOutcome(gesture: "drag") + } let start = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), in: rotationFrame, @@ -110,9 +112,17 @@ extension RunnerTests { hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." ) } + guard + let rotationFrame = synthesizedRotationFrame( + fallback: context.referenceFrame, + interfaceOrientation: orientation + ) + else { + return Self.unmeasuredSoleDisplayOutcome(gesture: "tap") + } let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: orientation), + in: rotationFrame, interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -184,9 +194,17 @@ extension RunnerTests { ) } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + guard + let rotationFrame = synthesizedRotationFrame( + fallback: context.referenceFrame, + interfaceOrientation: orientation + ) + else { + return Self.unmeasuredSoleDisplayOutcome(gesture: "long press") + } let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: orientation), + in: rotationFrame, interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -491,9 +509,10 @@ extension RunnerTests { guard RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil else { return nil } - let frame = soleDisplayFrame( - interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - ) + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let frame = soleDisplayPortraitSize().flatMap { + Self.soleScreenReferenceFrame(portraitSize: $0, interfaceOrientation: orientation) + } NSLog( "AGENT_DEVICE_RUNNER_SYNTHESIZED_SOLE_SCREEN_ROUTE resolved=%@", frame.map { NSCoder.string(for: $0) } ?? "none" @@ -503,21 +522,26 @@ extension RunnerTests { /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the - /// resolved window, which names its panel (ADR 0025). - func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect { - soleDisplayFrame(interfaceOrientation: interfaceOrientation) ?? fallback + /// resolved window, which names its panel (ADR 0025). Nil when the device's only screen would not + /// report its size: no box there is a known basis, so the gesture refuses before it dispatches. + func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect? { + guard XCUIScreen.screens.count == 1 else { return fallback } + guard let portraitSize = soleDisplayPortraitSize() else { return nil } + return Self.soleScreenReferenceFrame(portraitSize: portraitSize, interfaceOrientation: interfaceOrientation) + ?? fallback } - private func soleDisplayFrame(interfaceOrientation: Int) -> CGRect? { - guard XCUIScreen.screens.count == 1 else { return nil } - // Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). + /// Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). + private func soleDisplayPortraitSize() -> CGSize? { let screen = RunnerSystemApplicationScreenSize() - return Self.soleScreenReferenceFrame( - portraitSize: CGSize( - width: min(screen.width, screen.height), - height: max(screen.width, screen.height) - ), - interfaceOrientation: interfaceOrientation + guard screen.width > 0, screen.height > 0 else { return nil } + return CGSize(width: min(screen.width, screen.height), height: max(screen.width, screen.height)) + } + + static func unmeasuredSoleDisplayOutcome(gesture: String) -> RunnerInteractionOutcome { + .unsupported( + message: "synthesized \(gesture) could not read the size of the device's only screen, so its coordinates cannot be rotated", + hint: "Retry after the app is foregrounded; the runner log names the failed read as AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app." ) } #endif @@ -691,7 +715,9 @@ extension RunnerTests { width: plan.viewport.width, height: plan.viewport.height ) - let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) + guard let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) else { + return Self.unmeasuredSoleDisplayOutcome(gesture: "gesture") + } let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in pointer.samples.map { sample in let point = CoordinateSpaceRotation.native( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 0923a26a42..3abf2fe6da 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -35,7 +35,25 @@ private final class RunnerCompatibilityModeScreenStub: NSObject { @objc var nativeScale: CGFloat { 2 } } +/// Answers like an AXRuntime whose system application cannot be read, so the display size is unknown. +private final class RunnerUnreadableSystemApplicationStub: NSObject { + @objc class func systemApplication() -> AnyObject? { nil } +} + extension RunnerTests { + /// Installs `RunnerUnreadableSystemApplicationStub` on AXRuntime's `AXElement` until the returned + /// closure restores it. + private func simulateUnreadableSystemApplication() throws -> () -> Void { + // The bridge loads AXRuntime on its first read. + _ = RunnerSystemApplicationScreenSize() + let element: AnyClass = try XCTUnwrap(NSClassFromString("AXElement")) + let selector = NSSelectorFromString("systemApplication") + let target = try XCTUnwrap(class_getClassMethod(element, selector)) + let stub = try XCTUnwrap(class_getClassMethod(RunnerUnreadableSystemApplicationStub.self, selector)) + let original = method_setImplementation(target, method_getImplementation(stub)) + return { method_setImplementation(target, original) } + } + /// Installs `RunnerCompatibilityModeScreenStub` until the returned closure restores UIKit. private func simulateCompatibilityModeScreen() throws -> () -> Void { let getters = [ @@ -234,12 +252,45 @@ extension RunnerTests { (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), ] for (orientation, expected) in expectations { - let frame = synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: orientation) + let frame = try XCTUnwrap(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: orientation)) XCTAssertEqual(frame.origin, .zero) XCTAssertEqual(frame.width, expected.width, accuracy: 0.5) XCTAssertEqual(frame.height, expected.height, accuracy: 0.5) } } + + /// #4091: when the device's only screen will not report its size, a placed window's smaller box is + /// no basis to turn about either (#3899), so a tap through a resolved window refuses before it + /// dispatches instead of landing off target. + @MainActor + func testTapRefusesBeforeDispatchWhenTheSoleDisplayCannotBeMeasured() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + app.launch() + mainOwned.app = app + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let window = try XCTUnwrap(resolveRunnerWindowElement(app: app)).window + let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) + let context = try XCTUnwrap( + SynthesizedCoordinateContext(validating: placedWindow, resolvedWindow: window, accessibilityHealth: .healthy) + ) + let restoreAXRuntime = try simulateUnreadableSystemApplication() + defer { restoreAXRuntime() } + + XCTAssertNil( + synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) + ) + let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + guard case .unsupported(let message, _)? = outcome else { + return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") + } + guard case .unsupported(let refusal, _) = Self.unmeasuredSoleDisplayOutcome(gesture: "tap") else { + return XCTFail("the unmeasured-display outcome must be a refusal") + } + XCTAssertEqual(message, refusal) + } #endif func testSoleScreenReferenceFrameRefusesAnUnknownOrientationOrAnEmptyScreen() { From c3deb5d224422315240f11fc7a02ac5192d5bc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 10:45:01 +0100 Subject: [PATCH 05/18] fix(ios): log every failed system-app screen read and keep the hint key-free The system-app size read now logs an unavailable AXRuntime class too, the refusal hint points at the runner log without quoting its key, the rotation frame doc says an unknown orientation keeps the window box (no turn reads it), and the display test clears its cached target. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 5 +++-- .../RunnerXCTestEventBridge.m | 13 ++++++------- .../RunnerTests+SynthesizedInteractionTests.swift | 5 ++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 55892ce6bc..addeb23107 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -523,7 +523,8 @@ extension RunnerTests { /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the /// resolved window, which names its panel (ADR 0025). Nil when the device's only screen would not - /// report its size: no box there is a known basis, so the gesture refuses before it dispatches. + /// report its size: no box there is a known basis, so the gesture refuses before it dispatches. An + /// unknown orientation keeps `fallback`, which a turn that does not rotate never reads. func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect? { guard XCUIScreen.screens.count == 1 else { return fallback } guard let portraitSize = soleDisplayPortraitSize() else { return nil } @@ -541,7 +542,7 @@ extension RunnerTests { static func unmeasuredSoleDisplayOutcome(gesture: String) -> RunnerInteractionOutcome { .unsupported( message: "synthesized \(gesture) could not read the size of the device's only screen, so its coordinates cannot be rotated", - hint: "Retry after the app is foregrounded; the runner log names the failed read as AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app." + hint: "Retry after the app is foregrounded; the runner log records why the screen size could not be read." ) } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index ee005d1c24..3c6cc79b4f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -253,18 +253,17 @@ RunnerFullScreenEvidence RunnerFullScreenEvidenceFromForeground( CGSize RunnerSystemApplicationScreenSize(void) { Class element = RunnerForegroundElementClass(); - if (element == Nil || ![element respondsToSelector:@selector(systemApplication)]) { - return CGSizeZero; - } + BOOL available = element != Nil && [element respondsToSelector:@selector(systemApplication)]; __block CGRect screen = CGRectNull; - NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + NSString *exception = available ? [RunnerObjCExceptionCatcher catchException:^{ id systemApp = [element systemApplication]; if (systemApp == nil || ![systemApp respondsToSelector:@selector(frame)]) return; screen = [systemApp frame]; - }]; - if (exception != nil || !AgentDeviceForegroundFrameIsMeasurable(screen)) { + }] : nil; + if (!available || exception != nil || !AgentDeviceForegroundFrameIsMeasurable(screen)) { NSLog( - @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app frame=(%g,%g %gx%g) raised=%@", + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app available=%@ frame=(%g,%g %gx%g) raised=%@", + available ? @"yes" : @"no", screen.origin.x, screen.origin.y, screen.size.width, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 3abf2fe6da..7b81490df3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -235,7 +235,10 @@ extension RunnerTests { func testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") app.launch() - defer { app.terminate() } + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } try XCTSkipUnless(soleScreenRouteIsOpen(for: app), "the host app's frame is the display only while it fills it") let appFrame = app.frame let restoreUIKit = try simulateCompatibilityModeScreen() From b81d3cc44d31503981c16d133bbe9f841ff5431f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 12:23:15 +0100 Subject: [PATCH 06/18] docs(ios): say what the sole-screen route pays for after the display read Admission on an iPhone still skips the fills-screen probe, but every device now pays one AXRuntime read (~0.6-0.9 ms) when the caller sizes the display. Co-Authored-By: Claude Opus 5.5 --- .../AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 3c6cc79b4f..02edcb12b0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -345,8 +345,10 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application ); return @"private XCTest event synthesis unavailable: no resolved application window and no sole screen to route to"; } - // An iPhone keeps every app full-screen, so its idiom answers the rule and the route never pays - // for the probe. Off the phone idiom the route opens only on the fills-screen proof. + // An iPhone keeps every app full-screen, so its idiom answers the rule and admission never pays + // for the fills-screen probe. Off the phone idiom the route opens only on that proof. Every + // device still pays one AXRuntime read (~0.6–0.9 ms) when the caller sizes the display with + // `RunnerSystemApplicationScreenSize`. if (UIDevice.currentDevice.userInterfaceIdiom != UIUserInterfaceIdiomPhone && RunnerProbeApplicationFillsScreen(application) != RunnerFullScreenEvidenceFillsScreen) { return @"private XCTest event synthesis unavailable: no resolved application window, and the sole screen is not provably the app's full-screen frame"; From 7ea0bb305b697846f5543082c1b578027190fc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 12:49:10 +0100 Subject: [PATCH 07/18] perf(ios): read the sole display size once per runner A device's display size does not change while the runner runs, so the first successful AXRuntime read (~0.6-0.9 ms) is kept on the main-owned state and later gestures reuse it; a failed read is not kept, so the refusal stays reachable and the next gesture reads again. Co-Authored-By: Claude Opus 5.5 --- .../RunnerMainOwnedState.swift | 3 +++ .../RunnerTests+SynthesizedInteraction.swift | 11 ++++++++-- .../RunnerXCTestEventBridge.m | 5 ++--- ...nerTests+SynthesizedInteractionTests.swift | 22 +++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift index 6c26c2077b..958ad15913 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift @@ -9,6 +9,9 @@ final class RunnerMainOwnedState { var processIdentifier: Int? var accessibilityHealth: RunnerAccessibilityHealth = .unknown var needsPostSnapshotInteractionDelay = false + /// The device's only screen in portrait points, kept after its first successful read: a device's + /// display size does not change while the runner runs, and orientation is applied per gesture. + var soleDisplayPortraitSize: CGSize? nonisolated init() {} } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index addeb23107..4390a12e45 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -525,6 +525,7 @@ extension RunnerTests { /// resolved window, which names its panel (ADR 0025). Nil when the device's only screen would not /// report its size: no box there is a known basis, so the gesture refuses before it dispatches. An /// unknown orientation keeps `fallback`, which a turn that does not rotate never reads. + @MainActor func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect? { guard XCUIScreen.screens.count == 1 else { return fallback } guard let portraitSize = soleDisplayPortraitSize() else { return nil } @@ -532,11 +533,16 @@ extension RunnerTests { ?? fallback } - /// Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). + /// Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). A failed + /// read is not kept, so the next gesture reads again. + @MainActor private func soleDisplayPortraitSize() -> CGSize? { + if let cached = mainOwned.soleDisplayPortraitSize { return cached } let screen = RunnerSystemApplicationScreenSize() guard screen.width > 0, screen.height > 0 else { return nil } - return CGSize(width: min(screen.width, screen.height), height: max(screen.width, screen.height)) + let size = CGSize(width: min(screen.width, screen.height), height: max(screen.width, screen.height)) + mainOwned.soleDisplayPortraitSize = size + return size } static func unmeasuredSoleDisplayOutcome(gesture: String) -> RunnerInteractionOutcome { @@ -702,6 +708,7 @@ extension RunnerTests { : .sampled } + @MainActor func sampledPlannedGesture( app: XCUIApplication, plan: RunnerGesturePlan diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 02edcb12b0..22c22b7d9d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -346,9 +346,8 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application return @"private XCTest event synthesis unavailable: no resolved application window and no sole screen to route to"; } // An iPhone keeps every app full-screen, so its idiom answers the rule and admission never pays - // for the fills-screen probe. Off the phone idiom the route opens only on that proof. Every - // device still pays one AXRuntime read (~0.6–0.9 ms) when the caller sizes the display with - // `RunnerSystemApplicationScreenSize`. + // for the fills-screen probe. Off the phone idiom the route opens only on that proof. The display + // size is one more AXRuntime read (~0.6–0.9 ms), which the runner pays once and keeps. if (UIDevice.currentDevice.userInterfaceIdiom != UIUserInterfaceIdiomPhone && RunnerProbeApplicationFillsScreen(application) != RunnerFullScreenEvidenceFillsScreen) { return @"private XCTest event synthesis unavailable: no resolved application window, and the sole screen is not provably the app's full-screen frame"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 7b81490df3..626d33d9e8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -262,6 +262,26 @@ extension RunnerTests { } } + /// The display size costs one AXRuntime read, so the runner keeps the first successful answer and + /// later gestures take it even while AXRuntime would fail; a failed read is not kept. + @MainActor + func testSoleDisplaySizeIsReadOnceAndAFailedReadIsNotKept() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) + let portrait = RunnerInterfaceOrientation.portrait + mainOwned.soleDisplayPortraitSize = nil + let measured = try XCTUnwrap(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait)) + XCTAssertNotEqual(measured, placedWindow) + + // Nothing between the install and the restore throws, so the restore always runs. + let restoreAXRuntime = try simulateUnreadableSystemApplication() + XCTAssertEqual(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait), measured) + mainOwned.soleDisplayPortraitSize = nil + XCTAssertNil(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait)) + restoreAXRuntime() + XCTAssertEqual(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait), measured) + } + /// #4091: when the device's only screen will not report its size, a placed window's smaller box is /// no basis to turn about either (#3899), so a tap through a resolved window refuses before it /// dispatches instead of landing off target. @@ -281,6 +301,8 @@ extension RunnerTests { ) let restoreAXRuntime = try simulateUnreadableSystemApplication() defer { restoreAXRuntime() } + // An earlier gesture may have kept a successful read; this one must reach AXRuntime. + mainOwned.soleDisplayPortraitSize = nil XCTAssertNil( synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) From a81207302ba29111fa98e3a53d97e9b9b9d9d1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 13:22:36 +0100 Subject: [PATCH 08/18] fix(ios): keep the sole display size for one command, not the runner's life Display scaling can change the system application frame while the runner runs, so a size kept for the whole runner could turn later gestures about a stale box. Each command clears it and reads again; a command's gestures still share one successful read. Co-Authored-By: Claude Opus 5.5 --- .../RunnerMainOwnedState.swift | 5 +++-- .../RunnerTests+CommandExecution.swift | 1 + .../AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m | 3 ++- .../RunnerTests+SynthesizedInteractionTests.swift | 10 ++++++---- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift index 958ad15913..da6a98a004 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerMainOwnedState.swift @@ -9,8 +9,9 @@ final class RunnerMainOwnedState { var processIdentifier: Int? var accessibilityHealth: RunnerAccessibilityHealth = .unknown var needsPostSnapshotInteractionDelay = false - /// The device's only screen in portrait points, kept after its first successful read: a device's - /// display size does not change while the runner runs, and orientation is applied per gesture. + /// The device's only screen in portrait points, kept for the rest of the command after its first + /// successful read. Each command reads it again, because display scaling can change it while the + /// runner runs; orientation is applied per gesture. var soleDisplayPortraitSize: CGSize? nonisolated init() {} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 56041533a4..37616e1adf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -10,6 +10,7 @@ extension RunnerTests { alertDeadline: Date? = nil ) throws -> Response { var activeApp = activeApp + mainOwned.soleDisplayPortraitSize = nil if command.invalidatesRememberedTextEntryTap { clearRememberedTextEntryTap() } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 22c22b7d9d..b8c090ace2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -347,7 +347,8 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application } // An iPhone keeps every app full-screen, so its idiom answers the rule and admission never pays // for the fills-screen probe. Off the phone idiom the route opens only on that proof. The display - // size is one more AXRuntime read (~0.6–0.9 ms), which the runner pays once and keeps. + // size is one more AXRuntime read (~0.6–0.9 ms): a command keeps a successful one, and a failed + // one is read again by the next gesture. if (UIDevice.currentDevice.userInterfaceIdiom != UIUserInterfaceIdiomPhone && RunnerProbeApplicationFillsScreen(application) != RunnerFullScreenEvidenceFillsScreen) { return @"private XCTest event synthesis unavailable: no resolved application window, and the sole screen is not provably the app's full-screen frame"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 626d33d9e8..428c602083 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -262,10 +262,11 @@ extension RunnerTests { } } - /// The display size costs one AXRuntime read, so the runner keeps the first successful answer and - /// later gestures take it even while AXRuntime would fail; a failed read is not kept. + /// The display size costs one AXRuntime read, so a command keeps its first successful answer and + /// later gestures take it even while AXRuntime would fail. The next command reads again, because + /// display scaling can change the size, and a failed read is not kept. @MainActor - func testSoleDisplaySizeIsReadOnceAndAFailedReadIsNotKept() throws { + func testSoleDisplaySizeIsReadOncePerCommandAndAFailedReadIsNotKept() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) let portrait = RunnerInterfaceOrientation.portrait @@ -273,10 +274,11 @@ extension RunnerTests { let measured = try XCTUnwrap(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait)) XCTAssertNotEqual(measured, placedWindow) + let nextCommand = try runnerCommandFixture(#"{"command":"uptime","commandId":"next-command"}"#) // Nothing between the install and the restore throws, so the restore always runs. let restoreAXRuntime = try simulateUnreadableSystemApplication() XCTAssertEqual(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait), measured) - mainOwned.soleDisplayPortraitSize = nil + _ = try? executeOnMainPrepared(command: nextCommand, activeApp: app) XCTAssertNil(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait)) restoreAXRuntime() XCTAssertEqual(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait), measured) From a42f844e056b5e47f327d970ea30735f985559af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 14:13:34 +0100 Subject: [PATCH 09/18] fix(ios): turn about a full-screen app's window when the display size is unreadable A failed AXRuntime display read refused every synthesized gesture on a single-screen device, including iPhone taps that worked through the window before. When the sole-screen rule (RunnerResolveSoleScreen) proves the app fills the screen, its window is the display and stays the rotation basis; otherwise the gesture still refuses with UNSUPPORTED_OPERATION. Both routes log AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=rotation-basis. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/xctest-nightly.yml | 5 +- .../RunnerTests+SynthesizedInteraction.swift | 41 +++++++++---- ...nerTests+SynthesizedInteractionTests.swift | 61 ++++++++++++++++--- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index 44c3dfd232..62fabacc5d 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -151,8 +151,8 @@ jobs: # The iPad leg runs a named selection: the smallest set that can execute the off-phone half of # the sole-screen rule — the admission boundary itself, the display box a synthesized turn - # swaps (#4091 reported it off on iPads), one hostile-tree command that must reach synthesis, - # and `back`'s route witness. + # swaps (#4091 reported it off on iPads) and its refusal when that box cannot be read, one + # hostile-tree command that must reach synthesis, and `back`'s route witness. # Geometry and error cases live in the host-runnable evidence matrix instead. `xcodebuild` # exits 0 when a selection matches nothing, so the reporter below asserts the executed count # against this lane's reach (`XCTEST_LANE=nightly-ipad`, derived by check:xctest-selection). @@ -171,6 +171,7 @@ jobs: -destination "platform=iOS Simulator,id=$UDID" \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSoleScreenDisplayIDIsNamedOnlyForAnAppThatFillsItsScreen \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTapRefusesBeforeDispatchWhenTheSoleDisplayCannotBeMeasured \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryTapsThroughTheRouteThatOpens \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryWithUnprovableForegroundTapsThroughTheWindow \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 4390a12e45..b43ef5c9ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -28,7 +28,7 @@ extension RunnerTests { ) } let frame = context.referenceFrame - guard let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) else { + guard let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) else { return Self.unmeasuredSoleDisplayOutcome(gesture: "drag") } let start = CoordinateSpaceRotation.native( @@ -114,6 +114,7 @@ extension RunnerTests { } guard let rotationFrame = synthesizedRotationFrame( + app: app, fallback: context.referenceFrame, interfaceOrientation: orientation ) @@ -196,6 +197,7 @@ extension RunnerTests { let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) guard let rotationFrame = synthesizedRotationFrame( + app: app, fallback: context.referenceFrame, interfaceOrientation: orientation ) @@ -504,11 +506,7 @@ extension RunnerTests { /// anything else (a windowed iPad, a multi-screen foldable) keeps requiring a window. @MainActor func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? { - var resolvedScreen: AnyObject? - var resolvedDisplayID: UInt = 0 - guard RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil else { - return nil - } + guard appFillsSoleScreen(app) else { return nil } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let frame = soleDisplayPortraitSize().flatMap { Self.soleScreenReferenceFrame(portraitSize: $0, interfaceOrientation: orientation) @@ -520,15 +518,32 @@ extension RunnerTests { return frame } + /// The rule `RunnerResolveSoleScreen` owns: an iPhone by its idiom, an iPad only on the + /// fills-screen proof. + @MainActor + private func appFillsSoleScreen(_ app: XCUIApplication) -> Bool { + var resolvedScreen: AnyObject? + var resolvedDisplayID: UInt = 0 + return RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil + } + /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the - /// resolved window, which names its panel (ADR 0025). Nil when the device's only screen would not - /// report its size: no box there is a known basis, so the gesture refuses before it dispatches. An - /// unknown orientation keeps `fallback`, which a turn that does not rotate never reads. + /// resolved window `fallback`, which names its panel (ADR 0025). When the device's only screen + /// will not report its size, `fallback` is the basis only while the app provably fills that + /// screen; otherwise this is nil and the gesture refuses before it dispatches. An unknown + /// orientation keeps `fallback`, which a turn that does not rotate never reads. @MainActor - func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect? { + func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { guard XCUIScreen.screens.count == 1 else { return fallback } - guard let portraitSize = soleDisplayPortraitSize() else { return nil } + guard let portraitSize = soleDisplayPortraitSize() else { + let fillsScreen = appFillsSoleScreen(app) + NSLog( + "AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=rotation-basis basis=%@", + fillsScreen ? "window" : "refused" + ) + return fillsScreen ? fallback : nil + } return Self.soleScreenReferenceFrame(portraitSize: portraitSize, interfaceOrientation: interfaceOrientation) ?? fallback } @@ -547,7 +562,7 @@ extension RunnerTests { static func unmeasuredSoleDisplayOutcome(gesture: String) -> RunnerInteractionOutcome { .unsupported( - message: "synthesized \(gesture) could not read the size of the device's only screen, so its coordinates cannot be rotated", + message: "synthesized \(gesture) could not read the size of the device's only screen, and the app is not proven to fill it, so its coordinates cannot be rotated", hint: "Retry after the app is foregrounded; the runner log records why the screen size could not be read." ) } @@ -723,7 +738,7 @@ extension RunnerTests { width: plan.viewport.width, height: plan.viewport.height ) - guard let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) else { + guard let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) else { return Self.unmeasuredSoleDisplayOutcome(gesture: "gesture") } let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 428c602083..9b22390b7e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -255,7 +255,7 @@ extension RunnerTests { (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), ] for (orientation, expected) in expectations { - let frame = try XCTUnwrap(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: orientation)) + let frame = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: orientation)) XCTAssertEqual(frame.origin, .zero) XCTAssertEqual(frame.width, expected.width, accuracy: 0.5) XCTAssertEqual(frame.height, expected.height, accuracy: 0.5) @@ -271,22 +271,61 @@ extension RunnerTests { let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) let portrait = RunnerInterfaceOrientation.portrait mainOwned.soleDisplayPortraitSize = nil - let measured = try XCTUnwrap(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait)) + let measured = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait)) XCTAssertNotEqual(measured, placedWindow) let nextCommand = try runnerCommandFixture(#"{"command":"uptime","commandId":"next-command"}"#) // Nothing between the install and the restore throws, so the restore always runs. let restoreAXRuntime = try simulateUnreadableSystemApplication() - XCTAssertEqual(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait), measured) + XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait), measured) _ = try? executeOnMainPrepared(command: nextCommand, activeApp: app) - XCTAssertNil(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait)) + XCTAssertNotEqual( + synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait), + measured, + "the next command must read the size again rather than reuse the kept one" + ) restoreAXRuntime() - XCTAssertEqual(synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: portrait), measured) + XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait), measured) } - /// #4091: when the device's only screen will not report its size, a placed window's smaller box is - /// no basis to turn about either (#3899), so a tap through a resolved window refuses before it - /// dispatches instead of landing off target. + /// #4091: when the device's only screen will not report its size, an app the sole-screen rule + /// proves full-screen (an iPhone by its idiom) has a window that is the display, so its tap still + /// turns about that window and is dispatched. + @MainActor + func testTapTurnsAboutTheWindowWhenTheSoleDisplayCannotBeMeasuredOnAFullScreenApp() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + app.launch() + mainOwned.app = app + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) + let context = try XCTUnwrap( + SynthesizedCoordinateContext(validating: resolved.frame, resolvedWindow: resolved.window, accessibilityHealth: .healthy) + ) + let restoreAXRuntime = try simulateUnreadableSystemApplication() + defer { restoreAXRuntime() } + mainOwned.soleDisplayPortraitSize = nil + try XCTSkipUnless( + soleScreenRouteIsOpen(for: app), + "without AXRuntime an iPad cannot prove the app fills the screen; the refusal test covers it" + ) + + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + XCTAssertEqual( + synthesizedRotationFrame(app: app, fallback: resolved.frame, interfaceOrientation: orientation), + resolved.frame + ) + let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + guard case .performed? = outcome else { + return XCTFail("expected the tap to be dispatched about the window, got \(String(describing: outcome))") + } + } + + /// #4091: when the device's only screen will not report its size and the app is not proven to fill + /// it, a placed window's smaller box is no basis to turn about (#3899), so a tap through a resolved + /// window refuses before it dispatches instead of landing off target. @MainActor func testTapRefusesBeforeDispatchWhenTheSoleDisplayCannotBeMeasured() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") @@ -305,9 +344,13 @@ extension RunnerTests { defer { restoreAXRuntime() } // An earlier gesture may have kept a successful read; this one must reach AXRuntime. mainOwned.soleDisplayPortraitSize = nil + try XCTSkipIf( + soleScreenRouteIsOpen(for: app), + "an app proven full-screen turns about its window; the full-screen test covers it" + ) XCTAssertNil( - synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) + synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) ) let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .unsupported(let message, _)? = outcome else { From d7c5a0332da4c62e4c1a34ba8032405063d025e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 14:21:40 +0100 Subject: [PATCH 10/18] fix(ios): read the display box only for turns that move the axes Portrait and an unknown orientation never read the rotation box, so an unreadable display no longer refuses their gestures and they skip the AXRuntime read entirely. The AX-failure tap tests now run in landscape, where the box is read. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 11 ++++- ...nerTests+SynthesizedInteractionTests.swift | 43 ++++++++++++++++--- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index b43ef5c9ad..778062c327 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -531,11 +531,18 @@ extension RunnerTests { /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the /// resolved window `fallback`, which names its panel (ADR 0025). When the device's only screen /// will not report its size, `fallback` is the basis only while the app provably fills that - /// screen; otherwise this is nil and the gesture refuses before it dispatches. An unknown - /// orientation keeps `fallback`, which a turn that does not rotate never reads. + /// screen; otherwise this is nil and the gesture refuses before it dispatches. Portrait and an + /// unknown orientation keep the axes, so they read no box and keep `fallback` without a read. @MainActor func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { guard XCUIScreen.screens.count == 1 else { return fallback } + switch interfaceOrientation { + case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight, + RunnerInterfaceOrientation.portraitUpsideDown: + break + default: + return fallback + } guard let portraitSize = soleDisplayPortraitSize() else { let fillsScreen = appFillsSoleScreen(app) NSLog( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 9b22390b7e..67a409e999 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -54,6 +54,25 @@ extension RunnerTests { return { method_setImplementation(target, original) } } + /// Turns the device to landscape until the host app's interface follows, so a synthesized turn + /// needs its display box; the returned closure turns it back to portrait. + @MainActor + private func rotateHostAppToLandscape() throws -> () -> Void { + XCUIDevice.shared.orientation = .landscapeLeft + let restore = { XCUIDevice.shared.orientation = .portrait } + let landscape = [RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight] + let deadline = Date().addingTimeInterval(5) + while !landscape.contains(Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))), + Date() < deadline + { + RunLoop.current.run(until: Date().addingTimeInterval(0.1)) + } + let turned = landscape.contains(Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))) + if !turned { restore() } + _ = try XCTUnwrap(turned ? true : nil, "the host app did not turn to landscape") + return restore + } + /// Installs `RunnerCompatibilityModeScreenStub` until the returned closure restores UIKit. private func simulateCompatibilityModeScreen() throws -> () -> Void { let getters = [ @@ -249,9 +268,10 @@ extension RunnerTests { height: max(appFrame.width, appFrame.height) ) XCTAssertNotEqual(screen, CGSize(width: 768, height: 1024), "the host app must not fill a 768x1024 screen") - // Two AX readers of one screen; the fills-screen proof ties them within 0.5 pt. + // Two AX readers of one screen; the fills-screen proof ties them within 0.5 pt. Only the turns + // that move the axes read the box. let expectations = [ - (RunnerInterfaceOrientation.portrait, screen), + (RunnerInterfaceOrientation.portraitUpsideDown, screen), (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), ] for (orientation, expected) in expectations { @@ -269,23 +289,23 @@ extension RunnerTests { func testSoleDisplaySizeIsReadOncePerCommandAndAFailedReadIsNotKept() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let portrait = RunnerInterfaceOrientation.portrait + let landscape = RunnerInterfaceOrientation.landscapeRight mainOwned.soleDisplayPortraitSize = nil - let measured = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait)) + let measured = try XCTUnwrap(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape)) XCTAssertNotEqual(measured, placedWindow) let nextCommand = try runnerCommandFixture(#"{"command":"uptime","commandId":"next-command"}"#) // Nothing between the install and the restore throws, so the restore always runs. let restoreAXRuntime = try simulateUnreadableSystemApplication() - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait), measured) + XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) _ = try? executeOnMainPrepared(command: nextCommand, activeApp: app) XCTAssertNotEqual( - synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait), + synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured, "the next command must read the size again rather than reuse the kept one" ) restoreAXRuntime() - XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: portrait), measured) + XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) } /// #4091: when the device's only screen will not report its size, an app the sole-screen rule @@ -300,6 +320,8 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } + let restoreOrientation = try rotateHostAppToLandscape() + defer { restoreOrientation() } let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) let context = try XCTUnwrap( SynthesizedCoordinateContext(validating: resolved.frame, resolvedWindow: resolved.window, accessibilityHealth: .healthy) @@ -335,6 +357,8 @@ extension RunnerTests { invalidateCachedTarget(reason: "unit_test_cleanup") app.terminate() } + let restoreOrientation = try rotateHostAppToLandscape() + defer { restoreOrientation() } let window = try XCTUnwrap(resolveRunnerWindowElement(app: app)).window let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) let context = try XCTUnwrap( @@ -352,6 +376,11 @@ extension RunnerTests { XCTAssertNil( synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight) ) + XCTAssertEqual( + synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), + placedWindow, + "a portrait turn keeps the axes, so it reads no box and must not refuse" + ) let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .unsupported(let message, _)? = outcome else { return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") From 4314cf189300a86f141d4989a6449a369f694941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 14:22:26 +0100 Subject: [PATCH 11/18] ci(ios): keep the nightly iPad leg at five tests The display-box test is stub-based and runs on the iPhone full suite, so the iPad leg keeps the refusal test, which only an iPad can exercise. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/xctest-nightly.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index 62fabacc5d..8c7b4384ac 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -150,8 +150,8 @@ jobs: ' # The iPad leg runs a named selection: the smallest set that can execute the off-phone half of - # the sole-screen rule — the admission boundary itself, the display box a synthesized turn - # swaps (#4091 reported it off on iPads) and its refusal when that box cannot be read, one + # the sole-screen rule — the admission boundary itself, the refusal when the display box a + # synthesized turn swaps cannot be read and the app is not proven full-screen (#4091), one # hostile-tree command that must reach synthesis, and `back`'s route witness. # Geometry and error cases live in the host-runnable evidence matrix instead. `xcodebuild` # exits 0 when a selection matches nothing, so the reporter below asserts the executed count @@ -170,7 +170,6 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=$UDID" \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSoleScreenDisplayIDIsNamedOnlyForAnAppThatFillsItsScreen \ - -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTapRefusesBeforeDispatchWhenTheSoleDisplayCannotBeMeasured \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryTapsThroughTheRouteThatOpens \ From 409188eea307f0bd6f71e829ea0f1f8a846d0b9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 14:31:27 +0100 Subject: [PATCH 12/18] test(ios): share the AX-failure tap tests' launch, landscape and stub setup Co-Authored-By: Claude Opus 5.5 --- ...nerTests+SynthesizedInteractionTests.swift | 74 ++++++++++--------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 67a409e999..742cd64eb3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -73,6 +73,41 @@ extension RunnerTests { return restore } + /// The state both AX-failure tap tests start from: the host app launched and turned to landscape, + /// a tap context on its resolved window (measured at `windowFrame` when given), and AXRuntime's + /// display read failing with no kept size. The returned cleanup undoes every step. + @MainActor + private func launchLandscapeWithUnreadableDisplay( + windowFrame: CGRect? = nil + ) throws -> (context: SynthesizedCoordinateContext, cleanup: () -> Void) { + var cleanups: [() -> Void] = [] + let cleanup = { for step in cleanups.reversed() { step() } } + do { + app.launch() + mainOwned.app = app + cleanups.append { + self.invalidateCachedTarget(reason: "unit_test_cleanup") + self.app.terminate() + } + cleanups.append(try rotateHostAppToLandscape()) + let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) + let context = try XCTUnwrap( + SynthesizedCoordinateContext( + validating: windowFrame ?? resolved.frame, + resolvedWindow: resolved.window, + accessibilityHealth: .healthy + ) + ) + cleanups.append(try simulateUnreadableSystemApplication()) + // An earlier gesture may have kept a successful read; this one must reach AXRuntime. + mainOwned.soleDisplayPortraitSize = nil + return (context, cleanup) + } catch { + cleanup() + throw error + } + } + /// Installs `RunnerCompatibilityModeScreenStub` until the returned closure restores UIKit. private func simulateCompatibilityModeScreen() throws -> () -> Void { let getters = [ @@ -314,21 +349,8 @@ extension RunnerTests { @MainActor func testTapTurnsAboutTheWindowWhenTheSoleDisplayCannotBeMeasuredOnAFullScreenApp() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") - app.launch() - mainOwned.app = app - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let restoreOrientation = try rotateHostAppToLandscape() - defer { restoreOrientation() } - let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) - let context = try XCTUnwrap( - SynthesizedCoordinateContext(validating: resolved.frame, resolvedWindow: resolved.window, accessibilityHealth: .healthy) - ) - let restoreAXRuntime = try simulateUnreadableSystemApplication() - defer { restoreAXRuntime() } - mainOwned.soleDisplayPortraitSize = nil + let (context, cleanup) = try launchLandscapeWithUnreadableDisplay() + defer { cleanup() } try XCTSkipUnless( soleScreenRouteIsOpen(for: app), "without AXRuntime an iPad cannot prove the app fills the screen; the refusal test covers it" @@ -336,8 +358,8 @@ extension RunnerTests { let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: resolved.frame, interfaceOrientation: orientation), - resolved.frame + synthesizedRotationFrame(app: app, fallback: context.referenceFrame, interfaceOrientation: orientation), + context.referenceFrame ) let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .performed? = outcome else { @@ -351,23 +373,9 @@ extension RunnerTests { @MainActor func testTapRefusesBeforeDispatchWhenTheSoleDisplayCannotBeMeasured() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") - app.launch() - mainOwned.app = app - defer { - invalidateCachedTarget(reason: "unit_test_cleanup") - app.terminate() - } - let restoreOrientation = try rotateHostAppToLandscape() - defer { restoreOrientation() } - let window = try XCTUnwrap(resolveRunnerWindowElement(app: app)).window let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let context = try XCTUnwrap( - SynthesizedCoordinateContext(validating: placedWindow, resolvedWindow: window, accessibilityHealth: .healthy) - ) - let restoreAXRuntime = try simulateUnreadableSystemApplication() - defer { restoreAXRuntime() } - // An earlier gesture may have kept a successful read; this one must reach AXRuntime. - mainOwned.soleDisplayPortraitSize = nil + let (context, cleanup) = try launchLandscapeWithUnreadableDisplay(windowFrame: placedWindow) + defer { cleanup() } try XCTSkipIf( soleScreenRouteIsOpen(for: app), "an app proven full-screen turns about its window; the full-screen test covers it" From 4690ea7ba3edcd47aa1125b5cfb00262e616155f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 14:52:09 +0100 Subject: [PATCH 13/18] refactor(ios): name the sole-screen admission rule and the turns that read the box Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 778062c327..382f73c149 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -506,7 +506,7 @@ extension RunnerTests { /// anything else (a windowed iPad, a multi-screen foldable) keeps requiring a window. @MainActor func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? { - guard appFillsSoleScreen(app) else { return nil } + guard soleScreenRouteAdmits(app) else { return nil } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let frame = soleDisplayPortraitSize().flatMap { Self.soleScreenReferenceFrame(portraitSize: $0, interfaceOrientation: orientation) @@ -518,10 +518,10 @@ extension RunnerTests { return frame } - /// The rule `RunnerResolveSoleScreen` owns: an iPhone by its idiom, an iPad only on the - /// fills-screen proof. + /// The sole-screen admission rule `RunnerResolveSoleScreen` owns: an iPhone by its idiom, an iPad + /// only on the fills-screen proof. @MainActor - private func appFillsSoleScreen(_ app: XCUIApplication) -> Bool { + private func soleScreenRouteAdmits(_ app: XCUIApplication) -> Bool { var resolvedScreen: AnyObject? var resolvedDisplayID: UInt = 0 return RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil @@ -532,7 +532,8 @@ extension RunnerTests { /// resolved window `fallback`, which names its panel (ADR 0025). When the device's only screen /// will not report its size, `fallback` is the basis only while the app provably fills that /// screen; otherwise this is nil and the gesture refuses before it dispatches. Portrait and an - /// unknown orientation keep the axes, so they read no box and keep `fallback` without a read. + /// unknown orientation leave the point as it is, so they keep `fallback` without a read; landscape + /// swaps the box's sides and upside-down mirrors the point across them, so both read it. @MainActor func synthesizedRotationFrame(app: XCUIApplication, fallback: CGRect, interfaceOrientation: Int) -> CGRect? { guard XCUIScreen.screens.count == 1 else { return fallback } @@ -544,12 +545,12 @@ extension RunnerTests { return fallback } guard let portraitSize = soleDisplayPortraitSize() else { - let fillsScreen = appFillsSoleScreen(app) + let routeAdmits = soleScreenRouteAdmits(app) NSLog( "AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=rotation-basis basis=%@", - fillsScreen ? "window" : "refused" + routeAdmits ? "window" : "refused" ) - return fillsScreen ? fallback : nil + return routeAdmits ? fallback : nil } return Self.soleScreenReferenceFrame(portraitSize: portraitSize, interfaceOrientation: interfaceOrientation) ?? fallback From 3e2b177731d8971184c58349a16ccce94997460b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 15:33:12 +0100 Subject: [PATCH 14/18] test(ios): pin that a portrait tap dispatches when the display is unreadable Co-Authored-By: Claude Opus 5.5 --- ...nerTests+SynthesizedInteractionTests.swift | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 742cd64eb3..9dd9aebfc4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -73,11 +73,12 @@ extension RunnerTests { return restore } - /// The state both AX-failure tap tests start from: the host app launched and turned to landscape, - /// a tap context on its resolved window (measured at `windowFrame` when given), and AXRuntime's - /// display read failing with no kept size. The returned cleanup undoes every step. + /// The state the AX-failure tap tests start from: the host app launched (turned to landscape when + /// asked), a tap context on its resolved window (measured at `windowFrame` when given), and + /// AXRuntime's display read failing with no kept size. The returned cleanup undoes every step. @MainActor - private func launchLandscapeWithUnreadableDisplay( + private func launchWithUnreadableDisplay( + landscape: Bool, windowFrame: CGRect? = nil ) throws -> (context: SynthesizedCoordinateContext, cleanup: () -> Void) { var cleanups: [() -> Void] = [] @@ -89,7 +90,7 @@ extension RunnerTests { self.invalidateCachedTarget(reason: "unit_test_cleanup") self.app.terminate() } - cleanups.append(try rotateHostAppToLandscape()) + if landscape { cleanups.append(try rotateHostAppToLandscape()) } let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) let context = try XCTUnwrap( SynthesizedCoordinateContext( @@ -343,13 +344,32 @@ extension RunnerTests { XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) } + /// #4091: a portrait turn leaves the point as it is, so an unreadable display box neither refuses + /// it nor is read for it: the tap is dispatched even through a placed window on either family. + @MainActor + func testPortraitTapDispatchesWhenTheSoleDisplayCannotBeMeasured() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) + let (context, cleanup) = try launchWithUnreadableDisplay(landscape: false, windowFrame: placedWindow) + defer { cleanup() } + XCTAssertEqual( + Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)), + RunnerInterfaceOrientation.portrait + ) + + let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) + guard case .performed? = outcome else { + return XCTFail("expected a portrait tap to be dispatched, got \(String(describing: outcome))") + } + } + /// #4091: when the device's only screen will not report its size, an app the sole-screen rule /// proves full-screen (an iPhone by its idiom) has a window that is the display, so its tap still /// turns about that window and is dispatched. @MainActor func testTapTurnsAboutTheWindowWhenTheSoleDisplayCannotBeMeasuredOnAFullScreenApp() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") - let (context, cleanup) = try launchLandscapeWithUnreadableDisplay() + let (context, cleanup) = try launchWithUnreadableDisplay(landscape: true) defer { cleanup() } try XCTSkipUnless( soleScreenRouteIsOpen(for: app), @@ -374,7 +394,7 @@ extension RunnerTests { func testTapRefusesBeforeDispatchWhenTheSoleDisplayCannotBeMeasured() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let (context, cleanup) = try launchLandscapeWithUnreadableDisplay(windowFrame: placedWindow) + let (context, cleanup) = try launchWithUnreadableDisplay(landscape: true, windowFrame: placedWindow) defer { cleanup() } try XCTSkipIf( soleScreenRouteIsOpen(for: app), From 5c470cbba941783937f237333b527f044fc2e4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 16:20:31 +0100 Subject: [PATCH 15/18] docs(ios): place the display-size read after admission and spell longpress as the CLI does Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 14 ++++++++++---- .../RunnerXCTestEventBridge.m | 5 +++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 382f73c149..e36d6c6252 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -28,7 +28,9 @@ extension RunnerTests { ) } let frame = context.referenceFrame - guard let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) else { + guard + let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) + else { return Self.unmeasuredSoleDisplayOutcome(gesture: "drag") } let start = CoordinateSpaceRotation.native( @@ -202,7 +204,7 @@ extension RunnerTests { interfaceOrientation: orientation ) else { - return Self.unmeasuredSoleDisplayOutcome(gesture: "long press") + return Self.unmeasuredSoleDisplayOutcome(gesture: "longpress") } let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), @@ -570,7 +572,9 @@ extension RunnerTests { static func unmeasuredSoleDisplayOutcome(gesture: String) -> RunnerInteractionOutcome { .unsupported( - message: "synthesized \(gesture) could not read the size of the device's only screen, and the app is not proven to fill it, so its coordinates cannot be rotated", + message: + "synthesized \(gesture) could not read the size of the device's only screen, and the app is not " + + "proven to fill it, so its coordinates cannot be rotated", hint: "Retry after the app is foregrounded; the runner log records why the screen size could not be read." ) } @@ -746,7 +750,9 @@ extension RunnerTests { width: plan.viewport.width, height: plan.viewport.height ) - guard let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) else { + guard + let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) + else { return Self.unmeasuredSoleDisplayOutcome(gesture: "gesture") } let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index b8c090ace2..a16abd14c0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -347,8 +347,9 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application } // An iPhone keeps every app full-screen, so its idiom answers the rule and admission never pays // for the fills-screen probe. Off the phone idiom the route opens only on that proof. The display - // size is one more AXRuntime read (~0.6–0.9 ms): a command keeps a successful one, and a failed - // one is read again by the next gesture. + // size is a separate AXRuntime read (~0.6–0.9 ms) the caller makes after admission, the first time + // a command needs the box; a command keeps a successful read, and the next gesture retries a failed + // one. if (UIDevice.currentDevice.userInterfaceIdiom != UIUserInterfaceIdiomPhone && RunnerProbeApplicationFillsScreen(application) != RunnerFullScreenEvidenceFillsScreen) { return @"private XCTest event synthesis unavailable: no resolved application window, and the sole screen is not provably the app's full-screen frame"; From bf0b1ca48d9e068b1c9d2b4ab57220ddec7c4b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 16:50:29 +0100 Subject: [PATCH 16/18] fix(ios): dispatch a window-less portrait tap when the display size is unreadable With the window query rejected and the AXRuntime display read failing there is no box at all. A portrait tap or long press keeps its point, so on an admitted sole screen it is now dispatched as given; any other turn still refuses with UNSUPPORTED_OPERATION. Both log stage=rotation-basis window=none. Tap and long press share one point-gesture resolver. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 144 +++++++++++++----- ...nerTests+SynthesizedInteractionTests.swift | 72 +++++++-- 2 files changed, 161 insertions(+), 55 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index e36d6c6252..78719091a9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -108,37 +108,28 @@ extension RunnerTests { ) } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let context = context ?? synthesizedCoordinateContext(app: app) else { - return .unsupported( - message: "synthesized coordinate tap could not resolve an app window with a finite screen frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ) - } - guard - let rotationFrame = synthesizedRotationFrame( - app: app, - fallback: context.referenceFrame, - interfaceOrientation: orientation - ) - else { - return Self.unmeasuredSoleDisplayOutcome(gesture: "tap") - } - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: rotationFrame, - interfaceOrientation: orientation + let resolution = pointGestureTarget( + app: app, + x: x, + y: y, + orientation: orientation, + context: context ?? synthesizedCoordinateContext(app: app), + gesture: .tap ) + guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { + return resolution.refusal + } logSynthesizedDispatch( kind: "tap", start: point, end: nil, - referenceFrame: context.referenceFrame, + referenceFrame: referenceFrame, orientation: orientation ) var message: NSString? let status = RunnerSynthesizedGesture.synthesizeTap( withApplication: app, - resolvedWindow: context.resolvedWindow, + resolvedWindow: resolvedWindow, x: Double(point.x), y: Double(point.y), deadline: deadline, @@ -190,32 +181,23 @@ extension RunnerTests { hint: "Retry with finite x and y values and a positive durationMs." ) } - guard let context else { - return .unsupported( - message: "synthesized coordinate long press could not resolve an app window with a finite screen frame", - hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." - ) - } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard - let rotationFrame = synthesizedRotationFrame( - app: app, - fallback: context.referenceFrame, - interfaceOrientation: orientation - ) - else { - return Self.unmeasuredSoleDisplayOutcome(gesture: "longpress") - } - let point = CoordinateSpaceRotation.native( - point: CGPoint(x: x, y: y), - in: rotationFrame, - interfaceOrientation: orientation + let resolution = pointGestureTarget( + app: app, + x: x, + y: y, + orientation: orientation, + context: context, + gesture: .longpress ) + guard case .target(let point, let resolvedWindow, let referenceFrame) = resolution else { + return resolution.refusal + } logSynthesizedDispatch( kind: "longPress", start: point, end: nil, - referenceFrame: context.referenceFrame, + referenceFrame: referenceFrame, orientation: orientation ) let samples: [[String: NSNumber]] = [0, durationMs].map { offsetMs in @@ -227,7 +209,7 @@ extension RunnerTests { } if let message = RunnerSynthesizedGesture.synthesizeGesture( withApplication: app, - resolvedWindow: context.resolvedWindow, + resolvedWindow: resolvedWindow, pointerSamples: [samples] ) { return .unsupported( @@ -558,6 +540,84 @@ extension RunnerTests { ?? fallback } + private enum SynthesizedPointGesture: String { + case tap + case longpress + + /// The gesture's name in the unresolved-window refusal, which predates this enum. + var unresolvedWindowName: String { self == .tap ? "tap" : "long press" } + } + + private enum PointGestureResolution { + /// A nil `resolvedWindow` routes the record to the sole screen. + case target(point: CGPoint, resolvedWindow: XCUIElement?, referenceFrame: CGRect) + case refused(RunnerInteractionOutcome) + + /// Read only on the branch that is not a target, so its `.performed` answer is never returned. + var refusal: RunnerInteractionOutcome { + if case .refused(let outcome) = self { return outcome } + return .performed + } + } + + /// Where a synthesized point gesture lands: its native point and the window that routes it, or the + /// refusal that stops it before dispatch. + @MainActor + private func pointGestureTarget( + app: XCUIApplication, + x: Double, + y: Double, + orientation: Int, + context: SynthesizedCoordinateContext?, + gesture: SynthesizedPointGesture + ) -> PointGestureResolution { + if let context { + guard + let rotationFrame = synthesizedRotationFrame( + app: app, + fallback: context.referenceFrame, + interfaceOrientation: orientation + ) + else { + return .refused(Self.unmeasuredSoleDisplayOutcome(gesture: gesture.rawValue)) + } + let point = CoordinateSpaceRotation.native( + point: CGPoint(x: x, y: y), + in: rotationFrame, + interfaceOrientation: orientation + ) + return .target(point: point, resolvedWindow: context.resolvedWindow, referenceFrame: context.referenceFrame) + } + if let unrotated = unrotatedSoleScreenPoint(app: app, x: x, y: y, interfaceOrientation: orientation) { + return .target(point: unrotated, resolvedWindow: nil, referenceFrame: .null) + } + return .refused( + .unsupported( + message: "synthesized coordinate \(gesture.unresolvedWindowName) could not resolve an app window with a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + ) + } + + /// A point gesture with no context: no window resolved and the sole screen's size is unreadable, + /// so there is no box (#4091). On an admitted sole screen a portrait turn leaves the point as it + /// is, so it is dispatched as given; any other turn needs the box and refuses. Nil refuses. + @MainActor + private func unrotatedSoleScreenPoint( + app: XCUIApplication, + x: Double, + y: Double, + interfaceOrientation: Int + ) -> CGPoint? { + guard XCUIScreen.screens.count == 1, soleScreenRouteAdmits(app) else { return nil } + let unrotated = interfaceOrientation == RunnerInterfaceOrientation.portrait + NSLog( + "AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=rotation-basis basis=%@ window=none", + unrotated ? "unrotated" : "refused" + ) + return unrotated ? CGPoint(x: x, y: y) : nil + } + /// Not `UIScreen.main`: the runner process can read that in compatibility mode (#4091). A failed /// read is not kept, so the next gesture reads again. @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 9dd9aebfc4..448469432c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -74,13 +74,15 @@ extension RunnerTests { } /// The state the AX-failure tap tests start from: the host app launched (turned to landscape when - /// asked), a tap context on its resolved window (measured at `windowFrame` when given), and - /// AXRuntime's display read failing with no kept size. The returned cleanup undoes every step. + /// asked), a tap context on its resolved window (measured at `windowFrame` when given) or, when + /// the window query is rejected, an unserializable tree and no context, and AXRuntime's display + /// read failing with no kept size. The returned cleanup undoes every step. @MainActor private func launchWithUnreadableDisplay( landscape: Bool, - windowFrame: CGRect? = nil - ) throws -> (context: SynthesizedCoordinateContext, cleanup: () -> Void) { + windowFrame: CGRect? = nil, + windowQueryRejected: Bool = false + ) throws -> (context: SynthesizedCoordinateContext?, cleanup: () -> Void) { var cleanups: [() -> Void] = [] let cleanup = { for step in cleanups.reversed() { step() } } do { @@ -91,15 +93,19 @@ extension RunnerTests { self.app.terminate() } if landscape { cleanups.append(try rotateHostAppToLandscape()) } - let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) - let context = try XCTUnwrap( - SynthesizedCoordinateContext( - validating: windowFrame ?? resolved.frame, - resolvedWindow: resolved.window, - accessibilityHealth: .healthy + var context: SynthesizedCoordinateContext? + if !windowQueryRejected { + let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) + context = try XCTUnwrap( + SynthesizedCoordinateContext( + validating: windowFrame ?? resolved.frame, + resolvedWindow: resolved.window, + accessibilityHealth: .healthy + ) ) - ) + } cleanups.append(try simulateUnreadableSystemApplication()) + if windowQueryRejected { cleanups.append(try simulateUnserializableAppTree()) } // An earlier gesture may have kept a successful read; this one must reach AXRuntime. mainOwned.soleDisplayPortraitSize = nil return (context, cleanup) @@ -363,6 +369,45 @@ extension RunnerTests { } } + /// #4091: with the window query rejected (a tree XCTest cannot serialize) and the display + /// unreadable there is no box at all. A portrait turn leaves the point as it is, so an admitted sole + /// screen still takes the tap as given. + @MainActor + func testWindowlessPortraitTapDispatchesWhenTheSoleDisplayCannotBeMeasured() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + let (_, cleanup) = try launchWithUnreadableDisplay(landscape: false, windowQueryRejected: true) + defer { cleanup() } + try XCTSkipUnless( + soleScreenRouteIsOpen(for: app), + "without AXRuntime an iPad cannot admit the sole-screen route; the landscape test covers refusal" + ) + XCTAssertEqual( + Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)), + RunnerInterfaceOrientation.portrait + ) + + let outcome = synthesizedTapAt(app: app, x: 40, y: 40) + guard case .performed? = outcome else { + return XCTFail("expected a window-less portrait tap to be dispatched, got \(String(describing: outcome))") + } + } + + /// #4091: a landscape turn needs the box, and with the window query rejected no window can stand in + /// for an unreadable display, so the tap refuses before it dispatches instead of falling back to the + /// runner's own `UIScreen`. + @MainActor + func testWindowlessLandscapeTapRefusesWhenTheSoleDisplayCannotBeMeasured() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + let (_, cleanup) = try launchWithUnreadableDisplay(landscape: true, windowQueryRejected: true) + defer { cleanup() } + + let outcome = synthesizedTapAt(app: app, x: 40, y: 40) + guard case .unsupported(let message, _)? = outcome else { + return XCTFail("expected a refusal before dispatch, got \(String(describing: outcome))") + } + XCTAssertEqual(message, "synthesized coordinate tap could not resolve an app window with a finite screen frame") + } + /// #4091: when the device's only screen will not report its size, an app the sole-screen rule /// proves full-screen (an iPhone by its idiom) has a window that is the display, so its tap still /// turns about that window and is dispatched. @@ -377,9 +422,10 @@ extension RunnerTests { ) let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let windowFrame = try XCTUnwrap(context).referenceFrame XCTAssertEqual( - synthesizedRotationFrame(app: app, fallback: context.referenceFrame, interfaceOrientation: orientation), - context.referenceFrame + synthesizedRotationFrame(app: app, fallback: windowFrame, interfaceOrientation: orientation), + windowFrame ) let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .performed? = outcome else { From 00d36437bac8d07f5a7cae5b4180a428fd406a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 17:13:48 +0100 Subject: [PATCH 17/18] test(ios): name exactly which turns read the display box Co-Authored-By: Claude Opus 5.5 --- .../UnitTests/RunnerTests+SynthesizedInteractionTests.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 448469432c..652aad7c41 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -310,8 +310,8 @@ extension RunnerTests { height: max(appFrame.width, appFrame.height) ) XCTAssertNotEqual(screen, CGSize(width: 768, height: 1024), "the host app must not fill a 768x1024 screen") - // Two AX readers of one screen; the fills-screen proof ties them within 0.5 pt. Only the turns - // that move the axes read the box. + // Two AX readers of one screen; the fills-screen proof ties them within 0.5 pt. Portrait and an + // unknown orientation skip the read; landscape and upside-down read the box, so they are checked. let expectations = [ (RunnerInterfaceOrientation.portraitUpsideDown, screen), (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), @@ -453,7 +453,7 @@ extension RunnerTests { XCTAssertEqual( synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), placedWindow, - "a portrait turn keeps the axes, so it reads no box and must not refuse" + "portrait skips the box read (only landscape and upside-down read it), so it must not refuse" ) let outcome = synthesizedTapAt(app: app, x: 40, y: 40, context: context) guard case .unsupported(let message, _)? = outcome else { From 1fbe15b9ff45dcc98fe4fc4bf4cbf282b0386a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 3 Nov 2026 09:02:02 +0100 Subject: [PATCH 18/18] fix(ios): read sole-display size from XCTest screen metrics --- .../RunnerTests+SynthesizedInteraction.swift | 2 +- .../RunnerXCTestEventBridge.h | 9 +- .../RunnerXCTestEventBridge.m | 43 +++++---- ...nerTests+SynthesizedInteractionTests.swift | 92 ++++++++++++++----- 4 files changed, 98 insertions(+), 48 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 78719091a9..7a20a5de0f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -623,7 +623,7 @@ extension RunnerTests { @MainActor private func soleDisplayPortraitSize() -> CGSize? { if let cached = mainOwned.soleDisplayPortraitSize { return cached } - let screen = RunnerSystemApplicationScreenSize() + let screen = RunnerSoleScreenSize() guard screen.width > 0, screen.height > 0 else { return nil } let size = CGSize(width: min(screen.width, screen.height), height: max(screen.width, screen.height)) mainOwned.soleDisplayPortraitSize = size diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index db56d8e17d..8e0c1c6b4c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -71,12 +71,11 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreen( NSUInteger *displayID ); -// The device screen's size in points, read off the AXRuntime system application, which always -// fills the screen. It does not depend on the runner process's own `UIScreen`, which the generated -// runner app, having no launch screen, can read in compatibility mode (768x1024 on an iPad). Any -// failed read answers `CGSizeZero`. +// The sole XCTest display's pixel bounds divided by its scale, in points. Neither the runner's +// compatibility-mode `UIScreen` nor the system application's possibly empty AX frame supplies +// this measurement. Any failed read or a screen count other than one answers `CGSizeZero`. #if TARGET_OS_IOS -FOUNDATION_EXPORT CGSize RunnerSystemApplicationScreenSize(void); +FOUNDATION_EXPORT CGSize RunnerSoleScreenSize(void); #endif // What the full-screen proof concluded. The decision is a function of these values, so a test diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index a16abd14c0..b32a285818 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -251,28 +251,35 @@ RunnerFullScreenEvidence RunnerFullScreenEvidenceFromForeground( return cached; } -CGSize RunnerSystemApplicationScreenSize(void) { - Class element = RunnerForegroundElementClass(); - BOOL available = element != Nil && [element respondsToSelector:@selector(systemApplication)]; - __block CGRect screen = CGRectNull; - NSString *exception = available ? [RunnerObjCExceptionCatcher catchException:^{ - id systemApp = [element systemApplication]; - if (systemApp == nil || ![systemApp respondsToSelector:@selector(frame)]) return; - screen = [systemApp frame]; - }] : nil; - if (!available || exception != nil || !AgentDeviceForegroundFrameIsMeasurable(screen)) { +CGSize RunnerSoleScreenSize(void) { + __block NSUInteger screenCount = 0; + __block CGRect bounds = CGRectNull; + __block CGFloat scale = 0; + NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ + NSArray *screens = XCUIScreen.screens; + screenCount = screens.count; + if (screenCount != 1) return; + XCUIScreen *screen = screens.firstObject; + NSValue *boundsValue = [screen valueForKey:@"bounds"]; + [boundsValue getValue:&bounds size:sizeof(bounds)]; + NSNumber *scaleValue = [screen valueForKey:@"scale"]; + if ([scaleValue isKindOfClass:NSNumber.class]) scale = scaleValue.doubleValue; + }]; + if (exception != nil || screenCount != 1 || + !AgentDeviceForegroundFrameIsMeasurable(bounds) || !isfinite(scale) || scale <= 0) { NSLog( - @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=system-app available=%@ frame=(%g,%g %gx%g) raised=%@", - available ? @"yes" : @"no", - screen.origin.x, - screen.origin.y, - screen.size.width, - screen.size.height, + @"AGENT_DEVICE_RUNNER_APP_SCREEN_UNRESOLVED stage=sole-screen-size screens=%lu bounds=(%g,%g %gx%g) scale=%g raised=%@", + (unsigned long)screenCount, + bounds.origin.x, + bounds.origin.y, + bounds.size.width, + bounds.size.height, + scale, exception != nil ? @"yes" : @"no" ); return CGSizeZero; } - return screen.size; + return CGSizeMake(bounds.size.width / scale, bounds.size.height / scale); } // One contained probe: it checks each selector before sending it and reads the target pid and both @@ -347,7 +354,7 @@ static RunnerFullScreenEvidence RunnerProbeApplicationFillsScreen(id application } // An iPhone keeps every app full-screen, so its idiom answers the rule and admission never pays // for the fills-screen probe. Off the phone idiom the route opens only on that proof. The display - // size is a separate AXRuntime read (~0.6–0.9 ms) the caller makes after admission, the first time + // size is a separate XCTest screen read the caller makes after admission, the first time // a command needs the box; a command keeps a successful read, and the next gesture retries a failed // one. if (UIDevice.currentDevice.userInterfaceIdiom != UIUserInterfaceIdiomPhone && diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 652aad7c41..9ca703caee 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -35,7 +35,7 @@ private final class RunnerCompatibilityModeScreenStub: NSObject { @objc var nativeScale: CGFloat { 2 } } -/// Answers like an AXRuntime whose system application cannot be read, so the display size is unknown. +/// Physical iOS can expose no usable accessibility frame for the system application. private final class RunnerUnreadableSystemApplicationStub: NSObject { @objc class func systemApplication() -> AnyObject? { nil } } @@ -44,8 +44,7 @@ extension RunnerTests { /// Installs `RunnerUnreadableSystemApplicationStub` on AXRuntime's `AXElement` until the returned /// closure restores it. private func simulateUnreadableSystemApplication() throws -> () -> Void { - // The bridge loads AXRuntime on its first read. - _ = RunnerSystemApplicationScreenSize() + _ = dlopen("/System/Library/PrivateFrameworks/AXRuntime.framework/AXRuntime", RTLD_NOW) let element: AnyClass = try XCTUnwrap(NSClassFromString("AXElement")) let selector = NSSelectorFromString("systemApplication") let target = try XCTUnwrap(class_getClassMethod(element, selector)) @@ -54,28 +53,49 @@ extension RunnerTests { return { method_setImplementation(target, original) } } - /// Turns the device to landscape until the host app's interface follows, so a synthesized turn - /// needs its display box; the returned closure turns it back to portrait. + private func simulateUnreadableDisplay() throws -> () -> Void { + let selector = #selector(NSObject.value(forKey:)) + let inherited = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) + class_addMethod(XCUIScreen.self, selector, method_getImplementation(inherited), method_getTypeEncoding(inherited)) + let target = try XCTUnwrap(class_getInstanceMethod(XCUIScreen.self, selector)) + let original = method_getImplementation(target) + typealias ReadValue = @convention(c) (AnyObject, Selector, NSString) -> AnyObject? + let readValue = unsafeBitCast(original, to: ReadValue.self) + let read: @convention(block) (AnyObject, NSString) -> AnyObject? = { screen, key in + if key == "scale" { return NSNumber(value: 0) } + return readValue(screen, selector, key) + } + let replacement = imp_implementationWithBlock(read) + method_setImplementation(target, replacement) + return { + method_setImplementation(target, original) + imp_removeBlock(replacement) + } + } + + /// Sets the requested orientation and waits for the host app's interface to follow. @MainActor - private func rotateHostAppToLandscape() throws -> () -> Void { - XCUIDevice.shared.orientation = .landscapeLeft + private func rotateHostApp(landscape: Bool) throws -> () -> Void { + XCUIDevice.shared.orientation = landscape ? .landscapeLeft : .portrait let restore = { XCUIDevice.shared.orientation = .portrait } - let landscape = [RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight] + let expected = landscape + ? [RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight] + : [RunnerInterfaceOrientation.portrait] let deadline = Date().addingTimeInterval(5) - while !landscape.contains(Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))), + while !expected.contains(Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))), Date() < deadline { RunLoop.current.run(until: Date().addingTimeInterval(0.1)) } - let turned = landscape.contains(Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))) + let turned = expected.contains(Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))) if !turned { restore() } - _ = try XCTUnwrap(turned ? true : nil, "the host app did not turn to landscape") + _ = try XCTUnwrap(turned ? true : nil, "the host app did not follow the requested orientation") return restore } - /// The state the AX-failure tap tests start from: the host app launched (turned to landscape when + /// The state the unreadable-display tap tests start from: the host app launched (turned to landscape when /// asked), a tap context on its resolved window (measured at `windowFrame` when given) or, when - /// the window query is rejected, an unserializable tree and no context, and AXRuntime's display + /// the window query is rejected, an unserializable tree and no context, and XCTest's display /// read failing with no kept size. The returned cleanup undoes every step. @MainActor private func launchWithUnreadableDisplay( @@ -92,7 +112,7 @@ extension RunnerTests { self.invalidateCachedTarget(reason: "unit_test_cleanup") self.app.terminate() } - if landscape { cleanups.append(try rotateHostAppToLandscape()) } + cleanups.append(try rotateHostApp(landscape: landscape)) var context: SynthesizedCoordinateContext? if !windowQueryRejected { let resolved = try XCTUnwrap(resolveRunnerWindowElement(app: app)) @@ -104,9 +124,9 @@ extension RunnerTests { ) ) } - cleanups.append(try simulateUnreadableSystemApplication()) + cleanups.append(try simulateUnreadableDisplay()) if windowQueryRejected { cleanups.append(try simulateUnserializableAppTree()) } - // An earlier gesture may have kept a successful read; this one must reach AXRuntime. + // An earlier gesture may have kept a successful read; this one must reach XCTest. mainOwned.soleDisplayPortraitSize = nil return (context, cleanup) } catch { @@ -289,6 +309,31 @@ extension RunnerTests { XCTAssertEqual(displayID, 0) } + /// Physical iOS can report no system application's AX frame while its display is measurable. + @MainActor + func testSoleDisplaySizeDoesNotRequireASystemApplicationAccessibilityFrame() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "the rotation basis requires one display") + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + try XCTSkipUnless(soleScreenRouteIsOpen(for: app), "the host app must fill the display") + let frame = app.frame + let restoreUIKit = try simulateCompatibilityModeScreen() + defer { restoreUIKit() } + let restoreAXRuntime = try simulateUnreadableSystemApplication() + defer { restoreAXRuntime() } + mainOwned.soleDisplayPortraitSize = nil + let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) + let measured = try XCTUnwrap(synthesizedRotationFrame( + app: app, fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + )) + XCTAssertEqual(measured.origin, .zero) + XCTAssertEqual(measured.width, max(frame.width, frame.height), accuracy: 0.5) + XCTAssertEqual(measured.height, min(frame.width, frame.height), accuracy: 0.5) + } + /// #3899: a synthesized turn swaps the display's side lengths, never a placed window's smaller box. /// #4091: the display is the one the launched host app fills, not the runner process's own /// `UIScreen`, which the generated runner app (no launch screen) reads in compatibility mode. @@ -310,8 +355,7 @@ extension RunnerTests { height: max(appFrame.width, appFrame.height) ) XCTAssertNotEqual(screen, CGSize(width: 768, height: 1024), "the host app must not fill a 768x1024 screen") - // Two AX readers of one screen; the fills-screen proof ties them within 0.5 pt. Portrait and an - // unknown orientation skip the read; landscape and upside-down read the box, so they are checked. + // Portrait and an unknown orientation skip the read; landscape and upside-down read the box. let expectations = [ (RunnerInterfaceOrientation.portraitUpsideDown, screen), (RunnerInterfaceOrientation.landscapeRight, CGSize(width: screen.height, height: screen.width)), @@ -324,8 +368,8 @@ extension RunnerTests { } } - /// The display size costs one AXRuntime read, so a command keeps its first successful answer and - /// later gestures take it even while AXRuntime would fail. The next command reads again, because + /// A command keeps its first successful display size and later gestures take it even while the + /// display read would fail. The next command reads again, because /// display scaling can change the size, and a failed read is not kept. @MainActor func testSoleDisplaySizeIsReadOncePerCommandAndAFailedReadIsNotKept() throws { @@ -338,7 +382,7 @@ extension RunnerTests { let nextCommand = try runnerCommandFixture(#"{"command":"uptime","commandId":"next-command"}"#) // Nothing between the install and the restore throws, so the restore always runs. - let restoreAXRuntime = try simulateUnreadableSystemApplication() + let restoreDisplay = try simulateUnreadableDisplay() XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) _ = try? executeOnMainPrepared(command: nextCommand, activeApp: app) XCTAssertNotEqual( @@ -346,7 +390,7 @@ extension RunnerTests { measured, "the next command must read the size again rather than reuse the kept one" ) - restoreAXRuntime() + restoreDisplay() XCTAssertEqual(synthesizedRotationFrame(app: app, fallback: placedWindow, interfaceOrientation: landscape), measured) } @@ -379,7 +423,7 @@ extension RunnerTests { defer { cleanup() } try XCTSkipUnless( soleScreenRouteIsOpen(for: app), - "without AXRuntime an iPad cannot admit the sole-screen route; the landscape test covers refusal" + "the sole-screen rule must admit the app; the landscape test covers refusal" ) XCTAssertEqual( Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)), @@ -418,7 +462,7 @@ extension RunnerTests { defer { cleanup() } try XCTSkipUnless( soleScreenRouteIsOpen(for: app), - "without AXRuntime an iPad cannot prove the app fills the screen; the refusal test covers it" + "the app must provably fill the screen; the refusal test covers it" ) let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))