From cb853f8c1f9f8686221754dfb3f1010b878faf7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 14 Sep 2026 19:34:17 +0200 Subject: [PATCH 1/3] fix(runner): bound XCTest's idle wait around alert activation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An alert button is read as hittable before it is tapped, so XCTest's own wait for the app to idle before synthesizing that tap adds nothing the runner has not already checked. Its default outlives the command, so the tap lands after the caller was told the alert timed out (#2546). The helper that bounds that wait was named for the scroll path that owned it; gesture, type and swipe already went through it, and alert activation now does too, so it is named for the wait it bounds. Each caller states which of XCTest's two waits it gives up: the gesture, type and swipe paths skip both, because their next step is a poll of their own, while alert activation drops only the pre-event wait. Its verification reads the alert this tap replaced, and an alert that dismisses in order to present an identical replacement passes through a moment with no alert at all — a first read landing in that moment reported a dismissal nothing had proved. --- .../RunnerTests+Alert.swift | 11 ++++- .../RunnerTests+CommandExecution.swift | 8 ++-- .../RunnerTests+Lifecycle.swift | 40 +++++++++++++++++-- .../RunnerTests.swift | 2 +- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 8b43c19930..9e06410820 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -68,7 +68,16 @@ extension RunnerTests { guard waitUntilAlertButtonHittable(button, deadline: deadline) else { return alertVerificationResponse(.timedOut, action: action, activated: false) } - let outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + // The hittable read above is this activation's readiness gate, so XCTest's pre-synthesis wait + // adds nothing and can cost more than the command has: the tap would land after the deadline + // expired and the alert would be answered by a button the caller was told nothing about + // (#2546). The post-tap settle stays, because the verification below reads the alert this tap + // replaces; an alert that dismisses and presents an identical replacement passes through a + // window with no alert, and a first read landing there reports a dismissal nothing proved. + var outcome = RunnerInteractionOutcome.performed + withBoundedInteractionIdleTimeoutIfSupported(alert.ownerApp, waits: .preEventSkipped) { + outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + } if let response = unsupportedResponse(for: outcome) { return response } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 79d8ec6594..c73b412a16 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -97,7 +97,9 @@ extension RunnerTests { var outcome = RunnerInteractionOutcome.performed let timing = measureGesture { if idleTimeout { - withTemporaryScrollIdleTimeoutIfSupported(app) { outcome = action() } + withBoundedInteractionIdleTimeoutIfSupported(app, waits: .bothSkipped) { + outcome = action() + } } else { outcome = action() } @@ -1975,7 +1977,7 @@ extension RunnerTests { return Response(ok: true, data: DataPayload(message: "remote pressed")) case .type: var response: Response? - withTemporaryScrollIdleTimeoutIfSupported(activeApp) { + withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { response = executeTypeCommand(activeApp: activeApp, command: command) } return response ?? Response(ok: false, error: ErrorPayload(message: "type produced no response")) @@ -1987,7 +1989,7 @@ extension RunnerTests { // keeps raw measureGesture and only routes the success payload through gestureResponse. var executedFrame: DragVisualizationFrame? let timing = measureGesture { - withTemporaryScrollIdleTimeoutIfSupported(activeApp) { + withBoundedInteractionIdleTimeoutIfSupported(activeApp, waits: .bothSkipped) { executedFrame = swipe(app: activeApp, direction: direction) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift index 4ddd2ed792..92eda53057 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift @@ -21,6 +21,24 @@ func runnerCGImage(from image: RunnerImage) -> CGImage? { #endif } +/// Which of XCTest's two waits around one synthesized event a caller gives up. +/// +/// The two waits answer to different callers. The pre-event wait is what #2546 bounds: the runner +/// has already decided the interaction may be synthesized, and XCTest's default wait for the app to +/// idle outlives a bounded command, so the event lands after the caller was told it failed. The +/// post-event wait is the settle margin before the runner reads the app back, and a caller whose +/// verdict is that next read cannot give it up. +enum RunnerInteractionIdleWaits { + /// Neither wait, for a caller whose next step is its own poll rather than a verdict read off this + /// event: a scroll re-checks its own content, a text field was already located, a swipe has + /// nothing to verify. + case bothSkipped + /// Pre-event wait dropped, post-event quiescence kept under the same bound, for a caller whose + /// verdict is the state this event produced. Alert verification reads the alert the tap replaced, + /// and a read taken mid-transition finds no alert and reports a dismissal nothing proved. + case preEventSkipped +} + extension RunnerTests { // MARK: - Recording @@ -209,8 +227,14 @@ extension RunnerTests { return target } - func withTemporaryScrollIdleTimeoutIfSupported( + /// Bounds what XCTest waits around one synthesized event instead of letting it spend a command's + /// whole deadline, keeping whichever settle the caller named in `waits`. Callers gate the + /// interaction themselves first (a scroll needs no extra wait, a text field is located, an alert + /// button is read as hittable), which is what the dropped pre-event wait replaces rather than a + /// check the runner skips (#2546). + func withBoundedInteractionIdleTimeoutIfSupported( _ target: XCUIApplication, + waits: RunnerInteractionIdleWaits, operation: () -> Void ) { let setter = NSSelectorFromString("setWaitForIdleTimeout:") @@ -219,19 +243,20 @@ extension RunnerTests { ? (target.value(forKey: "waitForIdleTimeout") as? NSNumber) : nil if supportsWaitForIdleTimeout { - target.setValue(scrollInteractionIdleTimeoutDefault, forKey: "waitForIdleTimeout") + target.setValue(interactionIdleTimeoutDefault, forKey: "waitForIdleTimeout") } defer { if let previous { target.setValue(previous.doubleValue, forKey: "waitForIdleTimeout") } } - performWithQuiescenceSkippedIfSupported(target, operation: operation) + performWithQuiescenceSkippedIfSupported(target, waits: waits, operation: operation) } // Some apps never report post-gesture quiescence, even after XCTest has synthesized the event. private func performWithQuiescenceSkippedIfSupported( _ target: XCUIApplication, + waits: RunnerInteractionIdleWaits, operation: () -> Void ) { let selector = NSSelectorFromString("_performWithInteractionOptions:block:") @@ -252,12 +277,19 @@ extension RunnerTests { ) let skipPreEventQuiescence = UInt(1) let skipPostEventQuiescence = UInt(2) + let options: UInt + switch waits { + case .bothSkipped: + options = skipPreEventQuiescence | skipPostEventQuiescence + case .preEventSkipped: + options = skipPreEventQuiescence + } withoutActuallyEscaping(operation) { escapableOperation in let block: @convention(block) () -> Void = escapableOperation performWithOptions( target, selector, - skipPreEventQuiescence | skipPostEventQuiescence, + options, block ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 2d81ca78bc..718eb6733a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -66,7 +66,7 @@ final class RunnerTests: XCTestCase { let retryCooldown: TimeInterval = 0.2 let postSnapshotInteractionDelay: TimeInterval = 0.2 let firstInteractionAfterActivateDelay: TimeInterval = 0.25 - let scrollInteractionIdleTimeoutDefault: TimeInterval = 1.0 + let interactionIdleTimeoutDefault: TimeInterval = 1.0 let tvRemoteDoublePressDelayDefault: TimeInterval = 0.0 // Keep a periodic XCTest liveness marker in runner.log without flooding long-lived sessions. let xctestIdleKeepaliveInterval: TimeInterval = 60.0 From cdea5751fffdb3ced9ceec8e75f7f2a4da7fff22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 15 Sep 2026 15:00:01 +0200 Subject: [PATCH 2/3] test(ios-runner): cover alert activation against an app that never idles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverting the alert path to a plain `activateElement` kept every test green, so the #2546 failure had no guard: the deadline tests returned before activation, and the ones that reached it used a 10 s timeout on a fixture that had already settled. The fixture now animates without end until an alert button is answered, which is the state that made XCTest hold the event past the caller's deadline in the field. A test that reaches activation there and is given 6 s reports what actually happens: the response comes back in well under a second of activation, and if it ever reports `ALERT_DEADLINE_EXCEEDED` again the fixture says whether a button was activated behind that answer. Reverting the seam costs the run 18 s and produces exactly the original defect — `ALERT_DEADLINE_EXCEEDED` alongside `First actions: 1`. Only a `UIView` animation counts here: a repeating main-thread timer and a `CABasicAnimation` both left the app looking idle and the reverted path still passed. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 39 +++++++++++++++++++ .../RunnerTests+AlertObservationTests.swift | 32 +++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 4e91412dd9..7ed0af0bb4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -62,11 +62,46 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, assign) NSUInteger firstAlertActions; @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, assign) BOOL alertFixtureStarted; +@property(nonatomic, strong) NSTimer *alertActivationBusyBackstop; @end @implementation AgentDeviceRunnerViewController #if TARGET_OS_IOS +// An animation that never ends is what "busy" looks like to XCTest while it decides whether the app +// may receive an event: the app keeps reporting work in flight, which is the state that cost an alert +// command its whole deadline in #2546. It stops the moment an alert button is answered, since that +// answer is the event the runner is trying to land, and the backstop stops it even when no answer +// arrives so a regressed run finishes rather than waiting out XCTest's own timeout. A layer +// animation on its own is not enough; only a UIView animation counts as in-flight work here. +static NSTimeInterval const kAgentDeviceAlertActivationBusyWindow = 20.0; + +- (void)startAlertActivationBusy { + if (self.alertActivationBusyBackstop != nil) { + return; + } + self.alertActivationBusyBackstop = [NSTimer scheduledTimerWithTimeInterval:kAgentDeviceAlertActivationBusyWindow + target:self + selector:@selector(stopAlertActivationBusy) + userInfo:nil + repeats:NO]; + [UIView animateWithDuration:0.4 + delay:0 + options:(UIViewAnimationOptionRepeat | UIViewAnimationOptionAutoreverse) + animations:^{ + self.alertActionStatus.transform = CGAffineTransformMakeTranslation(0, 8); + } + completion:nil]; +} + +- (void)stopAlertActivationBusy { + [self.alertActionStatus.layer removeAllAnimations]; + self.alertActionStatus.transform = CGAffineTransformIdentity; + [self.alertActivationBusyBackstop invalidate]; + self.alertActivationBusyBackstop = nil; +} + + - (void)updateAlertActionStatus { self.alertActionStatus.text = [NSString stringWithFormat:@"First actions: %lu; replacement actions: %lu", (unsigned long)self.firstAlertActions, @@ -88,6 +123,7 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { ? UIAlertActionStyleCancel : UIAlertActionStyleDefault; [alert addAction:[UIAlertAction actionWithTitle:buttonTitle style:style handler:^(UIAlertAction *action) { (void)action; + [self stopAlertActivationBusy]; if (replacement) { self.replacementAlertActions += 1; } else { @@ -110,6 +146,9 @@ - (void)viewDidAppear:(BOOL)animated { [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) { self.alertFixtureStarted = YES; [self presentAlertFixtureReplacement:NO]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { + [self startAlertActivationBusy]; + } } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index 3d234c21f6..62fac5a51c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -57,6 +57,38 @@ extension RunnerTests { XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 0; replacement actions: 0") } + func testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline() throws { + app.launchArguments = [ + "--agent-device-alert-replacement-regression", + "--agent-device-alert-activation-busy" + ] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + let alert = try XCTUnwrap(resolveAlert(app: app, deadline: Date().addingTimeInterval(10))) + + let deadline = Date().addingTimeInterval(6) + let startedAt = Date() + let response = handleAlert(alert, action: "accept", deadline: deadline) + let elapsed = Date().timeIntervalSince(startedAt) + + // The fixture keeps its main thread busy until a button is answered, so this only comes back + // early because activation refused to wait for an app that has no intention of settling (#2546). + XCTAssertLessThan(elapsed, 9, "activation waited \(elapsed)s for a busy app to idle") + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") + if !response.ok, response.error?.code == "ALERT_DEADLINE_EXCEEDED" { + XCTAssertEqual( + app.staticTexts["agent-device-alert-actions"].label, + "First actions: 0; replacement actions: 0", + "a caller told about an expired deadline must not have a button activated behind it" + ) + } + } + private func assertReplacementAlertUntouched(action: String, arguments: [String], confirmed: Bool) throws { app.launchArguments = ["--agent-device-alert-replacement-regression"] + arguments app.launch() From 5dae7231460ad56ee324552bc6fd385565e2c16e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 15 Sep 2026 15:47:33 +0200 Subject: [PATCH 3/3] test(ios-runner): branch the busy-app assertion on what the command answered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The label check ran twice over the same response: it always expected `First actions: 1`, and the block after it expected `First actions: 0` only when the answer was a deadline failure, so the second could never pass and a correct refusal failed anyway. Now the assertion follows the answer — an accepted alert must show the button press, a refused one must not — and the comment said "main thread busy" where the fixture runs a repeating animation with the main thread free. The test also joins the targeted list in `.github/workflows/ios.yml`, which is where the alert regressions it guards are selected. --- .github/workflows/ios.yml | 1 + .../RunnerTests+AlertObservationTests.swift | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index e13f09e372..3c9da93ab5 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -187,6 +187,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDeadlineBeforeActivationLeavesTheOriginalUntouched \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertHittableProbeCompletingAfterDeadlineLeavesTheOriginalUntouched \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertActivationIgnoresAnAppThatNeverSettlesBeforeTheDeadline \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSystemModalProbeSliceSharesAndClampsToPlanDeadline \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDispatchRecoverySkipsBookkeepingWhileXCTestChannelOccupied \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrain \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift index 62fac5a51c..bf182ba5c0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertObservationTests.swift @@ -75,14 +75,17 @@ extension RunnerTests { let response = handleAlert(alert, action: "accept", deadline: deadline) let elapsed = Date().timeIntervalSince(startedAt) - // The fixture keeps its main thread busy until a button is answered, so this only comes back - // early because activation refused to wait for an app that has no intention of settling (#2546). + // The fixture keeps an animation in flight, which is what XCTest waits out before it synthesises + // an event, so this comes back early only because activation refused to wait for an app that has + // no intention of settling (#2546). XCTAssertLessThan(elapsed, 9, "activation waited \(elapsed)s for a busy app to idle") XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(app.staticTexts["agent-device-alert-actions"].label, "First actions: 1; replacement actions: 0") - if !response.ok, response.error?.code == "ALERT_DEADLINE_EXCEEDED" { + let recordedActions = app.staticTexts["agent-device-alert-actions"].label + if response.ok { + XCTAssertEqual(recordedActions, "First actions: 1; replacement actions: 0") + } else { XCTAssertEqual( - app.staticTexts["agent-device-alert-actions"].label, + recordedActions, "First actions: 0; replacement actions: 0", "a caller told about an expired deadline must not have a button activated behind it" )