From 232f6fc64d924aae1052ba915a6e882e9385ef81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 09:50:27 +0100 Subject: [PATCH 01/10] fix(ios): recognize and answer the localized Open in launch confirmation On a non-English iOS simulator, the `Open in ""?` confirmation that `open --launch-url` raises stayed unrecognized (`alert-unrecognized`), and `alert accept` refused its button (`Ouvrir`) with ALERT_BUTTON_NOT_FOUND. The runner now reads SpringBoard's own SchemeApproval strings (CoreServices.framework) with the strings it already reads for permission prompts. Their open and cancel labels join the alert vocabulary, so `alert accept` and `dismiss` answer the prompt in each preferred language. The runner also marks an alert read as the launch confirmation when its title and both buttons are the OS's wording of one language, and `open` accepts a marked prompt instead of matching English text. Unreadable strings recognize nothing, as before. Fixes #3922 Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerAlertLabelVocabulary.swift | 54 ++++---------- .../RunnerLaunchConfirmationPrompt.swift | 71 +++++++++++++++++++ .../RunnerSystemPromptStrings.swift | 71 +++++++++++++++++++ .../RunnerTests+Alert.swift | 7 +- .../RunnerTests+Models.swift | 3 + .../RunnerTests.swift | 5 +- ...rTests+LaunchConfirmationPromptTests.swift | 61 ++++++++++++++++ .../command-surface/src/management/app.ts | 2 +- .../src/launch-confirmation-runner.test.ts | 6 +- .../src/launch-confirmation.fixtures.ts | 6 +- .../src/launch-confirmation.test.ts | 19 ++--- .../platform-apple/src/launch-confirmation.ts | 25 ++----- website/docs/docs/commands.md | 5 +- 13 files changed, 258 insertions(+), 77 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift index 664ab52c08..d9f662160a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift @@ -37,55 +37,25 @@ extension RunnerAlertLabelVocabulary { /// preferred language the app localizes into, so every preferred language takes part. static let system = fromSystemStrings(languages: Locale.preferredLanguages) - /// The string tables that word the system permission prompts: notifications, privacy access - /// (including App Tracking Transparency), and location. - private static let promptStringTables = [ - ("/System/Library/PrivateFrameworks/UserNotificationsServer.framework", "Localizable"), - ("/System/Library/PrivateFrameworks/TCC.framework", "Localizable"), - ("/System/Library/Frameworks/CoreLocation.framework", "locationd") - ] - /// Pairs each English prompt string with the same key's value in each of `languages`. Every /// table must read, or none counts: a missing table could hide the translation that makes a /// label in another table claim both actions. static func fromSystemStrings(languages: [String]) -> RunnerAlertLabelVocabulary { - // The simulated OS, taken from where this process's Foundation loaded; the simulator process - // would otherwise read the host's /System. - let foundationPath = "/System/Library/Frameworks/Foundation.framework" - guard let foundation = Bundle(identifier: "com.apple.Foundation")?.bundlePath, foundation.hasSuffix(foundationPath) - else { return unavailable(source: foundationPath, languages: languages) } - let root = String(foundation.dropLast(foundationPath.count)) - var translations: [(english: String, localized: String)] = [] - for (path, table) in promptStringTables { - guard let bundle = Bundle(path: root + path), - let english = strings(in: bundle, table: table, localization: "en") - else { return unavailable(source: path, languages: languages) } - let localizations = Set(languages.compactMap { - Bundle.preferredLocalizations(from: bundle.localizations, forPreferences: [$0]).first - }).subtracting(["en"]) - for localization in localizations { - guard let localized = strings(in: bundle, table: table, localization: localization) - else { return unavailable(source: "\(path) \(localization)", languages: languages) } - for (key, value) in english { - if let translation = localized[key] { translations.append((english: value, localized: translation)) } + switch RunnerSystemPromptStrings.read(RunnerSystemPromptStrings.promptTables, languages: languages) { + case .failure(let failure): + RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) + return .english + case .success(let tables): + var translations: [(english: String, localized: String)] = [] + for table in tables { + for strings in table.localized.values { + for (key, value) in table.english { + if let translation = strings[key] { translations.append((english: value, localized: translation)) } + } } } + return RunnerAlertLabelVocabulary(translations: translations) } - return RunnerAlertLabelVocabulary(translations: translations) - } - - private static func unavailable(source: String, languages: [String]) -> RunnerAlertLabelVocabulary { - NSLog( - "AGENT_DEVICE_RUNNER_ALERT_VOCABULARY_UNAVAILABLE source=%@ languages=%@", - source, languages.joined(separator: ",") - ) - return .english - } - - private static func strings(in bundle: Bundle, table: String, localization: String) -> [String: String]? { - guard let url = bundle.url(forResource: table, withExtension: "strings", subdirectory: nil, localization: localization) - else { return nil } - return NSDictionary(contentsOf: url) as? [String: String] } } #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift new file mode 100644 index 0000000000..dc3601e228 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift @@ -0,0 +1,71 @@ +import Foundation + +/// SpringBoard's `Open in “”?` launch confirmation, recognized by the OS's own wording of its +/// title and its two buttons. A prompt counts only when both buttons are exactly the open and +/// cancel labels of one language, so an app alert that happens to start the same way is not it. +struct RunnerLaunchConfirmationPrompt { + /// One language's wording, as SpringBoard's `SchemeApproval` strings give it. + struct Wording { + let titleFormat: String + let open: String + let cancel: String + } + + static let unavailable = RunnerLaunchConfirmationPrompt(wordings: []) + + private let wordings: [(title: NSRegularExpression, open: String, cancel: String)] + + init(wordings: [Wording]) { + self.wordings = wordings.compactMap { wording in + let pattern = NSRegularExpression.escapedPattern(for: wording.titleFormat) + .replacingOccurrences(of: "%@", with: ".+") + guard let title = try? NSRegularExpression(pattern: "^\(pattern)$") else { return nil } + return (title, Self.normalized(wording.open), Self.normalized(wording.cancel)) + } + } + + func recognizes(title: String, buttons: [String]) -> Bool { + let title = title.trimmingCharacters(in: .whitespacesAndNewlines) + let titleRange = NSRange(title.startIndex..., in: title) + let labels = buttons.map(Self.normalized) + guard labels.count == 2 else { return false } + return wordings.contains { wording in + wording.title.firstMatch(in: title, range: titleRange) != nil + && Set(labels) == [wording.open, wording.cancel] + } + } + + private static func normalized(_ label: String) -> String { + label.trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +#if os(iOS) && targetEnvironment(simulator) +extension RunnerLaunchConfirmationPrompt { + /// English and every preferred language, read once from the OS strings. Unreadable strings + /// recognize nothing, so the prompt stays an unrecognized alert. + static let system = fromSystemStrings(languages: Locale.preferredLanguages) + + static func fromSystemStrings(languages: [String]) -> RunnerLaunchConfirmationPrompt { + switch RunnerSystemPromptStrings.read([RunnerSystemPromptStrings.launchConfirmationTable], languages: languages) { + case .failure(let failure): + RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) + return .unavailable + case .success(let tables): + guard let table = tables.first else { return .unavailable } + let wordings = ([table.english] + table.localized.values).compactMap { strings -> Wording? in + guard let title = strings["SCHEME_APPROVAL_PROMPT_TITLE_NO_SOURCE"], + let open = strings["SCHEME_APPROVAL_OPEN_BUTTON"], + let cancel = strings["SCHEME_APPROVAL_DO_NOT_OPEN_BUTTON"] + else { return nil } + return Wording(titleFormat: title, open: open, cancel: cancel) + } + return RunnerLaunchConfirmationPrompt(wordings: wordings) + } + } +} +#else +extension RunnerLaunchConfirmationPrompt { + static let system = unavailable +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift new file mode 100644 index 0000000000..ae29e48ee3 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift @@ -0,0 +1,71 @@ +import Foundation + +#if os(iOS) && targetEnvironment(simulator) +/// One OS string table read in English and in each preferred language that is not English. +struct RunnerSystemPromptStrings { + let english: [String: String] + /// The table's strings by localization, English excluded. + let localized: [String: [String: String]] + + /// The string tables that word the system permission prompts: notifications, privacy access + /// (including App Tracking Transparency), and location, plus SpringBoard's launch confirmation. + static let promptTables = [ + (path: "/System/Library/PrivateFrameworks/UserNotificationsServer.framework", table: "Localizable"), + (path: "/System/Library/PrivateFrameworks/TCC.framework", table: "Localizable"), + (path: "/System/Library/Frameworks/CoreLocation.framework", table: "locationd"), + launchConfirmationTable + ] + + static let launchConfirmationTable = ( + path: "/System/Library/Frameworks/CoreServices.framework", + table: "SchemeApproval" + ) + + /// Reads `tables` from the simulated OS, or names the source that did not read. The simulated OS + /// is taken from where this process's Foundation loaded; the simulator process would otherwise + /// read the host's /System. A table reads in every preferred language or not at all, because a + /// missing translation could hide the one that makes a label claim both actions. + static func read( + _ tables: [(path: String, table: String)], + languages: [String] + ) -> Result<[RunnerSystemPromptStrings], UnavailableSource> { + let foundationPath = "/System/Library/Frameworks/Foundation.framework" + guard let foundation = Bundle(identifier: "com.apple.Foundation")?.bundlePath, foundation.hasSuffix(foundationPath) + else { return .failure(UnavailableSource(source: foundationPath)) } + let root = String(foundation.dropLast(foundationPath.count)) + var read: [RunnerSystemPromptStrings] = [] + for (path, table) in tables { + guard let bundle = Bundle(path: root + path), let english = strings(in: bundle, table: table, localization: "en") + else { return .failure(UnavailableSource(source: path)) } + let localizations = Set(languages.compactMap { + Bundle.preferredLocalizations(from: bundle.localizations, forPreferences: [$0]).first + }).subtracting(["en"]) + var localized: [String: [String: String]] = [:] + for localization in localizations { + guard let strings = strings(in: bundle, table: table, localization: localization) + else { return .failure(UnavailableSource(source: "\(path) \(localization)")) } + localized[localization] = strings + } + read.append(RunnerSystemPromptStrings(english: english, localized: localized)) + } + return .success(read) + } + + struct UnavailableSource: Error { + let source: String + } + + static func logUnavailable(_ failure: UnavailableSource, languages: [String]) { + NSLog( + "AGENT_DEVICE_RUNNER_ALERT_VOCABULARY_UNAVAILABLE source=%@ languages=%@", + failure.source, languages.joined(separator: ",") + ) + } + + private static func strings(in bundle: Bundle, table: String, localization: String) -> [String: String]? { + guard let url = bundle.url(forResource: table, withExtension: "strings", subdirectory: nil, localization: localization) + else { return nil } + return NSDictionary(contentsOf: url) as? [String: String] + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 4409ff03f7..69996a23fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -137,11 +137,14 @@ extension RunnerTests { } } + let title = preferredAlertTitle(alert.root, buttons: alert.buttons) + let items = alert.buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) } return Response( ok: true, data: DataPayload( - message: preferredAlertTitle(alert.root, buttons: alert.buttons), - items: alert.buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) } + message: title, + items: items, + launchConfirmation: RunnerLaunchConfirmationPrompt.system.recognizes(title: title, buttons: items) ? true : nil ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 9d73aabea5..8d08852bc8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -445,6 +445,9 @@ struct DataPayload: Codable { var text: String? var found: Bool? var items: [String]? + /// Set on an alert read that is SpringBoard's `Open in “”?` launch confirmation, in the + /// device's language; absent otherwise. + var launchConfirmation: Bool? var nodes: [PresentedNode]? var truncated: Bool? var qualityPayload: SnapshotQualityPayload? = nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index d152fa5cda..4b1ff6f579 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -317,7 +317,10 @@ final class RunnerTests: XCTestCase { NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", configuration.port.rawValue) listener = try Self.makeRunnerListener(port: configuration.port) // Reads the OS prompt strings off the alert path, so the first accept/dismiss deadline does not pay for it. - DispatchQueue.global(qos: .utility).async { _ = RunnerAlertLabelVocabulary.system } + DispatchQueue.global(qos: .utility).async { + _ = RunnerAlertLabelVocabulary.system + _ = RunnerLaunchConfirmationPrompt.system + } listener?.stateUpdateHandler = { [weak self] state in switch state { case .ready: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift new file mode 100644 index 0000000000..bd447ed30f --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -0,0 +1,61 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + private static let launchConfirmationWordings = [ + RunnerLaunchConfirmationPrompt.Wording(titleFormat: "Open in “%@”?", open: "Open", cancel: "Cancel"), + RunnerLaunchConfirmationPrompt.Wording(titleFormat: "Ouvrir dans «\u{00A0}%@\u{00A0}»\u{00A0}?", open: "Ouvrir", cancel: "Annuler"), + RunnerLaunchConfirmationPrompt.Wording(titleFormat: "“%@”で開きますか?", open: "開く", cancel: "キャンセル") + ] + + /// The confirmation is recognized in each language its wordings carry, and only as a whole: the + /// title, and both buttons of that one language. + func testLaunchConfirmationIsRecognizedInEveryWordingLanguage() { + let prompt = RunnerLaunchConfirmationPrompt(wordings: Self.launchConfirmationWordings) + XCTAssertTrue(prompt.recognizes(title: "Open in “Test App”?", buttons: ["Cancel", "Open"])) + XCTAssertTrue(prompt.recognizes(title: "Ouvrir dans «\u{00A0}Modèle RN\u{00A0}»\u{00A0}?", buttons: ["Annuler", "Ouvrir"])) + XCTAssertTrue(prompt.recognizes(title: "“テスト”で開きますか?", buttons: ["キャンセル", "開く"])) + } + + /// The closest negatives: an English prompt of another wording, a prompt with a foreign button, + /// buttons from two languages, and a title that only starts like the confirmation. + func testLaunchConfirmationRefusesAlertsThatOnlyResembleIt() { + let prompt = RunnerLaunchConfirmationPrompt(wordings: Self.launchConfirmationWordings) + XCTAssertFalse(prompt.recognizes(title: "“Test App” Would Like to Send You Notifications", buttons: ["Don’t Allow", "Allow"])) + XCTAssertFalse(prompt.recognizes(title: "Ouvrir dans «\u{00A0}Modèle RN\u{00A0}»\u{00A0}?", buttons: ["Annuler", "Autoriser"])) + XCTAssertFalse(prompt.recognizes(title: "Ouvrir dans «\u{00A0}Modèle RN\u{00A0}»\u{00A0}?", buttons: ["Cancel", "Ouvrir"])) + XCTAssertFalse(prompt.recognizes(title: "Ouvrir dans «\u{00A0}Modèle RN\u{00A0}»\u{00A0}?", buttons: ["Annuler", "Ouvrir", "Plus tard"])) + XCTAssertFalse(prompt.recognizes(title: "Open in “Test App”? Really", buttons: ["Cancel", "Open"])) + XCTAssertFalse(RunnerLaunchConfirmationPrompt.unavailable.recognizes(title: "Open in “Test App”?", buttons: ["Cancel", "Open"])) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) && targetEnvironment(simulator) +extension RunnerTests { + /// The OS's own SchemeApproval strings supply the wording: a French or Japanese device reads the + /// confirmation, and `accept` picks its open button where the English labels cannot. + func testSystemStringsRecognizeAndAnswerTheLocalizedLaunchConfirmation() { + let languages = ["fr-FR", "ja-JP"] + let prompt = RunnerLaunchConfirmationPrompt.fromSystemStrings(languages: languages) + let vocabulary = RunnerAlertLabelVocabulary.fromSystemStrings(languages: languages) + + XCTAssertTrue(prompt.recognizes(title: "Ouvrir dans «\u{00A0}Modèle RN\u{00A0}»\u{00A0}?", buttons: ["Annuler", "Ouvrir"])) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "accept", vocabulary: vocabulary), 1) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "dismiss", vocabulary: vocabulary), 0) + + XCTAssertTrue(prompt.recognizes(title: "“テスト”で開きますか?", buttons: ["キャンセル", "開く"])) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["キャンセル", "開く"], action: "accept", vocabulary: vocabulary), 1) + + XCTAssertFalse(prompt.recognizes(title: "Öffnen in „Test“?", buttons: ["Abbrechen", "Öffnen"]), "German is not preferred") + } + + /// English stays recognized on an English device, and the English labels still answer it. + func testSystemStringsStillRecognizeTheEnglishLaunchConfirmation() { + let prompt = RunnerLaunchConfirmationPrompt.fromSystemStrings(languages: ["en-US"]) + let vocabulary = RunnerAlertLabelVocabulary.fromSystemStrings(languages: ["en-US"]) + XCTAssertTrue(prompt.recognizes(title: "Open in “Test App”?", buttons: ["Cancel", "Open"])) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "Open"], action: "accept", vocabulary: vocabulary), 1) + } +} +#endif diff --git a/packages/command-surface/src/management/app.ts b/packages/command-surface/src/management/app.ts index efbb1acb08..a73cc90a33 100644 --- a/packages/command-surface/src/management/app.ts +++ b/packages/command-surface/src/management/app.ts @@ -209,7 +209,7 @@ export const openCommandFacet = defineCommandFacet({ text: { summary: 'Open an app, deep link or URL, save replays', cliDetail: - 'Use --platform to bind URL/deep-link opens to the target platform. For local iOS simulator initial stdout/stderr, put --launch-console on this open command, for example agent-device open "Agent Device Tester" --platform ios --launch-console artifacts/launch-console.log. Expo Go/dev-client shells accept host + URL, for example agent-device open "Expo Go" exp://127.0.0.1:8081 --platform ios. macOS also supports --surface app|frontmost-app|desktop|menubar. --metro-host/--metro-port/--bundle-url choose the app\'s Metro server and save that address for later metro reload calls in the same session. A fresh open without Metro hints clears an older same-name binding; close also clears it. --launch-url supplies the app\'s launch URL; it does not select the reload server. On an iOS simulator without --launch-url, open connects a detected Expo development client through its expo-development-client URL. The app must declare a single exp+ scheme that no other installed app owns. If detection is inconclusive, the app opens normally. Bare React Native apps receive their server settings before launch. On an iOS simulator, when --launch-url raises an "Open in ?" prompt that keeps open from seeing the app, open accepts it if the URL scheme belongs to the session app (data.launchConfirmation: "accepted"); if the scheme belongs to another installed app, the prompt stays on screen and open fails with details.reason launch_confirmation_foreign_app; if the scheme owner cannot be resolved because no single installed app owns it or the lookup failed, the prompt stays on screen and open returns with data.launchConfirmation: "unanswered", launchConfirmationReason url-owner-unresolved, and a warning; a prompt that is seen but not recognized is reported the same way with launchConfirmationReason alert-unrecognized and its own warning; for url-owner-unresolved, answer the prompt with alert accept; for alert-unrecognized, read the alert with alert get first, because it can be the app\'s own alert (such as a permission prompt) rather than a translated launch confirmation, and answer with alert accept only when it is the launch confirmation; when agent-device cannot read whether a prompt is up at all (the alert read, the accept, or the runner itself failed, including after a re-hand of the URL), open stays green, reports no launchConfirmation plus data.launchConfirmationProbeFailure naming the failed step (alert-read, alert-accept, or runner), and appends a warning that says to check with alert get. The prompt is read whenever open cannot see the app afterwards, including on a slow host where the app\'s target discovery fails on its own deadline. When an accept leaves the app provably not running, open hands the same URL over once more and reads again; still not running, it fails with details.reason launch_confirmation_unanswered. When the runner reports its main-thread timeout for the accept and the prompt is still up, open presses Open again, but only when the time the open request has left covers a whole press and the read that checks it. A recognized prompt still visible after a failed accept, or once that time is gone, also fails with this reason, even if the app is running behind it; answer it with alert accept or alert dismiss. A simulator openurl that CoreSimulator never answers fails within 20s with details.reason ios-simulator-openurl-timeout. Only the English prompt title with Cancel and Open buttons is recognized, so on a Simulator set to another language the prompt stays on screen. A prompt that appears after open has already seen the app is not answered; use alert accept. A device another session is holding refuses at once; add --wait to block up to that budget for it and only then fail with DEVICE_IN_USE naming the owning session.', + 'Use --platform to bind URL/deep-link opens to the target platform. For local iOS simulator initial stdout/stderr, put --launch-console on this open command, for example agent-device open "Agent Device Tester" --platform ios --launch-console artifacts/launch-console.log. Expo Go/dev-client shells accept host + URL, for example agent-device open "Expo Go" exp://127.0.0.1:8081 --platform ios. macOS also supports --surface app|frontmost-app|desktop|menubar. --metro-host/--metro-port/--bundle-url choose the app\'s Metro server and save that address for later metro reload calls in the same session. A fresh open without Metro hints clears an older same-name binding; close also clears it. --launch-url supplies the app\'s launch URL; it does not select the reload server. On an iOS simulator without --launch-url, open connects a detected Expo development client through its expo-development-client URL. The app must declare a single exp+ scheme that no other installed app owns. If detection is inconclusive, the app opens normally. Bare React Native apps receive their server settings before launch. On an iOS simulator, when --launch-url raises an "Open in ?" prompt that keeps open from seeing the app, open accepts it if the URL scheme belongs to the session app (data.launchConfirmation: "accepted"); if the scheme belongs to another installed app, the prompt stays on screen and open fails with details.reason launch_confirmation_foreign_app; if the scheme owner cannot be resolved because no single installed app owns it or the lookup failed, the prompt stays on screen and open returns with data.launchConfirmation: "unanswered", launchConfirmationReason url-owner-unresolved, and a warning; a prompt that is seen but not recognized is reported the same way with launchConfirmationReason alert-unrecognized and its own warning; for url-owner-unresolved, answer the prompt with alert accept; for alert-unrecognized, read the alert with alert get first, because it can be the app\'s own alert (such as a permission prompt) rather than the launch confirmation, and answer with alert accept only when it is the launch confirmation; when agent-device cannot read whether a prompt is up at all (the alert read, the accept, or the runner itself failed, including after a re-hand of the URL), open stays green, reports no launchConfirmation plus data.launchConfirmationProbeFailure naming the failed step (alert-read, alert-accept, or runner), and appends a warning that says to check with alert get. The prompt is read whenever open cannot see the app afterwards, including on a slow host where the app\'s target discovery fails on its own deadline. When an accept leaves the app provably not running, open hands the same URL over once more and reads again; still not running, it fails with details.reason launch_confirmation_unanswered. When the runner reports its main-thread timeout for the accept and the prompt is still up, open presses Open again, but only when the time the open request has left covers a whole press and the read that checks it. A recognized prompt still visible after a failed accept, or once that time is gone, also fails with this reason, even if the app is running behind it; answer it with alert accept or alert dismiss. A simulator openurl that CoreSimulator never answers fails within 20s with details.reason ios-simulator-openurl-timeout. The prompt is recognized by the OS\'s own wording of its title and buttons in the Simulator\'s preferred languages; when the runner cannot read that wording, the prompt stays on screen as alert-unrecognized. A prompt that appears after open has already seen the app is not answered; use alert accept. A device another session is holding refuses at once; add --wait to block up to that budget for it and only then fail with DEVICE_IN_USE naming the owning session.', mcpDetail: "metroHost, metroPort, and bundleUrl choose the app's Metro server and save its address for later reloads. launchUrl selects the app's launch URL and does not select the reload server. A fresh open without Metro hints clears an older same-name binding; close also clears it.", }, diff --git a/packages/platform-apple/src/launch-confirmation-runner.test.ts b/packages/platform-apple/src/launch-confirmation-runner.test.ts index 7dc0547e58..d8af77d324 100644 --- a/packages/platform-apple/src/launch-confirmation-runner.test.ts +++ b/packages/platform-apple/src/launch-confirmation-runner.test.ts @@ -69,7 +69,11 @@ const { answerSimulatorLaunchConfirmation } = await import('./launch-confirmatio const NOW_MS = 1_000_000; /** What the daemon gives a default `open`: its 90 s client envelope less the 30 s result margin. */ const OPEN_DEADLINE_MS = 60_000; -const CONFIRMATION = { message: 'Open in “Example App”?', items: ['Cancel', 'Open'] }; +const CONFIRMATION = { + message: 'Open in “Example App”?', + items: ['Cancel', 'Open'], + launchConfirmation: true, +}; type ScriptedAnswer = | Readonly<{ kind: 'ok'; data: Record }> diff --git a/packages/platform-apple/src/launch-confirmation.fixtures.ts b/packages/platform-apple/src/launch-confirmation.fixtures.ts index 92062364e9..1c569ebe9f 100644 --- a/packages/platform-apple/src/launch-confirmation.fixtures.ts +++ b/packages/platform-apple/src/launch-confirmation.fixtures.ts @@ -2,7 +2,11 @@ import { ALERT_NOT_FOUND_RUNNER_CODE } from '@agent-device/contracts/alert-contr import { AppError } from '@agent-device/kernel/errors'; /** The prompt a session app's own launch URL raises. */ -export const CONFIRMATION = { message: 'Open in “Example App”?', items: ['Cancel', 'Open'] }; +export const CONFIRMATION = { + message: 'Open in “Example App”?', + items: ['Cancel', 'Open'], + launchConfirmation: true, +}; /** The runner's typed absence: `alert get` looked once and found no alert. */ export function alertNotFound(): AppError { diff --git a/packages/platform-apple/src/launch-confirmation.test.ts b/packages/platform-apple/src/launch-confirmation.test.ts index 6eaeccce92..e024fd2587 100644 --- a/packages/platform-apple/src/launch-confirmation.test.ts +++ b/packages/platform-apple/src/launch-confirmation.test.ts @@ -73,15 +73,16 @@ function port( } test.each([ - ['curly', 'Open in “Example App”?'], - ['straight', 'Open in "Example App"?'], - ['localized-name', 'Open in “Beispiel-App”?'], + ['English', 'Open in “Example App”?', ['Cancel', 'Open']], + ['French', 'Ouvrir dans «\u00A0Modèle RN\u00A0»\u00A0?', ['Annuler', 'Ouvrir']], + ['Japanese', '“テスト”で開きますか?', ['キャンセル', '開く']], ])( - 'accepts a confirmation (%s title) when the URL owner is the session app', - async (_case, message) => { + 'accepts a %s confirmation the runner marked when the URL owner is the session app', + async (_language, message, items) => { const { port: device, acceptAlert } = port(async () => ({ message, - items: ['Cancel', 'Open'], + items, + launchConfirmation: true, })); await expect(answerLaunchConfirmation(device)).resolves.toEqual({ outcome: 'accepted' }); @@ -143,7 +144,7 @@ test('an alert that is not a launch confirmation is left for the caller and repo }); }); -test('a launch confirmation in another language is left on screen and reported', async () => { +test('a launch-like alert the runner did not mark is left on screen and reported', async () => { const { port: device, acceptAlert } = port(async () => ({ message: 'In „Beispiel-App“ öffnen?', items: ['Abbrechen', 'Öffnen'], @@ -580,9 +581,9 @@ test('a confirmed persistent prompt reports its observed outcome without an acce }); }); -test('a matching launch title with other buttons is left unanswered on the first read', async () => { +test('a launch title the runner did not mark is left unanswered on the first read', async () => { const { port: device, acceptAlert } = port(async () => ({ - ...CONFIRMATION, + message: CONFIRMATION.message, items: ['Cancel', 'Continue'], })); await expect(answerLaunchConfirmation(device)).resolves.toMatchObject({ diff --git a/packages/platform-apple/src/launch-confirmation.ts b/packages/platform-apple/src/launch-confirmation.ts index c5d7fe3ad7..3286c2a099 100644 --- a/packages/platform-apple/src/launch-confirmation.ts +++ b/packages/platform-apple/src/launch-confirmation.ts @@ -35,14 +35,6 @@ export type LaunchConfirmationAttempt = /** A step of the answer that failed, keyed for diagnostics and tests. */ export type LaunchConfirmationStep = 'alert-read' | 'alert-accept' | 'url-owner' | 'runner'; -/** - * SpringBoard's English title for a URL it holds until the user confirms the app that will open - * it. The runner reports an alert only as its localized title and button labels, and a name match - * is unsafe (a permission alert names the app too), so a Simulator in another language is not - * recognized; the unrecognized alert is reported instead. - */ -const LAUNCH_CONFIRMATION_TITLE = /^Open in [“"].+[”"]\?$/u; - /** * One shared deadline for the URL-owner lookup (the app listing and every Info.plist read). The * alert read and accept are each bounded by the runner's `DEFAULT_ALERT_TIMEOUT_MS`; the lookup gets @@ -190,17 +182,14 @@ function accepted(): LaunchConfirmationAttempt { return { outcome: 'accepted' }; } +/** + * The runner marks SpringBoard's `Open in “”?` prompt, for a URL it holds until the user + * confirms the app that will open it, by the OS's own wording of its title and buttons in the + * device's language. A title or button match here is unsafe: a permission alert names the app too. + * An alert the runner did not mark is reported as unrecognized. + */ function isLaunchConfirmation(alert: Record): boolean { - const title = alert['message']; - const buttons = alert['items']; - return ( - typeof title === 'string' && - LAUNCH_CONFIRMATION_TITLE.test(title) && - Array.isArray(buttons) && - buttons.length === 2 && - buttons.includes('Cancel') && - buttons.includes('Open') - ); + return alert['launchConfirmation'] === true; } /** A failed step leaves the open unanswered; the failure is reported, not thrown. */ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 12fbc94b45..6d60ade9c0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -109,7 +109,8 @@ iOS Simulator launch behavior: - An `open` that launches an app reports `timing.postOpenObservation`. When it is present and not `"observable"`, the next command that reads the screen waits briefly until two reads agree, so a `click` lands where the element comes to rest. If the screen is still moving after that wait, the command proceeds and warns that the surface was still changing. - If CoreSimulator never answers a URL hand-off, `open` fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout`. The Simulator may still have received the URL; retry the open or answer the prompt with `alert accept`. - When `open --launch-url ` raises an `Open in ""?` prompt and the URL's scheme belongs to the session app, `open` taps **Open** and returns `launchConfirmation: "accepted"`. If the app still never starts, `open` hands over the URL once more. -- When that prompt stays unanswered, `open` returns `launchConfirmation: "unanswered"` with `launchConfirmationReason`: `alert-unrecognized` for a prompt it did not recognize (such as a non-English Simulator), or `url-owner-unresolved` when no single installed app owns the URL's scheme or that lookup failed. The prompt stays on screen; `warnings` name the next step. For `url-owner-unresolved`, answer the prompt with `alert accept`. For `alert-unrecognized`, read the alert with `alert get` first: it can be the app's own alert, such as a permission prompt, rather than a translated launch confirmation. Answer with `alert accept` only when it is the launch confirmation. +- `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. +- When that prompt stays unanswered, `open` returns `launchConfirmation: "unanswered"` with `launchConfirmationReason`: `alert-unrecognized` for a prompt it did not recognize, or `url-owner-unresolved` when no single installed app owns the URL's scheme or that lookup failed. The prompt stays on screen; `warnings` name the next step. For `url-owner-unresolved`, answer the prompt with `alert accept`. For `alert-unrecognized`, read the alert with `alert get` first: it can be the app's own alert, such as a permission prompt, rather than the launch confirmation. Answer with `alert accept` only when it is the launch confirmation. - When `open` could not read whether a prompt is up at all (the alert read, the accept, or the runner itself failed), `open` succeeds with `launchConfirmationProbeFailure` set to `alert-read`, `alert-accept`, or `runner` and no `launchConfirmation`, plus a warning to check with `alert get` before retrying. - `details.reason: launch_confirmation_unanswered` means the app never started after the prompt was accepted, or the prompt stayed visible after a failed tap. Answer it with `alert accept` or `alert dismiss` before continuing. - When the runner reports its main-thread timeout for that tap and the prompt is still up, `open` taps `Open` again, but only when the time the `open` request has left covers a whole tap and the read that checks it. The first tap is always sent. @@ -441,7 +442,7 @@ agent-device alert dismiss - On those backends a slow capture is never lost to its budget: a check that straddles the deadline still answers, which is why a quiet `alert accept`/`dismiss` always reports `alert-not-found` and never a timing-dependent failure (`alert get` answers `status: none` and `alert wait` times out on a quiet screen). The cost is a wait: the command stays open until the capture it started returns. On Android your cancellation stops the capture and the tap (the adb child is killed). On Limrun a request already sent to the instance cannot be stopped: the command ends when that last request answers (the Limrun client gives a request that stops answering its own ~30-second timeout) or the connection drops, and cancelling does not free it early. If a Limrun alert command stalls, restarting the instance session is what clears it. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), and location prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. +- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. From 1511ffbf2bc191b30ebda1ffccd214087987f228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 11:51:21 +0100 Subject: [PATCH 02/10] fix(ios): read the launch confirmation strings apart from the permission prompts A runtime without the SchemeApproval table no longer drops the notification, tracking, and location vocabulary. The test shares the CONFIRMATION fixture. Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerAlertLabelVocabulary.swift | 29 ++++++++++++------- .../RunnerSystemPromptStrings.swift | 7 +++-- .../src/launch-confirmation-runner.test.ts | 6 +--- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift index d9f662160a..384ad8ce2c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift @@ -38,24 +38,33 @@ extension RunnerAlertLabelVocabulary { static let system = fromSystemStrings(languages: Locale.preferredLanguages) /// Pairs each English prompt string with the same key's value in each of `languages`. Every - /// table must read, or none counts: a missing table could hide the translation that makes a - /// label in another table claim both actions. + /// permission table must read, or none counts: a missing table could hide the translation that + /// makes a label in another table claim both actions. The launch confirmation's table only adds + /// its own open and cancel labels, so without it the permission vocabulary stays whole. static func fromSystemStrings(languages: [String]) -> RunnerAlertLabelVocabulary { + var tables: [RunnerSystemPromptStrings] switch RunnerSystemPromptStrings.read(RunnerSystemPromptStrings.promptTables, languages: languages) { case .failure(let failure): RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) return .english - case .success(let tables): - var translations: [(english: String, localized: String)] = [] - for table in tables { - for strings in table.localized.values { - for (key, value) in table.english { - if let translation = strings[key] { translations.append((english: value, localized: translation)) } - } + case .success(let read): + tables = read + } + switch RunnerSystemPromptStrings.read([RunnerSystemPromptStrings.launchConfirmationTable], languages: languages) { + case .failure(let failure): + RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) + case .success(let read): + tables += read + } + var translations: [(english: String, localized: String)] = [] + for table in tables { + for strings in table.localized.values { + for (key, value) in table.english { + if let translation = strings[key] { translations.append((english: value, localized: translation)) } } } - return RunnerAlertLabelVocabulary(translations: translations) } + return RunnerAlertLabelVocabulary(translations: translations) } } #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift index ae29e48ee3..c41cdf3fdf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift @@ -8,14 +8,15 @@ struct RunnerSystemPromptStrings { let localized: [String: [String: String]] /// The string tables that word the system permission prompts: notifications, privacy access - /// (including App Tracking Transparency), and location, plus SpringBoard's launch confirmation. + /// (including App Tracking Transparency), and location. static let promptTables = [ (path: "/System/Library/PrivateFrameworks/UserNotificationsServer.framework", table: "Localizable"), (path: "/System/Library/PrivateFrameworks/TCC.framework", table: "Localizable"), - (path: "/System/Library/Frameworks/CoreLocation.framework", table: "locationd"), - launchConfirmationTable + (path: "/System/Library/Frameworks/CoreLocation.framework", table: "locationd") ] + /// SpringBoard's launch confirmation. It is read apart from `promptTables`, so a runtime without + /// it keeps the permission prompts' vocabulary. static let launchConfirmationTable = ( path: "/System/Library/Frameworks/CoreServices.framework", table: "SchemeApproval" diff --git a/packages/platform-apple/src/launch-confirmation-runner.test.ts b/packages/platform-apple/src/launch-confirmation-runner.test.ts index d8af77d324..a61eab8e9d 100644 --- a/packages/platform-apple/src/launch-confirmation-runner.test.ts +++ b/packages/platform-apple/src/launch-confirmation-runner.test.ts @@ -11,6 +11,7 @@ import { type RunnerCommand, } from '@agent-device/apple-runner'; import { appleRunnerHost } from './core/runner-host.ts'; +import { CONFIRMATION } from './launch-confirmation.fixtures.ts'; /** * The launch-confirmation answer over the real Apple alert legs and the real runner client: command @@ -69,11 +70,6 @@ const { answerSimulatorLaunchConfirmation } = await import('./launch-confirmatio const NOW_MS = 1_000_000; /** What the daemon gives a default `open`: its 90 s client envelope less the 30 s result margin. */ const OPEN_DEADLINE_MS = 60_000; -const CONFIRMATION = { - message: 'Open in “Example App”?', - items: ['Cancel', 'Open'], - launchConfirmation: true, -}; type ScriptedAnswer = | Readonly<{ kind: 'ok'; data: Record }> From 36b85c9450eeb1722818fec918aecedda787336a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 11:57:29 +0100 Subject: [PATCH 03/10] fix(ios): keep the launch confirmation mark apart from open's outcome Rename the runner's alert mark to launchConfirmationPrompt so it cannot be read as open's launchConfirmation outcome, and document it. Accept positional app-name placeholders in the OS title format, skip a wording with no placeholder, and name the failing subsystem in the unavailable-strings log. Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerAlertLabelVocabulary.swift | 4 ++-- .../RunnerLaunchConfirmationPrompt.swift | 18 +++++++++++++++--- .../RunnerSystemPromptStrings.swift | 6 +++--- .../RunnerTests+Alert.swift | 2 +- .../RunnerTests+Models.swift | 2 +- ...erTests+LaunchConfirmationPromptTests.swift | 18 ++++++++++++++++++ .../src/launch-confirmation.fixtures.ts | 2 +- .../src/launch-confirmation.test.ts | 2 +- .../platform-apple/src/launch-confirmation.ts | 2 +- website/docs/docs/commands.md | 2 +- 10 files changed, 44 insertions(+), 14 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift index 384ad8ce2c..00e39ce20e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift @@ -45,14 +45,14 @@ extension RunnerAlertLabelVocabulary { var tables: [RunnerSystemPromptStrings] switch RunnerSystemPromptStrings.read(RunnerSystemPromptStrings.promptTables, languages: languages) { case .failure(let failure): - RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) + RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "alert-vocabulary", languages: languages) return .english case .success(let read): tables = read } switch RunnerSystemPromptStrings.read([RunnerSystemPromptStrings.launchConfirmationTable], languages: languages) { case .failure(let failure): - RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) + RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "alert-vocabulary-launch-confirmation", languages: languages) case .success(let read): tables += read } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift index dc3601e228..6a39921bb3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift @@ -15,10 +15,22 @@ struct RunnerLaunchConfirmationPrompt { private let wordings: [(title: NSRegularExpression, open: String, cancel: String)] + /// A wording whose title format has no app-name placeholder (`%@` or `%1$@`) is skipped, because + /// its pattern would match a title with no app name. init(wordings: [Wording]) { + let placeholder = try! NSRegularExpression(pattern: "%(?:\\d+\\$)?@") self.wordings = wordings.compactMap { wording in - let pattern = NSRegularExpression.escapedPattern(for: wording.titleFormat) - .replacingOccurrences(of: "%@", with: ".+") + let format = wording.titleFormat as NSString + let slots = placeholder.matches(in: wording.titleFormat, range: NSRange(location: 0, length: format.length)) + guard !slots.isEmpty else { return nil } + var pattern = "" + var from = 0 + for slot in slots { + pattern += NSRegularExpression.escapedPattern(for: format.substring(with: NSRange(location: from, length: slot.range.location - from))) + pattern += ".+" + from = slot.range.location + slot.range.length + } + pattern += NSRegularExpression.escapedPattern(for: format.substring(from: from)) guard let title = try? NSRegularExpression(pattern: "^\(pattern)$") else { return nil } return (title, Self.normalized(wording.open), Self.normalized(wording.cancel)) } @@ -49,7 +61,7 @@ extension RunnerLaunchConfirmationPrompt { static func fromSystemStrings(languages: [String]) -> RunnerLaunchConfirmationPrompt { switch RunnerSystemPromptStrings.read([RunnerSystemPromptStrings.launchConfirmationTable], languages: languages) { case .failure(let failure): - RunnerSystemPromptStrings.logUnavailable(failure, languages: languages) + RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "launch-confirmation", languages: languages) return .unavailable case .success(let tables): guard let table = tables.first else { return .unavailable } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift index c41cdf3fdf..2ad1536550 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift @@ -56,10 +56,10 @@ struct RunnerSystemPromptStrings { let source: String } - static func logUnavailable(_ failure: UnavailableSource, languages: [String]) { + static func logUnavailable(_ failure: UnavailableSource, subsystem: String, languages: [String]) { NSLog( - "AGENT_DEVICE_RUNNER_ALERT_VOCABULARY_UNAVAILABLE source=%@ languages=%@", - failure.source, languages.joined(separator: ",") + "AGENT_DEVICE_RUNNER_SYSTEM_PROMPT_STRINGS_UNAVAILABLE subsystem=%@ source=%@ languages=%@", + subsystem, failure.source, languages.joined(separator: ",") ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 69996a23fa..a498dd9711 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -144,7 +144,7 @@ extension RunnerTests { data: DataPayload( message: title, items: items, - launchConfirmation: RunnerLaunchConfirmationPrompt.system.recognizes(title: title, buttons: items) ? true : nil + launchConfirmationPrompt: RunnerLaunchConfirmationPrompt.system.recognizes(title: title, buttons: items) ? true : nil ) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 8d08852bc8..789ead6a0c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -447,7 +447,7 @@ struct DataPayload: Codable { var items: [String]? /// Set on an alert read that is SpringBoard's `Open in “”?` launch confirmation, in the /// device's language; absent otherwise. - var launchConfirmation: Bool? + var launchConfirmationPrompt: Bool? var nodes: [PresentedNode]? var truncated: Bool? var qualityPayload: SnapshotQualityPayload? = nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift index bd447ed30f..8fc10d8576 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -59,3 +59,21 @@ extension RunnerTests { } } #endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + /// A wording with no app-name placeholder would match a title with no app name, and a positional + /// placeholder still marks the app-name slot. + func testLaunchConfirmationWordingNeedsAnAppNameSlot() { + let withoutSlot = RunnerLaunchConfirmationPrompt(wordings: [ + .init(titleFormat: "Open in the app?", open: "Open", cancel: "Cancel") + ]) + XCTAssertFalse(withoutSlot.recognizes(title: "Open in the app?", buttons: ["Cancel", "Open"])) + + let positional = RunnerLaunchConfirmationPrompt(wordings: [ + .init(titleFormat: "Open in “%1$@”?", open: "Open", cancel: "Cancel") + ]) + XCTAssertTrue(positional.recognizes(title: "Open in “Test App”?", buttons: ["Cancel", "Open"])) + } +} +#endif diff --git a/packages/platform-apple/src/launch-confirmation.fixtures.ts b/packages/platform-apple/src/launch-confirmation.fixtures.ts index 1c569ebe9f..937d838b98 100644 --- a/packages/platform-apple/src/launch-confirmation.fixtures.ts +++ b/packages/platform-apple/src/launch-confirmation.fixtures.ts @@ -5,7 +5,7 @@ import { AppError } from '@agent-device/kernel/errors'; export const CONFIRMATION = { message: 'Open in “Example App”?', items: ['Cancel', 'Open'], - launchConfirmation: true, + launchConfirmationPrompt: true, }; /** The runner's typed absence: `alert get` looked once and found no alert. */ diff --git a/packages/platform-apple/src/launch-confirmation.test.ts b/packages/platform-apple/src/launch-confirmation.test.ts index e024fd2587..343e1c2316 100644 --- a/packages/platform-apple/src/launch-confirmation.test.ts +++ b/packages/platform-apple/src/launch-confirmation.test.ts @@ -82,7 +82,7 @@ test.each([ const { port: device, acceptAlert } = port(async () => ({ message, items, - launchConfirmation: true, + launchConfirmationPrompt: true, })); await expect(answerLaunchConfirmation(device)).resolves.toEqual({ outcome: 'accepted' }); diff --git a/packages/platform-apple/src/launch-confirmation.ts b/packages/platform-apple/src/launch-confirmation.ts index 3286c2a099..44c870349a 100644 --- a/packages/platform-apple/src/launch-confirmation.ts +++ b/packages/platform-apple/src/launch-confirmation.ts @@ -189,7 +189,7 @@ function accepted(): LaunchConfirmationAttempt { * An alert the runner did not mark is reported as unrecognized. */ function isLaunchConfirmation(alert: Record): boolean { - return alert['launchConfirmation'] === true; + return alert['launchConfirmationPrompt'] === true; } /** A failed step leaves the open unanswered; the failure is reported, not thrown. */ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 6d60ade9c0..ad62ac28f3 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -442,7 +442,7 @@ agent-device alert dismiss - On those backends a slow capture is never lost to its budget: a check that straddles the deadline still answers, which is why a quiet `alert accept`/`dismiss` always reports `alert-not-found` and never a timing-dependent failure (`alert get` answers `status: none` and `alert wait` times out on a quiet screen). The cost is a wait: the command stays open until the capture it started returns. On Android your cancellation stops the capture and the tap (the adb child is killed). On Limrun a request already sent to the instance cannot be stopped: the command ends when that last request answers (the Limrun client gives a request that stops answering its own ~30-second timeout) or the connection drops, and cancelling does not free it early. If a Limrun alert command stalls, restarting the instance session is what clears it. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. +- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. From 81577da41076d528699db476b4ff5b5e35a3b519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 12:41:41 +0100 Subject: [PATCH 04/10] docs: scope the launch confirmation mark to local iOS simulators Co-Authored-By: Claude Sonnet 5.5 --- website/docs/docs/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index ad62ac28f3..0f024bbf66 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -442,7 +442,7 @@ agent-device alert dismiss - On those backends a slow capture is never lost to its budget: a check that straddles the deadline still answers, which is why a quiet `alert accept`/`dismiss` always reports `alert-not-found` and never a timing-dependent failure (`alert get` answers `status: none` and `alert wait` times out on a quiet screen). The cost is a wait: the command stays open until the capture it started returns. On Android your cancellation stops the capture and the tap (the adb child is killed). On Limrun a request already sent to the instance cannot be stopped: the command ends when that last request answers (the Limrun client gives a request that stops answering its own ~30-second timeout) or the connection drops, and cancelling does not free it early. If a Limrun alert command stalls, restarting the instance session is what clears it. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. +- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. On local iOS simulators, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below. From cdba0c18b0d859fdbb11d6ae3cf0f9de5f537b81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 12:58:07 +0100 Subject: [PATCH 05/10] fix(ios): mark only SpringBoard's modal and read bracketed title formats The launch-confirmation mark now needs the SpringBoard blocking-modal source, so an app alert with the same wording and buttons is not auto-accepted by open. The title placeholder accepts Apple's bracketed object format (Finnish `%[tt]@`). Tests load every installed SchemeApproval localization, cover the missing-table branch, and cover the app-owned alert. Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerAlertLabelVocabulary.swift | 24 +++-- .../RunnerLaunchConfirmationPrompt.swift | 7 +- .../RunnerTests+Alert.swift | 10 +- ...rTests+LaunchConfirmationPromptTests.swift | 93 +++++++++++++++++++ website/docs/docs/commands.md | 2 +- 5 files changed, 125 insertions(+), 11 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift index 00e39ce20e..4a53dc64de 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift @@ -37,20 +37,32 @@ extension RunnerAlertLabelVocabulary { /// preferred language the app localizes into, so every preferred language takes part. static let system = fromSystemStrings(languages: Locale.preferredLanguages) - /// Pairs each English prompt string with the same key's value in each of `languages`. Every - /// permission table must read, or none counts: a missing table could hide the translation that - /// makes a label in another table claim both actions. The launch confirmation's table only adds - /// its own open and cancel labels, so without it the permission vocabulary stays whole. + /// Pairs each English prompt string with the same key's value in each of `languages`. static func fromSystemStrings(languages: [String]) -> RunnerAlertLabelVocabulary { + composed( + permission: RunnerSystemPromptStrings.read(RunnerSystemPromptStrings.promptTables, languages: languages), + launchConfirmation: RunnerSystemPromptStrings.read([RunnerSystemPromptStrings.launchConfirmationTable], languages: languages), + languages: languages + ) + } + + /// Every permission table must read, or none counts: a missing table could hide the translation + /// that makes a label in another table claim both actions. The launch confirmation's table only + /// adds its own open and cancel labels, so without it the permission vocabulary stays whole. + static func composed( + permission: Result<[RunnerSystemPromptStrings], RunnerSystemPromptStrings.UnavailableSource>, + launchConfirmation: Result<[RunnerSystemPromptStrings], RunnerSystemPromptStrings.UnavailableSource>, + languages: [String] + ) -> RunnerAlertLabelVocabulary { var tables: [RunnerSystemPromptStrings] - switch RunnerSystemPromptStrings.read(RunnerSystemPromptStrings.promptTables, languages: languages) { + switch permission { case .failure(let failure): RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "alert-vocabulary", languages: languages) return .english case .success(let read): tables = read } - switch RunnerSystemPromptStrings.read([RunnerSystemPromptStrings.launchConfirmationTable], languages: languages) { + switch launchConfirmation { case .failure(let failure): RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "alert-vocabulary-launch-confirmation", languages: languages) case .success(let read): diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift index 6a39921bb3..8d2a270a98 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift @@ -15,10 +15,11 @@ struct RunnerLaunchConfirmationPrompt { private let wordings: [(title: NSRegularExpression, open: String, cancel: String)] - /// A wording whose title format has no app-name placeholder (`%@` or `%1$@`) is skipped, because - /// its pattern would match a title with no app name. + /// A wording whose title format has no app-name placeholder is skipped, because its pattern would + /// match a title with no app name. A placeholder is `%@`, a positional `%1$@`, or Apple's + /// bracketed object format `%[tt]@`, as Finnish words it. init(wordings: [Wording]) { - let placeholder = try! NSRegularExpression(pattern: "%(?:\\d+\\$)?@") + let placeholder = try! NSRegularExpression(pattern: "%(?:\\d+\\$)?(?:\\[[^\\]]*\\])?@") self.wordings = wordings.compactMap { wording in let format = wording.titleFormat as NSString let slots = placeholder.matches(in: wording.titleFormat, range: NSRange(location: 0, length: format.length)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index a498dd9711..ccdd208447 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -144,11 +144,19 @@ extension RunnerTests { data: DataPayload( message: title, items: items, - launchConfirmationPrompt: RunnerLaunchConfirmationPrompt.system.recognizes(title: title, buttons: items) ? true : nil + launchConfirmationPrompt: marksLaunchConfirmation(alert, title: title, items: items) ? true : nil ) ) } + /// The launch confirmation is SpringBoard's own modal. An app alert can carry the same wording + /// and buttons, and `open` accepts a marked alert, so the mark needs the SpringBoard source too. + func marksLaunchConfirmation(_ alert: RunnerAlert, title: String, items: [String]) -> Bool { + alert.source == .blockingSystemModal + && alert.ownerApp === springboard + && RunnerLaunchConfirmationPrompt.system.recognizes(title: title, buttons: items) + } + @MainActor func activateAlertButton( _ alert: RunnerAlert, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift index 8fc10d8576..af7c75aa69 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { @@ -75,5 +76,97 @@ extension RunnerTests { ]) XCTAssertTrue(positional.recognizes(title: "Open in “Test App”?", buttons: ["Cancel", "Open"])) } + + /// Apple's bracketed object format, as Finnish words the title, marks the app-name slot too, + /// also beside a position. + func testLaunchConfirmationWordingAcceptsBracketedObjectFormats() { + let finnish = RunnerLaunchConfirmationPrompt(wordings: [ + .init(titleFormat: "Avataanko %[tt]@-apissa?", open: "Avaa", cancel: "Peruuta") + ]) + XCTAssertTrue(finnish.recognizes(title: "Avataanko Agent Device Tester-apissa?", buttons: ["Peruuta", "Avaa"])) + XCTAssertFalse(finnish.recognizes(title: "Avataanko -apissa?", buttons: ["Peruuta", "Avaa"])) + + let positional = RunnerLaunchConfirmationPrompt(wordings: [ + .init(titleFormat: "Avataanko %1$[tt]@-apissa?", open: "Avaa", cancel: "Peruuta") + ]) + XCTAssertTrue(positional.recognizes(title: "Avataanko Testi-apissa?", buttons: ["Peruuta", "Avaa"])) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) && targetEnvironment(simulator) +extension RunnerTests { + /// Every `SchemeApproval` localization the simulated OS installs yields a matcher that + /// recognizes its own rendered title with its own buttons, so the next format variant fails here. + func testEveryInstalledSchemeApprovalWordingBecomesAMatcher() throws { + let foundationPath = "/System/Library/Frameworks/Foundation.framework" + let foundation = try XCTUnwrap(Bundle(identifier: "com.apple.Foundation")?.bundlePath) + let root = String(foundation.dropLast(foundationPath.count)) + let table = RunnerSystemPromptStrings.launchConfirmationTable + let bundle = try XCTUnwrap(Bundle(path: root + table.path)) + let localizations = bundle.localizations.filter { + $0 != "en" && $0 != "Base" + && bundle.url(forResource: table.table, withExtension: "strings", subdirectory: nil, localization: $0) != nil + } + XCTAssertGreaterThan(localizations.count, 10, "the runtime installs many SchemeApproval localizations") + let placeholder = try NSRegularExpression(pattern: "%(?:\\d+\\$)?(?:\\[[^\\]]*\\])?@") + for localization in localizations { + let url = try XCTUnwrap( + bundle.url(forResource: table.table, withExtension: "strings", subdirectory: nil, localization: localization) + ) + let strings = try XCTUnwrap(NSDictionary(contentsOf: url) as? [String: String]) + let format = try XCTUnwrap(strings["SCHEME_APPROVAL_PROMPT_TITLE_NO_SOURCE"], localization) + let open = try XCTUnwrap(strings["SCHEME_APPROVAL_OPEN_BUTTON"], localization) + let cancel = try XCTUnwrap(strings["SCHEME_APPROVAL_DO_NOT_OPEN_BUTTON"], localization) + let title = placeholder.stringByReplacingMatches( + in: format, range: NSRange(format.startIndex..., in: format), withTemplate: "Test App" + ) + XCTAssertNotEqual(title, format, "\(localization): the title format has an app-name slot the matcher knows") + let prompt = RunnerLaunchConfirmationPrompt.fromSystemStrings(languages: [localization]) + XCTAssertTrue(prompt.recognizes(title: title, buttons: [cancel, open]), "\(localization): \(format)") + } + } + + /// Without the `SchemeApproval` table the permission prompts keep their translations; only the + /// launch confirmation's own labels are lost. + func testMissingSchemeApprovalTableKeepsThePermissionVocabulary() { + let permission = RunnerSystemPromptStrings(english: ["DENY": "Don’t Allow"], localized: ["fr": ["DENY": "Refuser"]]) + let launch = RunnerSystemPromptStrings(english: ["OPEN": "Open"], localized: ["fr": ["OPEN": "Ouvrir"]]) + let missing = RunnerSystemPromptStrings.UnavailableSource(source: "CoreServices.framework") + + let without = RunnerAlertLabelVocabulary.composed( + permission: .success([permission]), launchConfirmation: .failure(missing), languages: ["fr"] + ) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Refuser", "Autoriser"], action: "dismiss", vocabulary: without), 0) + XCTAssertNil(RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "accept", vocabulary: without)) + + let with = RunnerAlertLabelVocabulary.composed( + permission: .success([permission]), launchConfirmation: .success([launch]), languages: ["fr"] + ) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "accept", vocabulary: with), 1) + + let noPermission = RunnerAlertLabelVocabulary.composed( + permission: .failure(missing), launchConfirmation: .success([launch]), languages: ["fr"] + ) + XCTAssertNil(RunnerAlertButtonChoice.buttonIndex(for: ["Refuser", "Autoriser"], action: "dismiss", vocabulary: noPermission)) + } + + /// An app alert can carry the exact wording and buttons, and `open` accepts a marked alert, so + /// only SpringBoard's own blocking modal is marked. + func testOnlySpringBoardsBlockingModalIsMarkedAsTheLaunchConfirmation() { + let title = "Open in “Test App”?" + let items = ["Cancel", "Open"] + let foreign = XCUIApplication(bundleIdentifier: "com.example.app") + func alert(owner: XCUIApplication, source: AlertSurfaceSource) -> RunnerAlert { + RunnerAlert(root: owner, ownerApp: owner, buttons: [], source: source, window: nil) + } + XCTAssertTrue(marksLaunchConfirmation(alert(owner: springboard, source: .blockingSystemModal), title: title, items: items)) + XCTAssertFalse(marksLaunchConfirmation(alert(owner: foreign, source: .appAlert), title: title, items: items)) + XCTAssertFalse(marksLaunchConfirmation(alert(owner: foreign, source: .blockingSystemModal), title: title, items: items)) + XCTAssertFalse(marksLaunchConfirmation(alert(owner: springboard, source: .appAlert), title: title, items: items)) + XCTAssertFalse( + marksLaunchConfirmation(alert(owner: springboard, source: .blockingSystemModal), title: "Allow Test App?", items: items) + ) + } } #endif diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 0f024bbf66..582d859de5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -109,7 +109,7 @@ iOS Simulator launch behavior: - An `open` that launches an app reports `timing.postOpenObservation`. When it is present and not `"observable"`, the next command that reads the screen waits briefly until two reads agree, so a `click` lands where the element comes to rest. If the screen is still moving after that wait, the command proceeds and warns that the surface was still changing. - If CoreSimulator never answers a URL hand-off, `open` fails within 20 seconds with `details.reason: ios-simulator-openurl-timeout`. The Simulator may still have received the URL; retry the open or answer the prompt with `alert accept`. - When `open --launch-url ` raises an `Open in ""?` prompt and the URL's scheme belongs to the session app, `open` taps **Open** and returns `launchConfirmation: "accepted"`. If the app still never starts, `open` hands over the URL once more. -- `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. +- On a local iOS Simulator, `open` recognizes the prompt in the Simulator's preferred languages, through the OS's own wording of its title and buttons, so a French Simulator's `Ouvrir dans « » ?` is answered like the English one. `alert accept` and `alert dismiss` answer it by the same wording. If the runner cannot read that wording, the prompt is not recognized. - When that prompt stays unanswered, `open` returns `launchConfirmation: "unanswered"` with `launchConfirmationReason`: `alert-unrecognized` for a prompt it did not recognize, or `url-owner-unresolved` when no single installed app owns the URL's scheme or that lookup failed. The prompt stays on screen; `warnings` name the next step. For `url-owner-unresolved`, answer the prompt with `alert accept`. For `alert-unrecognized`, read the alert with `alert get` first: it can be the app's own alert, such as a permission prompt, rather than the launch confirmation. Answer with `alert accept` only when it is the launch confirmation. - When `open` could not read whether a prompt is up at all (the alert read, the accept, or the runner itself failed), `open` succeeds with `launchConfirmationProbeFailure` set to `alert-read`, `alert-accept`, or `runner` and no `launchConfirmation`, plus a warning to check with `alert get` before retrying. - `details.reason: launch_confirmation_unanswered` means the app never started after the prompt was accepted, or the prompt stayed visible after a failed tap. Answer it with `alert accept` or `alert dismiss` before continuing. From f862d9ef3761b237f0130c6f7f2deddad135c308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 13:00:02 +0100 Subject: [PATCH 06/10] test(ios): cover Finnish in the system-strings launch confirmation test Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerTests+LaunchConfirmationPromptTests.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift index af7c75aa69..4639f0722b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -37,7 +37,7 @@ extension RunnerTests { /// The OS's own SchemeApproval strings supply the wording: a French or Japanese device reads the /// confirmation, and `accept` picks its open button where the English labels cannot. func testSystemStringsRecognizeAndAnswerTheLocalizedLaunchConfirmation() { - let languages = ["fr-FR", "ja-JP"] + let languages = ["fr-FR", "ja-JP", "fi-FI"] let prompt = RunnerLaunchConfirmationPrompt.fromSystemStrings(languages: languages) let vocabulary = RunnerAlertLabelVocabulary.fromSystemStrings(languages: languages) @@ -48,6 +48,9 @@ extension RunnerTests { XCTAssertTrue(prompt.recognizes(title: "“テスト”で開きますか?", buttons: ["キャンセル", "開く"])) XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["キャンセル", "開く"], action: "accept", vocabulary: vocabulary), 1) + XCTAssertTrue(prompt.recognizes(title: "Avataanko Testi-apissa?", buttons: ["Kumoa", "Avaa"])) + XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Kumoa", "Avaa"], action: "accept", vocabulary: vocabulary), 1) + XCTAssertFalse(prompt.recognizes(title: "Öffnen in „Test“?", buttons: ["Abbrechen", "Öffnen"]), "German is not preferred") } From 943506040589bd3d0dcba1411184160eaae44eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 13:43:39 +0100 Subject: [PATCH 07/10] fix(ios): take only the open and cancel labels of the SchemeApproval table Another key of that undocumented table cannot join the accept or dismiss labels. The placeholder regex no longer force-unwraps. Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerAlertLabelVocabulary.swift | 4 ++-- .../RunnerLaunchConfirmationPrompt.swift | 7 ++++++- .../RunnerSystemPromptStrings.swift | 12 ++++++++++++ .../RunnerTests+LaunchConfirmationPromptTests.swift | 9 ++++++++- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift index 4a53dc64de..b1b9ebc4c6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift @@ -48,7 +48,7 @@ extension RunnerAlertLabelVocabulary { /// Every permission table must read, or none counts: a missing table could hide the translation /// that makes a label in another table claim both actions. The launch confirmation's table only - /// adds its own open and cancel labels, so without it the permission vocabulary stays whole. + /// adds its own open and cancel labels (only those two keys are taken), so without it the permission vocabulary stays whole. static func composed( permission: Result<[RunnerSystemPromptStrings], RunnerSystemPromptStrings.UnavailableSource>, launchConfirmation: Result<[RunnerSystemPromptStrings], RunnerSystemPromptStrings.UnavailableSource>, @@ -66,7 +66,7 @@ extension RunnerAlertLabelVocabulary { case .failure(let failure): RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "alert-vocabulary-launch-confirmation", languages: languages) case .success(let read): - tables += read + tables += read.map { $0.limited(to: RunnerSystemPromptStrings.launchConfirmationLabelKeys) } } var translations: [(english: String, localized: String)] = [] for table in tables { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift index 8d2a270a98..21ef4ffeb2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift @@ -13,13 +13,18 @@ struct RunnerLaunchConfirmationPrompt { static let unavailable = RunnerLaunchConfirmationPrompt(wordings: []) + private static let appNamePlaceholder = try? NSRegularExpression(pattern: "%(?:\\d+\\$)?(?:\\[[^\\]]*\\])?@") + private let wordings: [(title: NSRegularExpression, open: String, cancel: String)] /// A wording whose title format has no app-name placeholder is skipped, because its pattern would /// match a title with no app name. A placeholder is `%@`, a positional `%1$@`, or Apple's /// bracketed object format `%[tt]@`, as Finnish words it. init(wordings: [Wording]) { - let placeholder = try! NSRegularExpression(pattern: "%(?:\\d+\\$)?(?:\\[[^\\]]*\\])?@") + guard let placeholder = Self.appNamePlaceholder else { + self.wordings = [] + return + } self.wordings = wordings.compactMap { wording in let format = wording.titleFormat as NSString let slots = placeholder.matches(in: wording.titleFormat, range: NSRange(location: 0, length: format.length)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift index 2ad1536550..0a81e000cc 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift @@ -17,6 +17,10 @@ struct RunnerSystemPromptStrings { /// SpringBoard's launch confirmation. It is read apart from `promptTables`, so a runtime without /// it keeps the permission prompts' vocabulary. + /// The only keys of that table the alert vocabulary takes, so another key of an undocumented + /// table cannot join the accept or dismiss labels. + static let launchConfirmationLabelKeys: Set = ["SCHEME_APPROVAL_OPEN_BUTTON", "SCHEME_APPROVAL_DO_NOT_OPEN_BUTTON"] + static let launchConfirmationTable = ( path: "/System/Library/Frameworks/CoreServices.framework", table: "SchemeApproval" @@ -52,6 +56,14 @@ struct RunnerSystemPromptStrings { return .success(read) } + /// This table restricted to `keys`. + func limited(to keys: Set) -> RunnerSystemPromptStrings { + RunnerSystemPromptStrings( + english: english.filter { keys.contains($0.key) }, + localized: localized.mapValues { $0.filter { keys.contains($0.key) } } + ) + } + struct UnavailableSource: Error { let source: String } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift index 4639f0722b..d33222a661 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -134,7 +134,10 @@ extension RunnerTests { /// launch confirmation's own labels are lost. func testMissingSchemeApprovalTableKeepsThePermissionVocabulary() { let permission = RunnerSystemPromptStrings(english: ["DENY": "Don’t Allow"], localized: ["fr": ["DENY": "Refuser"]]) - let launch = RunnerSystemPromptStrings(english: ["OPEN": "Open"], localized: ["fr": ["OPEN": "Ouvrir"]]) + let launch = RunnerSystemPromptStrings( + english: ["SCHEME_APPROVAL_OPEN_BUTTON": "Open", "SCHEME_APPROVAL_OTHER": "Continue"], + localized: ["fr": ["SCHEME_APPROVAL_OPEN_BUTTON": "Ouvrir", "SCHEME_APPROVAL_OTHER": "Poursuivre"]] + ) let missing = RunnerSystemPromptStrings.UnavailableSource(source: "CoreServices.framework") let without = RunnerAlertLabelVocabulary.composed( @@ -147,6 +150,10 @@ extension RunnerTests { permission: .success([permission]), launchConfirmation: .success([launch]), languages: ["fr"] ) XCTAssertEqual(RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "accept", vocabulary: with), 1) + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex(for: ["Refuser", "Poursuivre"], action: "accept", vocabulary: with), + "a key of the table other than its open and cancel labels adds no label" + ) let noPermission = RunnerAlertLabelVocabulary.composed( permission: .failure(missing), launchConfirmation: .success([launch]), languages: ["fr"] From b5eddc6119f0cd090a80be5fa579cbd5cc6bbca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 14:04:19 +0100 Subject: [PATCH 08/10] refactor(ios): tidy the system prompt strings docs and warm both readers apart Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerSystemPromptStrings.swift | 11 ++++++----- .../AgentDeviceRunnerUITests/RunnerTests.swift | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift index 0a81e000cc..d3d7e1d6c8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemPromptStrings.swift @@ -17,19 +17,20 @@ struct RunnerSystemPromptStrings { /// SpringBoard's launch confirmation. It is read apart from `promptTables`, so a runtime without /// it keeps the permission prompts' vocabulary. - /// The only keys of that table the alert vocabulary takes, so another key of an undocumented - /// table cannot join the accept or dismiss labels. - static let launchConfirmationLabelKeys: Set = ["SCHEME_APPROVAL_OPEN_BUTTON", "SCHEME_APPROVAL_DO_NOT_OPEN_BUTTON"] - static let launchConfirmationTable = ( path: "/System/Library/Frameworks/CoreServices.framework", table: "SchemeApproval" ) + /// The only keys of the launch confirmation table that the alert vocabulary takes, so another key + /// of an undocumented table cannot join the accept or dismiss labels. + static let launchConfirmationLabelKeys: Set = ["SCHEME_APPROVAL_OPEN_BUTTON", "SCHEME_APPROVAL_DO_NOT_OPEN_BUTTON"] + /// Reads `tables` from the simulated OS, or names the source that did not read. The simulated OS /// is taken from where this process's Foundation loaded; the simulator process would otherwise /// read the host's /System. A table reads in every preferred language or not at all, because a - /// missing translation could hide the one that makes a label claim both actions. + /// missing translation could hide the one that makes a label claim both actions. The read fails + /// as a whole when any listed table fails, so pass tables that must stand or fall together. static func read( _ tables: [(path: String, table: String)], languages: [String] diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 4b1ff6f579..b80bd8d09e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -317,10 +317,8 @@ final class RunnerTests: XCTestCase { NSLog("AGENT_DEVICE_RUNNER_DESIRED_PORT=%d", configuration.port.rawValue) listener = try Self.makeRunnerListener(port: configuration.port) // Reads the OS prompt strings off the alert path, so the first accept/dismiss deadline does not pay for it. - DispatchQueue.global(qos: .utility).async { - _ = RunnerAlertLabelVocabulary.system - _ = RunnerLaunchConfirmationPrompt.system - } + DispatchQueue.global(qos: .utility).async { _ = RunnerAlertLabelVocabulary.system } + DispatchQueue.global(qos: .utility).async { _ = RunnerLaunchConfirmationPrompt.system } listener?.stateUpdateHandler = { [weak self] state in switch state { case .ready: From af69d176bdaef6e0685e38fe670d69a841ca1017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 15:19:38 +0100 Subject: [PATCH 09/10] fix(ios): keep the launch confirmation labels when a permission table is unreadable The two reads are independent, so an unreadable permission table no longer discards Ouvrir and Annuler. The placeholder regex traps on a bad literal instead of silently disabling recognition. Co-Authored-By: Claude Sonnet 5.5 --- .../RunnerAlertLabelVocabulary.swift | 10 +++++----- .../RunnerLaunchConfirmationPrompt.swift | 7 ++----- .../RunnerTests+LaunchConfirmationPromptTests.swift | 10 ++++++++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift index b1b9ebc4c6..5649c2e55a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertLabelVocabulary.swift @@ -46,19 +46,19 @@ extension RunnerAlertLabelVocabulary { ) } - /// Every permission table must read, or none counts: a missing table could hide the translation - /// that makes a label in another table claim both actions. The launch confirmation's table only - /// adds its own open and cancel labels (only those two keys are taken), so without it the permission vocabulary stays whole. + /// Every permission table must read, or none of them counts: a missing table could hide the + /// translation that makes a label in another table claim both actions. The launch confirmation's + /// table is independent of them. Only its open and cancel labels are taken, so they stay when the + /// permission read fails, and the permission vocabulary stays whole when that table is missing. static func composed( permission: Result<[RunnerSystemPromptStrings], RunnerSystemPromptStrings.UnavailableSource>, launchConfirmation: Result<[RunnerSystemPromptStrings], RunnerSystemPromptStrings.UnavailableSource>, languages: [String] ) -> RunnerAlertLabelVocabulary { - var tables: [RunnerSystemPromptStrings] + var tables: [RunnerSystemPromptStrings] = [] switch permission { case .failure(let failure): RunnerSystemPromptStrings.logUnavailable(failure, subsystem: "alert-vocabulary", languages: languages) - return .english case .success(let read): tables = read } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift index 21ef4ffeb2..cd3f81601f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerLaunchConfirmationPrompt.swift @@ -13,7 +13,7 @@ struct RunnerLaunchConfirmationPrompt { static let unavailable = RunnerLaunchConfirmationPrompt(wordings: []) - private static let appNamePlaceholder = try? NSRegularExpression(pattern: "%(?:\\d+\\$)?(?:\\[[^\\]]*\\])?@") + private static let appNamePlaceholder = try! NSRegularExpression(pattern: "%(?:\\d+\\$)?(?:\\[[^\\]]*\\])?@") private let wordings: [(title: NSRegularExpression, open: String, cancel: String)] @@ -21,10 +21,7 @@ struct RunnerLaunchConfirmationPrompt { /// match a title with no app name. A placeholder is `%@`, a positional `%1$@`, or Apple's /// bracketed object format `%[tt]@`, as Finnish words it. init(wordings: [Wording]) { - guard let placeholder = Self.appNamePlaceholder else { - self.wordings = [] - return - } + let placeholder = Self.appNamePlaceholder self.wordings = wordings.compactMap { wording in let format = wording.titleFormat as NSString let slots = placeholder.matches(in: wording.titleFormat, range: NSRange(location: 0, length: format.length)) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift index d33222a661..8043005827 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+LaunchConfirmationPromptTests.swift @@ -158,7 +158,17 @@ extension RunnerTests { let noPermission = RunnerAlertLabelVocabulary.composed( permission: .failure(missing), launchConfirmation: .success([launch]), languages: ["fr"] ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "accept", vocabulary: noPermission), + 1, + "the launch confirmation's labels survive an unreadable permission table" + ) XCTAssertNil(RunnerAlertButtonChoice.buttonIndex(for: ["Refuser", "Autoriser"], action: "dismiss", vocabulary: noPermission)) + + let neither = RunnerAlertLabelVocabulary.composed( + permission: .failure(missing), launchConfirmation: .failure(missing), languages: ["fr"] + ) + XCTAssertNil(RunnerAlertButtonChoice.buttonIndex(for: ["Annuler", "Ouvrir"], action: "accept", vocabulary: neither)) } /// An app alert can carry the exact wording and buttons, and `open` accepts a marked alert, so From f1a8f0565eceff7125295eadebea18dcc3720f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Nov 2026 16:24:02 +0100 Subject: [PATCH 10/10] docs: describe the split alert string read Co-Authored-By: Claude Sonnet 5.5 --- website/docs/docs/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 582d859de5..244a240ce5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -442,7 +442,7 @@ agent-device alert dismiss - On those backends a slow capture is never lost to its budget: a check that straddles the deadline still answers, which is why a quiet `alert accept`/`dismiss` always reports `alert-not-found` and never a timing-dependent failure (`alert get` answers `status: none` and `alert wait` times out on a quiet screen). The cost is a wait: the command stays open until the capture it started returns. On Android your cancellation stops the capture and the tap (the adb child is killed). On Limrun a request already sent to the instance cannot be stopped: the command ends when that last request answers (the Limrun client gives a request that stops answering its own ~30-second timeout) or the connection drops, and cancelling does not free it early. If a Limrun alert command stalls, restarting the instance session is what clears it. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If any of those string tables cannot be read, the runner keeps the English labels only. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. On local iOS simulators, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. +- The Apple XCUITest runner (iOS, tvOS, and visionOS simulators and devices) and Limrun iOS use the same English label preferences after finding an alert: `Allow While Using App` outranks `Allow Once`, and equal-ranked choices refuse. `Open`, `Done`, and `Confirm…` accept; `Keep Browsing`, `Deny`, and `Skip` dismiss. On iOS simulators the local XCUITest runner also recognizes the wording of these labels in each of the device's preferred languages, read from the OS's own notification, privacy (including App Tracking Transparency), location, and `Open in …?` launch-confirmation prompt strings, at the rank of the English label they translate. The translations apply to any alert whose labels match them, not only permission prompts; a translation both roles claim stays unknown. If a notification, privacy, or location table cannot be read, the runner drops the permission translations and keeps the English labels; the `Open in …?` table is read apart, so its labels survive that, and the permission translations survive it being unreadable. Physical iOS devices, tvOS, and visionOS always use the English labels. Limrun iOS recognizes English labels only, because its tree capture cannot read the device's string tables. On local iOS simulators, `alert get` marks the `Open in …?` launch confirmation with `launchConfirmationPrompt: true`; `open` reads that mark to answer the prompt. Unknown labels refuse even for a lone button; read `alert get`, then choose explicitly with `snapshot -i` and `press @ref`. Limrun still needs at least two actionable buttons to recognize an alert. Ordinary Android dialogs also recognize these labels; Android permission-button ids keep priority. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or localized decline copy outside the preferred languages' prompt strings fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`); `no_text_input_at_point` and `text_input_probe_unavailable` mean a `fill` refused before dispatching anything (`dispatched: no`) — see Text entry below.