diff --git a/.lane/attic/native2/Sources/DesktopMac/Pointer.swift/01M6ERHPXM4ZVJSQWDFV4GWZBF-chromium-exposes-axpress-onl.md b/.lane/attic/native2/Sources/DesktopMac/Pointer.swift/01M6ERHPXM4ZVJSQWDFV4GWZBF-chromium-exposes-axpress-onl.md new file mode 100644 index 0000000..ed6733a --- /dev/null +++ b/.lane/attic/native2/Sources/DesktopMac/Pointer.swift/01M6ERHPXM4ZVJSQWDFV4GWZBF-chromium-exposes-axpress-onl.md @@ -0,0 +1,12 @@ +--- +id: 01M6ERHPXM4ZVJSQWDFV4GWZBF +anchor: '@file' +created: 2026-11-02T18:32:34Z +norm: '1' +sig: 27abcba5e975f16e +body_hash: 26774f711766dcfe +raw_hash: 0518e62bb9b3ac20 +lines: 1-516 +--- + +Chromium exposes AXPress only on elements with their own click listener; pages that delegate clicks at the root (jsaction, frameworks) expose nothing, so a Chromium click with no pressable candidate must fall back to a routed pointer click, which Chromium accepts in the background diff --git a/.lane/memory/native2/Sources/DesktopHost/Click.swift/01M6ERHPXTY4WPFS8YAQED9FTP-a-background-app-s-context-m.md b/.lane/memory/native2/Sources/DesktopHost/Click.swift/01M6ERHPXTY4WPFS8YAQED9FTP-a-background-app-s-context-m.md new file mode 100644 index 0000000..92ceeb5 --- /dev/null +++ b/.lane/memory/native2/Sources/DesktopHost/Click.swift/01M6ERHPXTY4WPFS8YAQED9FTP-a-background-app-s-context-m.md @@ -0,0 +1,12 @@ +--- +id: 01M6ERHPXTY4WPFS8YAQED9FTP +anchor: '@file' +created: 2026-11-02T18:32:34Z +norm: '1' +sig: 1574ff353918c008 +body_hash: 41da1377acf8f7eb +raw_hash: e4a74ca0b8f98d06 +lines: 1-130 +--- + +a background app's context menu closes when other input arrives, so items must be pressed in the call that opened it; AXShowMenu blocks while the menu is open, so a menu seen open is a finished delivery diff --git a/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5CM5YN1XDGY65VM1CHFA5PH-combinedsessionstate-counter.md b/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5CM5YN1XDGY65VM1CHFA5PH-combinedsessionstate-counter.md index e1b0f55..ba1fad8 100644 --- a/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5CM5YN1XDGY65VM1CHFA5PH-combinedsessionstate-counter.md +++ b/.lane/memory/native2/Sources/DesktopMac/ForegroundPointer.swift/01M5CM5YN1XDGY65VM1CHFA5PH-combinedsessionstate-counter.md @@ -4,9 +4,9 @@ anchor: '@file' created: 2026-10-20T12:22:04Z norm: '1' sig: 27abcba5e975f16e -body_hash: 672dfcdec2f063af -raw_hash: dbf52ab1720e2cc0 -vouched: 2026-11-02T16:12:35Z +body_hash: 6a3d6b95132b9fb4 +raw_hash: b9fc531ce6abe093 +vouched: 2026-11-02T18:32:47Z lines: 1-582 --- diff --git a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BM221JZ273DQQTVKJVZBQW-routed-events-name-the-windo.md b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BM221JZ273DQQTVKJVZBQW-routed-events-name-the-windo.md index aee5603..2a6099d 100644 --- a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BM221JZ273DQQTVKJVZBQW-routed-events-name-the-windo.md +++ b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BM221JZ273DQQTVKJVZBQW-routed-events-name-the-windo.md @@ -4,9 +4,9 @@ anchor: '@file' created: 2026-10-20T03:00:42Z norm: '1' sig: 27abcba5e975f16e -body_hash: 88fb3f87934b3c36 -raw_hash: 7fdf393a2afc7183 -vouched: 2026-11-02T16:12:35Z +body_hash: 99f766697f5cf49a +raw_hash: 2e2cf668fd7553e6 +vouched: 2026-11-03T02:45:55Z lines: 1-360 --- diff --git a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BVVPB930XQY34PWD5NYMMY-the-point-click-descendant-w.md b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BVVPB930XQY34PWD5NYMMY-the-point-click-descendant-w.md index f7a3a2c..69560ba 100644 --- a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BVVPB930XQY34PWD5NYMMY-the-point-click-descendant-w.md +++ b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M5BVVPB930XQY34PWD5NYMMY-the-point-click-descendant-w.md @@ -4,9 +4,9 @@ anchor: '@file' created: 2026-10-20T05:17:02Z norm: '1' sig: 27abcba5e975f16e -body_hash: 88fb3f87934b3c36 -raw_hash: 7fdf393a2afc7183 -vouched: 2026-11-02T16:12:35Z +body_hash: 99f766697f5cf49a +raw_hash: 2e2cf668fd7553e6 +vouched: 2026-11-03T02:45:55Z lines: 1-360 --- diff --git a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M6FH0PW0B2MSDCKFQ89Z7W8A-a-routed-left-click-into-a-c.md b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M6FH0PW0B2MSDCKFQ89Z7W8A-a-routed-left-click-into-a-c.md new file mode 100644 index 0000000..959d2b5 --- /dev/null +++ b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M6FH0PW0B2MSDCKFQ89Z7W8A-a-routed-left-click-into-a-c.md @@ -0,0 +1,13 @@ +--- +id: 01M6FH0PW0B2MSDCKFQ89Z7W8A +anchor: '@file' +created: 2026-11-03T01:40:11Z +norm: '1' +sig: 27abcba5e975f16e +body_hash: 99f766697f5cf49a +raw_hash: 2e2cf668fd7553e6 +vouched: 2026-11-03T02:45:55Z +lines: 1-560 +--- + +a routed left click into a Chromium window that isn't key in its app is dropped (right clicks still arrive); AXFocusedWindow can't tell, only the app's own AXFrontmost after prepareBackgroundWindow confirms it diff --git a/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M6FH414M25XSGQ0GX40CHS7N-chromium-exposes-axpress-onl.md b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M6FH414M25XSGQ0GX40CHS7N-chromium-exposes-axpress-onl.md new file mode 100644 index 0000000..38583c0 --- /dev/null +++ b/.lane/memory/native2/Sources/DesktopMac/Pointer.swift/01M6FH414M25XSGQ0GX40CHS7N-chromium-exposes-axpress-onl.md @@ -0,0 +1,14 @@ +--- +id: 01M6FH414M25XSGQ0GX40CHS7N +anchor: '@file' +created: 2026-11-03T01:42:00Z +norm: '1' +sig: 27abcba5e975f16e +body_hash: 99f766697f5cf49a +raw_hash: 2e2cf668fd7553e6 +vouched: 2026-11-03T02:45:55Z +lines: 1-560 +supersedes: 01M6ERHPXM4ZVJSQWDFV4GWZBF +--- + +Chromium exposes AXPress only on elements with their own click listener; pages that delegate clicks at the root (jsaction, frameworks) expose nothing, so a Chromium click with no pressable candidate falls back to a routed pointer click, after making the window key in its app (see 01M6FH0PW0) diff --git a/.lane/memory/src/tools.ts/01M6EGHDDEY51YSVQ7T6TS53SX-this-text-is-what-native2-se.md b/.lane/memory/src/tools.ts/01M6EGHDDEY51YSVQ7T6TS53SX-this-text-is-what-native2-se.md index b78dd0a..62bc3be 100644 --- a/.lane/memory/src/tools.ts/01M6EGHDDEY51YSVQ7T6TS53SX-this-text-is-what-native2-se.md +++ b/.lane/memory/src/tools.ts/01M6EGHDDEY51YSVQ7T6TS53SX-this-text-is-what-native2-se.md @@ -4,8 +4,9 @@ anchor: '@file' created: 2026-11-02T16:12:36Z norm: '1' sig: 78473a106add9cfc -body_hash: 7f06e23d5bcdb8a5 -raw_hash: 038b784f6508c201 +body_hash: 20a6956cc14ca9d2 +raw_hash: 0fb3a60c49e77ca1 +vouched: 2026-11-03T02:45:55Z lines: 1-980 supersedes: 01M5D682W73BEYBRAPTSK672XW --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 06c14b7..e71221d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Notable changes to Desktop Tools, newest first. Each release's GitHub release no ### Added - Chrome and Electron web accessibility that Desktop Tools turned on is turned back off once no read of that app has relied on it for 10 minutes, so a heavy page doesn't stay slow until the app quits. `web_accessibility: false` on `desktop_inspect`, `desktop_read` or `desktop_wait` turns it off at once and reports `restored`. Web accessibility another assistive app turned on is never turned off. ([#231](https://github.com/githubnext/desktop-tools/issues/231)) +- `desktop_click` with `kind: "right"` takes `menu_path`, the exact titles of a context-menu item, and opens the menu and presses that item in one call. A background app closes its context menu as soon as other input arrives, so a menu opened by one call was usually gone before the next could press an item. A right click without `menu_path` lists the menu it opened in `context_menu.items`. A wrong path closes the menu again, choosing nothing, and lists the titles found. ([#227](https://github.com/githubnext/desktop-tools/issues/227)) +- `desktop_click` takes `foreground: true` for a plain single, double or triple click with the real pointer, as modifier clicks already worked: the app is brought forward, clicked, and the pointer and focus are given back, with an `unknown` result. A background single click is an Accessibility press or selection, which apps that check `NSApp.currentEvent` for user input ignore; the description now says so, and a background click that selects a row says so in its `note`. ([#225](https://github.com/githubnext/desktop-tools/issues/225)) ### Fixed @@ -16,6 +18,7 @@ Notable changes to Desktop Tools, newest first. Each release's GitHub release no - Keys from `desktop_key` reach a SwiftUI `List` in the background. A window of an app that isn't active is made key first, without bringing the app forward (`key_window`: `made_key`, or `already_key`), since a `List`, `onDeleteCommand` and web views take some keys only in the key window. When only the window itself has keyboard focus, as after a background click selects a row, which selects it without focusing its list, the window's one list or table with a selected row is focused first (`list_focused`). Delete and Return were reported `completed` but never arrived. A window that can't be made key, or has no focused view, now gives an unknown result (`KEY_WINDOW_UNAVAILABLE`, `KEY_NO_FOCUSED_VIEW`) instead. In Chrome and other Chromium apps, the window is made key by a click on the blank part of the tab strip, since they have no standard title bar. A second preparation soon after the first no longer clicks the title bar again, which macOS took as a double-click that zoomed the window. ([#226](https://github.com/githubnext/desktop-tools/issues/226)) - `desktop_insert` into a web terminal's input, such as xterm.js in Google Cloud Shell, types the text instead of pasting it, and `route: "paste"` there is refused with `PASTE_UNVERIFIABLE_RECEIVER`. xterm clears its input after every keystroke or paste, so a paste could never be confirmed, stayed on the clipboard, and the shell could run it later. Typed text into a terminal is marked `web_terminal` and unverified, with a hint to check the screen with `ocr` and press Return with `desktop_key` rather than type again. ([#229](https://github.com/githubnext/desktop-tools/issues/229)) - A WebKit app's page, such as a WKWebView web app that isn't frontmost, shows its fields on the first `desktop_inspect`: WebKit builds the page's Accessibility tree only when first asked, so the first read found an empty group, and the read is now repeated, for up to 2 seconds, until the web area appears. Background `desktop_key` Command-V pastes into the page's focused field: the app names no focused element for a web view, so the page's focused field, which reports focus itself, is used instead, and the window is made key first. ([#5](https://github.com/githubnext/desktop-tools/issues/5)) +- A background click in Chrome and other Chromium apps no longer refuses clickable page content with "No pressable accessibility element", such as a Google Cloud console link or a row's ⋮ button, or a Google Calendar settings row clicked by element ID. Chromium marks only elements with their own click listener as pressable, and pages that listen once at their root mark nothing, so where nothing at the point is pressable the click is now sent as a pointer click routed to the window, which the page handles as a real click, with a `note` saying so. Chrome drops such a click in a window that isn't key in its app, as in a Chrome that was opened in the background and never brought forward, so the window is made key first, without bringing Chrome forward, as background keys do. When that can't be done the click is refused with nothing sent, and when the window can't be confirmed key afterwards the result is `unknown`. ([#230](https://github.com/githubnext/desktop-tools/issues/230)) ## [0.19.0] - 2026-11-02 diff --git a/dist/tools.js b/dist/tools.js index 4717db6..d320009 100644 --- a/dist/tools.js +++ b/dist/tools.js @@ -613,7 +613,7 @@ export function desktopTools({ place, app }) { { name: "desktop_click", op: "click", - description: `Click an element or point from a fresh desktop_inspect snapshot: kind single (default), double, right, middle or triple. Background, without moving the pointer; completed means delivered to that window, and a single click on a text field also reports whether it got focus. With modifiers (command, option, shift) it is foreground: the host activates the app and clicks with the real pointer and keys, then restores them, and the result is unknown, so inspect to check. A Control-click is kind right. Stale or unsupported targets are refused before input. ${expectNote}`, + description: `Click an element or point from a fresh desktop_inspect snapshot: kind single (default), double, right, middle or triple. Background, without moving the pointer; completed means delivered to that window, and a single click on a text field also reports whether it got focus. A background single click is an Accessibility press or selection, not a mouse event (in Chromium apps, a pointer click routed to the window where nothing is pressable), so an app that acts only on real user input may ignore it. With foreground true, or modifiers (command, option, shift), it is foreground: the host activates the app and clicks with the real pointer and keys, then restores them, and the result is unknown, so inspect to check. A Control-click is kind right. A right click lists the context_menu it opened; a background app's menu closes when other input arrives, so pass menu_path (exact titles) to choose an item in the same call. Stale or unsupported targets are refused before input. ${expectNote}`, parameters: { type: "object", required: ["snapshot"], @@ -629,6 +629,8 @@ export function desktopTools({ place, app }) { maxItems: 3, uniqueItems: true, }, + ...foreground, + menu_path: menuPath, expect, }, }, diff --git a/docs/tools.md b/docs/tools.md index 5783a33..3af18d9 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -121,7 +121,7 @@ A status item is pressed through Accessibility in its owner process without movi | Tool | Does | | --- | --- | -| `desktop_click` | Single, double, right, middle or triple click on an element or a screenshot point, optionally holding Command, Option or Shift | +| `desktop_click` | Single, double, right, middle or triple click on an element or a screenshot point, optionally in the foreground or holding Command, Option or Shift; a right click can choose a context-menu item by `menu_path` | | `desktop_scroll` | Scroll an element or point up, down, left or right | | `desktop_drag` | Press, move in a straight line or along a path of up to 16 points, and release inside one window, or into another window or app; optionally hold modifier keys | | `desktop_hover` | Move the real pointer over an element or point, to show tooltips and hover states | @@ -146,7 +146,7 @@ When the host turns on the window highlight, the window an inspection or action `desktop_key` goes to the control the snapshot reports as focused. When there is none, as in Finder lists, outlines, tables, canvases and many web areas, it goes to whatever has keyboard focus in that window; the result's `receiver` is `observed_control` or `window_focus`. Either way the key is sent in the background, and is refused unless the app's focused element is still inside the window, and the window is still the app's key window, before each event. -A `desktop_click` with `modifiers` (any of `command`, `option` and `shift`), such as Command-click or Shift-click to extend a selection in Finder or Mail, is foreground input even on an element, because an Accessibility press ignores modifier keys. The host activates the app, moves the real pointer to the point (an element's observed center), holds the keys with real key events around a real click, then restores the pointer and the previously focused app and window; the result reports `cursor_restoration` and `focus_restoration`. Nothing reads back what the click selected, so its result is `unknown`. A Control-click is a right click: use `kind: "right"` without modifiers. Modifiers work with single, double and triple clicks. +A `desktop_click` with `modifiers` (any of `command`, `option` and `shift`), such as Command-click or Shift-click to extend a selection in Finder or Mail, is foreground input even on an element, because an Accessibility press ignores modifier keys. The host activates the app, moves the real pointer to the point (an element's observed center), holds the keys with real key events around a real click, then restores the pointer and the previously focused app and window; the result reports `cursor_restoration` and `focus_restoration`. Nothing reads back what the click selected, so its result is `unknown`. A Control-click is a right click: use `kind: "right"` without modifiers. Modifiers work with single, double and triple clicks. `foreground: true` sends a plain single, double or triple click the same way, without keys, for apps that act only on real user input: a background single click is an Accessibility press or selection, so an app whose handler checks that `NSApp.currentEvent` is a mouse or key event, as many do to tell user clicks from programmatic changes, ignores it. A background click that selects a row says so in its `note`. ([#225](https://github.com/githubnext/desktop-tools/issues/225)) `desktop_key` takes `keys`, a sequence of up to 16 chords each with its own modifiers, or `repeat` with one `key` to press it up to 32 times, all against one snapshot. Every key of a sequence goes to the window's keyboard focus, since focus may move as it runs (as Tab does), and before each one the host checks that the app's focused element is still in this window and that it is still the app's key window. The sequence stops at the first key that is refused or fails, with `KEY_SEQUENCE_STOPPED`; `keys_sent` says how many went, and a sequence that stopped early is `unknown`. A key that opens another window, such as Command-N, moves focus out of this one, so the next key stops the sequence; if the host can't tell whether that key was posted, the message says it may have been sent. @@ -202,7 +202,7 @@ A secure text field, such as a password field, reports the role `AXTextField` wi The host replaces the field's contents with key events. The fill is completed when the field then shows only mask characters, exactly as many as the password has; otherwise it is `unknown`. Results and the activity log record that a fill happened and where, never the password or its length. A fill refused by the user, by the Keychain, or after a minute without an answer sends nothing, and is `refused` with `FILL_REFUSED`; so is a Keychain item whose server doesn't match the page, or a field that lost focus before typing. System password prompts, such as the administrator prompt (SecurityAgent), are refused. Touch ID isn't supported. -`desktop_menu` refuses a Paste command the same way while the app's focused element is a secure field. The item counts as Paste by its shortcut, Command-V with or without other modifiers, or by an identifier naming a paste action, so localized titles and variants such as Paste and Match Style are covered. Two ways to paste remain outside these checks: clicking Paste in a context menu with `desktop_click`, and dragging text onto the field. +`desktop_menu` refuses a Paste command the same way while the app's focused element is a secure field. The item counts as Paste by its shortcut, Command-V with or without other modifiers, or by an identifier naming a paste action, so localized titles and variants such as Paste and Match Style are covered. A context menu's Paste item chosen with `desktop_click`'s `menu_path` is refused the same way. One way to paste remains outside these checks: dragging text onto the field. `desktop_insert` also pastes an image or files into a native text area, such as TextEdit's rich text, Mail or Notes, in place of text; see [the details](#details-by-tool) for how it confirms the paste and when it refuses. @@ -268,7 +268,9 @@ These are the rest of each tool's contract, beyond its description. - **After an action,** the automatic observation uses accessibility mode. If it fails, inspect again with pixels mode. - **In a Chromium or Electron app,** it first turns on the app's web accessibility, once per launch, unless `web_accessibility` is false. The result's `web_accessibility` says what happened; see [Input](#input). -**`desktop_click`.** A single left click on a supported editable text field, by element or point, also requests keyboard focus. The result says whether focus was verified; use `desktop_select` to choose a range or caret position. Points stay bound to the captured window, even when the returned screenshot was scaled. A point click presses the Accessibility element there, and is refused before input when that element belongs to another window, so a covered window can't be clicked through. The result's `pressed` names the role and label of the element that got the press. WebKit presses a web menu item, button or tab through the first native button inside it, so one that holds a nested control, such as a menu row with a hover-only remove button, is never pressed through Accessibility: it gets a real click routed to its window at the point when that window is frontmost, and is otherwise refused, naming the `nested_control`. A sheet that Accessibility lists only inside its parent window, such as System Settings' Search Privacy sheet, or an Open panel on it, takes point clicks from a pixels snapshot of the sheet's own window. +**`desktop_click`.** A single left click on a supported editable text field, by element or point, also requests keyboard focus. The result says whether focus was verified; use `desktop_select` to choose a range or caret position. Points stay bound to the captured window, even when the returned screenshot was scaled. A point click presses the Accessibility element there, and is refused before input when that element belongs to another window, so a covered window can't be clicked through. The result's `pressed` names the role and label of the element that got the press. WebKit presses a web menu item, button or tab through the first native button inside it, so one that holds a nested control, such as a menu row with a hover-only remove button, is never pressed through Accessibility: it gets a real click routed to its window at the point when that window is frontmost, and is otherwise refused, naming the `nested_control`. In Chrome and other Chromium apps, a point with nothing pressable, such as a row or link on a page that listens for clicks once at its root (Google's apps, many frameworks), gets a pointer click routed to the window, where the page's own hit testing handles it, and the result's `note` says so. Chrome drops that click in a window that isn't key in its app, such as a window of a Chrome opened in the background and never brought forward, so the window is first made key, without activating Chrome; the click is refused when that can't be done, and `unknown` when the window can't be confirmed key afterwards. Chromium marks only elements with their own click listener as pressable ([#230](https://github.com/githubnext/desktop-tools/issues/230)). A sheet that Accessibility lists only inside its parent window, such as System Settings' Search Privacy sheet, or an Open panel on it, takes point clicks from a pixels snapshot of the sheet's own window. + +**Context menus.** A right click (`kind: "right"`) lists the context menu it opened in `context_menu.items`, each item's literal title path with `enabled: false` or `submenu: true` when they apply. An app in the background closes its context menu as soon as other input arrives, often before the next call, so choose an item in the same call: pass `menu_path`, the exact titles from the menu down to the command, such as `["More", "Archive"]`. The host opens the menu, presses that item through Accessibility and returns `completed` with the `menu_path` it pressed. A missing, ambiguous, disabled or submenu title closes the menu again without choosing anything, and the result is `unknown`, since the right click itself went out, with the titles found. ([#227](https://github.com/githubnext/desktop-tools/issues/227)) **`desktop_scroll`.** `amount` is in native scroll units. Their distance depends on the route the target supports, not on pixels. diff --git a/native2/Sources/DesktopHost/Click.swift b/native2/Sources/DesktopHost/Click.swift index 401e505..faa7508 100644 --- a/native2/Sources/DesktopHost/Click.swift +++ b/native2/Sources/DesktopHost/Click.swift @@ -12,16 +12,26 @@ extension Tools { guard let kind = ClickKind(rawValue: arguments["kind"]?.string ?? "single") else { throw ToolError("Choose single, double, right, middle, or triple click.") } - let modifiers = try clickModifiers(arguments, kind: kind) + var modifiers = try clickModifiers(arguments, kind: kind) + if try foregroundFlag(arguments) { + guard [.single, .double, .triple].contains(kind) else { + throw ToolError("foreground applies to single, double and triple clicks; right and middle clicks are already pointer events.") + } + modifiers = modifiers ?? [] + } + let path = try arguments["menu_path"].map { try menuPath($0) } + if path != nil, kind != .right { + throw ToolError("menu_path chooses an item of the context menu a right click opens; set kind right.") + } let send = { if let modifiers { return await modifierClick(snapshotID, element: element, fraction: fraction, kind: kind, modifiers: modifiers) } - return try await clickOnce(snapshotID, element: element, fraction: fraction, kind: kind) + return try await clickOnce(snapshotID, element: element, fraction: fraction, kind: kind, menuPath: path) } if let expect = try expectation(arguments) { return try await expectAfter(expect, send) } return try await send() } - private static func clickOnce(_ snapshotID: String, element: String?, fraction: CGPoint?, kind: ClickKind) async throws -> ToolResult { + private static func clickOnce(_ snapshotID: String, element: String?, fraction: CGPoint?, kind: ClickKind, menuPath: [String]?) async throws -> ToolResult { await act("click", snapshotID: snapshotID, element: element) { snapshot, target in let window = window(snapshot) let global = fraction.map { globalPoint($0, in: window.bounds) } ?? CGPoint(x: target!.frame.midX, y: target!.frame.midY) @@ -35,12 +45,86 @@ extension Tools { case (.single, nil): return try Mac.clickPoint(global, window: window) case (.right, let element?): - do { return try Mac.showMenu(element.reference, window: window) } catch is Unsupported { - return try Mac.routedClick(at: global, kind: kind, window: window) + let opened: Delivery + do { opened = try Mac.showMenu(element.reference, window: window) } catch is Unsupported { + opened = try Mac.routedClick(at: global, kind: kind, window: window) } + return try contextMenu(after: opened, pid: window.pid, path: menuPath) + case (.right, nil): + return try contextMenu(after: Mac.routedClick(at: global, kind: kind, window: window), pid: window.pid, path: menuPath) default: return try Mac.routedClick(at: global, kind: kind, window: window) } } } + + /// After a right click: the context menu it opened, listed, or with menu_path, the item chosen in the same call. A + /// background app's context menu closes as soon as other input arrives, often before the next call (#227). + static func contextMenu(after opened: Delivery, pid: pid_t, path: [String]?) throws -> Delivery { + guard let menu = Mac.openContextMenu(pid: pid, seconds: path == nil ? 0.6 : 1) else { + guard let path else { return opened } + var delivery = opened + delivery.unverified = true + delivery.fields.append(("error", errorJSON( + "DESKTOP_ACTION_FAILED", + "The right click was sent, but no context menu that Accessibility can see opened within a second, so \(String(reflecting: path)) was not chosen.", + hint: "Inspect the window to see what the right click did. Never repeat it blindly." + ))) + return delivery + } + // The menu seen open confirms the right click arrived. AXShowMenu doesn't return while its menu is open, so it + // reports a still-running or timed-out action even though it has done its part. + var base = Delivery.accepted(opened.mechanism ?? "accessibility_action", units: opened.units ?? 1) + base.fields = opened.fields + let items = contextMenuJSON(Mac.contextMenuRows(menu)) + guard let path else { + base.fields.append(("context_menu", .object([ + ("open", true), ("items", items), + ("note", "A background app's context menu closes as soon as other input arrives, so it may be gone by the next call. To choose an item, right-click again with menu_path set to its path."), + ]))) + return base + } + func failed(_ refusal: Refusal, listing: Bool) -> Delivery { + let closed = Mac.closeContextMenu(menu) + var delivery = base + delivery.unverified = true + if listing { delivery.fields.append(("context_menu", .object([("items", items)]))) } + delivery.fields.append(("error", errorJSON( + refusal.code, + refusal.message + (closed ? " The right click opened the menu and it was closed again without choosing anything." : " The right click opened the menu, and it may still be open; nothing in it was chosen."), + hint: refusal.hint ?? "Inspect the window before trying again." + ))) + return delivery + } + let item: AXUIElement + switch Mac.contextMenuItem(menu, path: path) { + case let .failure(refusal): return failed(refusal, listing: true) + case let .success(found): item = found + } + let result: AXError + do { result = try Mac.pressContextMenuItem(item, pid: pid) } catch let refusal as Refusal { + return failed(refusal, listing: false) + } + guard result == .success else { + if result == .actionUnsupported || result == .invalidUIElement { + return failed(Refusal("DESKTOP_ACTION_FAILED", "The menu item \(String(reflecting: path.last!)) couldn't be pressed (\(result.rawValue))."), listing: false) + } + var delivery = Delivery(state: .indeterminate, mechanism: "composite", evidence: "completion_unknown", units: (base.units ?? 1) + 1) + delivery.fields = base.fields + [("menu_path", .array(path.map(JSON.string)))] + return delivery + } + var delivery = Delivery.accepted(base.mechanism == "accessibility_action" ? "accessibility_action" : "composite", units: (base.units ?? 1) + 1) + delivery.fields = base.fields + [("menu_path", .array(path.map(JSON.string)))] + return delivery + } + + /// Each item's literal path, flagged when it is disabled or opens a submenu. + static func contextMenuJSON(_ rows: [ContextMenuRow]) -> JSON { + .array(rows.map { row in + var fields: [(String, JSON)] = [("path", .array(row.path.map(JSON.string)))] + if !row.enabled { fields.append(("enabled", false)) } + if row.submenu { fields.append(("submenu", true)) } + return .object(fields) + }) + } } diff --git a/native2/Sources/DesktopHost/Menus.swift b/native2/Sources/DesktopHost/Menus.swift index e30e600..672b35f 100644 --- a/native2/Sources/DesktopHost/Menus.swift +++ b/native2/Sources/DesktopHost/Menus.swift @@ -27,7 +27,7 @@ func appTargetJSON(_ pid: pid_t, _ generation: UInt64) -> JSON { .object([("pid", JSON(pid)), ("process_start_identity_decimal", JSON(String(generation)))]) } -private func menuPath(_ value: JSON?) throws -> [String] { +func menuPath(_ value: JSON?) throws -> [String] { guard let list = value?.array, (1...8).contains(list.count), let titles = Optional(list.compactMap(\.string)), titles.count == list.count, titles.allSatisfy({ !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && $0.utf16.count <= 512 }) diff --git a/native2/Sources/DesktopMac/ForegroundPointer.swift b/native2/Sources/DesktopMac/ForegroundPointer.swift index c3bf1e6..a414b59 100644 --- a/native2/Sources/DesktopMac/ForegroundPointer.swift +++ b/native2/Sources/DesktopMac/ForegroundPointer.swift @@ -261,8 +261,9 @@ extension Mac { // MARK: Modifier click - /// A click holding command, option or shift, which Accessibility presses can't carry: activate the app, click - /// with the real pointer, then give the pointer and focus back. + /// A click with the real pointer, holding command, option or shift or none: activate the app, click, then give the + /// pointer and focus back. Modifiers need it, since Accessibility presses can't carry them, and so do apps that act + /// only when NSApp.currentEvent is a real mouse event (#225). public static func modifierClick(at point: CGPoint, kind: ClickKind, modifiers: [String], window: ClickWindow) throws -> Delivery { guard window.bounds.contains(point) else { var refusal = Refusal("DESKTOP_ACTION_FAILED", "Modifier-click point must be finite and inside the exact target window.") @@ -406,7 +407,7 @@ extension Mac { ("cursor_restoration", JSON(cursor.rawValue)), ("focus_restoration", JSON(focusRestoration.rawValue)), ("error", .object([ ("code", "DESKTOP_ACTION_UNVERIFIED"), - ("message", JSON("The click was sent as real pointer input holding \(modifiers.joined(separator: " and ")), which activated the app and moved the pointer, then restored them (pointer \(cursor.rawValue), focus \(focusRestoration.rawValue)). Its effect can't be confirmed.")), + ("message", JSON("The click was sent as real pointer input\(modifiers.isEmpty ? "" : " holding \(modifiers.joined(separator: " and "))"), which activated the app and moved the pointer, then restored them (pointer \(cursor.rawValue), focus \(focusRestoration.rawValue)). Its effect can't be confirmed.")), ("hint", "Inspect the window to see what the click did, such as which items are selected. Never repeat the click blindly."), ])), ] diff --git a/native2/Sources/DesktopMac/Menus.swift b/native2/Sources/DesktopMac/Menus.swift index cb016e4..dbd96c4 100644 --- a/native2/Sources/DesktopMac/Menus.swift +++ b/native2/Sources/DesktopMac/Menus.swift @@ -346,3 +346,148 @@ extension Mac { return Set(list.compactMap { ($0[kCGWindowNumber as String] as? NSNumber)?.intValue }) } } + +// MARK: Context menus (#227) + +/// One item of an open context menu, as a click result lists it. +public struct ContextMenuRow: Sendable, Equatable { + public let path: [String] + public let enabled: Bool + public let submenu: Bool + public init(path: [String], enabled: Bool, submenu: Bool) { + self.path = path + self.enabled = enabled + self.submenu = submenu + } +} + +/// A context menu open in an app, held only for the call that opened it: a background app's menu closes as soon as +/// anything else takes input, so its items are listed or pressed in that call. +public struct OpenMenu: @unchecked Sendable { + let element: AXUIElement + let pid: pid_t +} + +extension Mac { + /// The context menu the app has open, waiting up to `seconds` for it: an AXMenu among the app's children, else the + /// AXMenu at the center of the app's new pop-up menu window. nil when none opened that Accessibility can see. + public static func openContextMenu(pid: pid_t, seconds: Double = 1) -> OpenMenu? { + let app = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(app, 0.5) + let deadline = Date().addingTimeInterval(seconds) + repeat { + if let menu = children(app).first(where: { AX.string($0, kAXRoleAttribute) == kAXMenuRole }) { + return OpenMenu(element: menu, pid: pid) + } + for frame in popUpMenuFrames(pid: pid) { + var hit: AXUIElement? + guard AXUIElementCopyElementAtPosition(app, Float(frame.midX), Float(frame.midY), &hit) == .success, var current = hit else { continue } + for _ in 0..<4 { + if AX.string(current, kAXRoleAttribute) == kAXMenuRole { return OpenMenu(element: current, pid: pid) } + guard let parent = AX.element(AX.value(current, kAXParentAttribute)) else { break } + current = parent + } + } + Thread.sleep(forTimeInterval: 0.05) + } while Date() < deadline + return nil + } + + /// The app's on-screen windows at the pop-up menu level, where context menus draw. + static func popUpMenuFrames(pid: pid_t) -> [CGRect] { + let rows = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? [] + return rows.compactMap { row in + guard row[kCGWindowOwnerPID as String] as? pid_t == pid, row[kCGWindowLayer as String] as? Int == Int(CGWindowLevelForKey(.popUpMenuWindow)), + let raw = row[kCGWindowBounds as String] as? NSDictionary, let bounds = CGRect(dictionaryRepresentation: raw) + else { return nil } + return bounds + } + } + + /// The open menu's items, depth first through submenus, at most 60; separators and untitled items are left out. + public static func contextMenuRows(_ menu: OpenMenu) -> [ContextMenuRow] { + var rows: [ContextMenuRow] = [] + func walk(_ element: AXUIElement, path: [String], depth: Int) { + for item in children(element) where rows.count < 60 { + guard AX.string(item, kAXRoleAttribute) == kAXMenuItemRole, let title = menuTitle(item), !title.isEmpty else { continue } + let submenu = children(item).first { AX.string($0, kAXRoleAttribute) == kAXMenuRole } + rows.append(ContextMenuRow(path: path + [title], enabled: (AX.value(item, kAXEnabledAttribute) as? Bool) != false, submenu: submenu != nil)) + if let submenu, depth < 3 { walk(submenu, path: path + [title], depth: depth + 1) } + } + } + walk(menu.element, path: [], depth: 1) + return rows + } + + private static func menuTitle(_ item: AXUIElement) -> String? { + AX.string(item, kAXTitleAttribute) ?? AX.string(item, kAXValueAttribute) + } + + /// The one enabled leaf at the literal path in an open context menu, or why there is none. + public static func contextMenuItem(_ menu: OpenMenu, path: [String]) -> Result { + resolveMenuPath( + menu.element, path: path, + items: { children($0).filter { AX.string($0, kAXRoleAttribute) == kAXMenuItemRole } }, + title: menuTitle, + submenu: { children($0).first { AX.string($0, kAXRoleAttribute) == kAXMenuRole } }, + enabled: { (AX.value($0, kAXEnabledAttribute) as? Bool) != false } + ) + } + + /// Walks literal titles through a menu's items and submenus to one enabled leaf. + static func resolveMenuPath( + _ root: Node, path: [String], items: (Node) -> [Node], title: (Node) -> String?, submenu: (Node) -> Node?, enabled: (Node) -> Bool + ) -> Result { + var level = root + for (index, wanted) in path.enumerated() { + let candidates = items(level) + let matches = candidates.filter { title($0).map { $0.utf16.elementsEqual(wanted.utf16) } == true } + guard matches.count == 1, let item = matches.first else { + let place = index == 0 ? "in the context menu" : "under \(String(reflecting: path[index - 1]))" + let titles = candidates.compactMap(title).filter { !$0.isEmpty }.prefix(30).map { String(reflecting: $0) } + return .failure(Refusal( + "DESKTOP_ACTION_FAILED", + matches.isEmpty + ? "No menu item titled \(String(reflecting: wanted)) \(place). Titles must match exactly. Titles \(place): \(titles.joined(separator: ", "))." + : "The menu path is ambiguous: \(matches.count) items titled \(String(reflecting: wanted)) \(place).", + hint: "Right-click again with a menu_path from these titles." + )) + } + let next = submenu(item) + if index == path.count - 1 { + guard next == nil else { + return .failure(Refusal("DESKTOP_ACTION_FAILED", "\(String(reflecting: wanted)) opens a submenu; add the title of one of its items to menu_path.", hint: "Right-click again with the full path to a command.")) + } + guard enabled(item) else { + return .failure(Refusal("DESKTOP_ACTION_FAILED", "The menu item \(String(reflecting: wanted)) is disabled.", hint: "Inspect the target to see why, then right-click again.")) + } + return .success(item) + } + guard let next else { + return .failure(Refusal("DESKTOP_ACTION_FAILED", "\(String(reflecting: wanted)) has no submenu.", hint: "Right-click again with a menu_path whose last title is a command.")) + } + level = next + } + return .failure(Refusal("DESKTOP_ACTION_FAILED", "The menu path names no item.")) + } + + /// Presses an item of an open context menu, which closes the menu. Throws a Refusal, having pressed nothing, for a + /// Paste item while a password field has focus. + public static func pressContextMenuItem(_ item: AXUIElement, pid: pid_t) throws -> AXError { + if focusIsSecure(pid), isPasteMenuItem(item) { throw secureFocusRefusal } + AXUIElementSetMessagingTimeout(item, 2) + return AXUIElementPerformAction(item, kAXPressAction as CFString) + } + + /// Closes an open context menu without choosing anything; true when it is gone afterwards. + @discardableResult + public static func closeContextMenu(_ menu: OpenMenu) -> Bool { + AXUIElementSetMessagingTimeout(menu.element, 1) + _ = AXUIElementPerformAction(menu.element, kAXCancelAction as CFString) + for _ in 0..<10 { + if popUpMenuFrames(pid: menu.pid).isEmpty { return true } + Thread.sleep(forTimeInterval: 0.05) + } + return false + } +} diff --git a/native2/Sources/DesktopMac/Pointer.swift b/native2/Sources/DesktopMac/Pointer.swift index 57b484b..a3e460c 100644 --- a/native2/Sources/DesktopMac/Pointer.swift +++ b/native2/Sources/DesktopMac/Pointer.swift @@ -190,7 +190,11 @@ extension Mac { return nil }() guard let (element, how) = pick else { - throw Refusal("DESKTOP_ACTION_FAILED", "No pressable accessibility element was found at (\(Int(point.x)), \(Int(point.y))) in PID \(window.pid). Background clicks press the accessibility element at the target point, and nothing pressable was found there — the point may be empty, a custom-drawn view, or an element that exposes no press action. Nothing was clicked. Inspect again and click an element, or click with a modifier to send a real pointer click in the foreground.") + // Chromium marks only elements with their own click listener as pressable, and pages that listen once at the + // root (Google's jsaction, React and other frameworks) mark nothing (#230). Chromium takes a click routed to + // its window in the background, so the page's own hit testing decides, as for a real click. + if routesThroughSkyLight(window.pid) { return try chromiumClick(at: point, window: window) } + throw Refusal("DESKTOP_ACTION_FAILED", "No pressable accessibility element was found at (\(Int(point.x)), \(Int(point.y))) in PID \(window.pid). Background clicks press the accessibility element at the target point, and nothing pressable was found there — the point may be empty, a custom-drawn view, or an element that exposes no press action. Nothing was clicked. Inspect again and click an element, or click with foreground true to send a real pointer click.") } // Only the same app's other windows are caught here; another app's window on top is not. if let owner = AX.element(AX.value(element, kAXWindowAttribute)), let id = AX.windowID(owner), id != window.windowID { @@ -205,8 +209,62 @@ extension Mac { try window.check() let error = AXUIElementSetAttributeValue(element, kAXSelectedAttribute as CFString, kCFBooleanTrue) guard error == .success else { return Delivery(state: .indeterminate, mechanism: "accessibility_value", evidence: "completion_unknown", units: 1) } - return .accepted("accessibility_value") + var delivery = Delivery.accepted("accessibility_value") + delivery.fields.append(("note", JSON(selectedRowNote))) + return delivery + } + } + + /// A row selected through Accessibility changes the selection without a mouse event, so an app that acts on a + /// selection only when NSApp.currentEvent is user input ignores it (#225). + public static let selectedRowNote = "The row was selected through Accessibility, not with a mouse event, so an app that acts only on user input, by checking NSApp.currentEvent, may ignore it. If the app didn't respond, click again with foreground true." + + /// A pointer click routed to a Chromium window where nothing is pressable (#230). AppKit drops a routed left click + /// in a window that isn't key in its app, as in a Chrome launched in the background and never activated, and a + /// window's Accessibility focus doesn't tell. So unless the app is in front, the window is first made key the way + /// background keys are sent (prepareBackgroundWindow), and the click is unknown unless the app then takes itself + /// for active with this window focused. + static func chromiumClick(at point: CGPoint, window: ClickWindow) throws -> Delivery { + let where_ = "(\(Int(point.x)), \(Int(point.y)))" + var prepared = 0 + if !isActive(window) { + do { prepared = try prepareBackgroundWindow(window) } catch let refusal as Refusal { + let cause = refusal.preparationCause + throw Refusal( + "DESKTOP_ACTION_FAILED", + "Nothing at \(where_) exposes an Accessibility press, and Chrome takes a background pointer click only in a window that is key in its app, which couldn't be arranged: \(cause) Nothing was clicked.", + hint: "Click with foreground true, or bring the window forward with desktop_focus and click again." + ) + } catch let stopped as PreparationStopped { + var delivery = Delivery(state: .indeterminate, mechanism: "window_targeted_events", evidence: "completion_unknown", units: stopped.units) + delivery.fields.append(("error", .object([ + ("code", "DESKTOP_ACTION_FAILED"), + ("message", JSON("Making the window key for a routed click stopped partway, and the click wasn't sent: \(stopped.cause)")), + ("hint", "Inspect the window before trying again; click with foreground true if the page didn't respond."), + ]))) + return delivery + } } + var delivery = try routedClick(at: point, kind: .single, window: window) + delivery = Delivery(state: delivery.state, mechanism: prepared > 0 ? "composite" : delivery.mechanism, evidence: delivery.evidence, units: (delivery.units ?? 0) + prepared) + delivery.fields.append(("note", "Nothing at the point exposes an Accessibility press, so the click was sent as a pointer click routed to the window, where the page handles it.")) + if !isActive(window) && !takesItselfForActive(window) { + delivery.unverified = true + delivery.fields.append(("error", .object([ + ("code", "DESKTOP_ACTION_UNVERIFIED"), + ("message", "The click was routed to the window, but the window couldn't be confirmed key in its app, and Chrome drops a click in a window that isn't, so it may not have reached the page."), + ("hint", "Inspect the page to see whether the click took effect; if not, click with foreground true."), + ]))) + } + return delivery + } + + /// Whether the app considers itself active with this window focused, as it does after prepareBackgroundWindow. + static func takesItselfForActive(_ window: ClickWindow) -> Bool { + let app = AXUIElementCreateApplication(window.pid) + AXUIElementSetMessagingTimeout(app, 0.5) + return AX.value(app, kAXFrontmostAttribute) as? Bool == true + && AX.element(AX.value(app, kAXFocusedWindowAttribute)).flatMap(AX.windowID) == window.windowID } // MARK: Presses WebKit would redirect diff --git a/native2/Sources/DesktopProtocol/ContractText.swift b/native2/Sources/DesktopProtocol/ContractText.swift index 88d380f..e340f41 100644 --- a/native2/Sources/DesktopProtocol/ContractText.swift +++ b/native2/Sources/DesktopProtocol/ContractText.swift @@ -1476,7 +1476,7 @@ Recipes for multi-step tasks, such as printing to PDF, installing from a DMG or }, { "name": "desktop_click", - "description": "Click an element or point from a fresh desktop_inspect snapshot: kind single (default), double, right, middle or triple. Background, without moving the pointer; completed means delivered to that window, and a single click on a text field also reports whether it got focus. With modifiers (command, option, shift) it is foreground: the host activates the app and clicks with the real pointer and keys, then restores them, and the result is unknown, so inspect to check. A Control-click is kind right. Stale or unsupported targets are refused before input. Optional expect is one desktop_wait condition this input should make true. It must not hold beforehand, or nothing is sent; if the outcome is unknown and it holds within timeout_ms (default 3000, at most 10000), the outcome becomes completed.", + "description": "Click an element or point from a fresh desktop_inspect snapshot: kind single (default), double, right, middle or triple. Background, without moving the pointer; completed means delivered to that window, and a single click on a text field also reports whether it got focus. A background single click is an Accessibility press or selection, not a mouse event (in Chromium apps, a pointer click routed to the window where nothing is pressable), so an app that acts only on real user input may ignore it. With foreground true, or modifiers (command, option, shift), it is foreground: the host activates the app and clicks with the real pointer and keys, then restores them, and the result is unknown, so inspect to check. A Control-click is kind right. A right click lists the context_menu it opened; a background app's menu closes when other input arrives, so pass menu_path (exact titles) to choose an item in the same call. Stale or unsupported targets are refused before input. Optional expect is one desktop_wait condition this input should make true. It must not hold beforehand, or nothing is sent; if the outcome is unknown and it holds within timeout_ms (default 3000, at most 10000), the outcome becomes completed.", "inputSchema": { "type": "object", "required": [ @@ -1537,6 +1537,19 @@ Recipes for multi-step tasks, such as printing to PDF, installing from a DMG or "maxItems": 3, "uniqueItems": true }, + "foreground": { + "type": "boolean" + }, + "menu_path": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 512 + }, + "minItems": 1, + "maxItems": 8 + }, "expect": { "type": "object", "description": "until and its arguments, as desktop_wait takes them, and an optional timeout_ms. A pid, and a window for window, element and text conditions, default to the action's snapshot.", diff --git a/native2/Tests/RuntimeTests/ContextMenuTests.swift b/native2/Tests/RuntimeTests/ContextMenuTests.swift new file mode 100644 index 0000000..9e71fe1 --- /dev/null +++ b/native2/Tests/RuntimeTests/ContextMenuTests.swift @@ -0,0 +1,77 @@ +@testable import DesktopHost +@testable import DesktopMac +import DesktopProtocol +import Testing + +/// A right click's context menu closes before a later call can press it in a background app, so the click lists it +/// or chooses an item by literal path in the same call (#227); a plain click can be sent in the foreground (#225). +struct ContextMenuTests { + final class Item: Sendable { + let title: String + let enabled: Bool + let submenu: [Item]? + init(_ title: String, enabled: Bool = true, submenu: [Item]? = nil) { + self.title = title + self.enabled = enabled + self.submenu = submenu + } + } + + static let menu = Item("", submenu: [ + Item("Rename"), Item("Delete", enabled: false), Item("Rename…"), Item("Dup"), Item("Dup"), + Item("More", submenu: [Item("Archive")]), + ]) + + static func resolve(_ path: [String]) -> Result { + Mac.resolveMenuPath(menu, path: path, items: { $0.submenu ?? [] }, title: { $0.title }, submenu: { item in item.submenu.map { _ in item } }, enabled: { $0.enabled }) + } + + static func message(_ result: Result) -> String { + if case let .failure(refusal) = result { return refusal.message } + return "" + } + + @Test + func `a literal path finds the leaf, through a submenu`() throws { + #expect(try Self.resolve(["Rename"]).get().title == "Rename") + #expect(try Self.resolve(["More", "Archive"]).get().title == "Archive") + } + + @Test + func `missing, ambiguous, disabled and submenu items are refused with what was found`() { + #expect(Self.message(Self.resolve(["Copy"])).contains("Titles in the context menu: \"Rename\", \"Delete\"")) + #expect(Self.message(Self.resolve(["Dup"])).contains("ambiguous: 2 items")) + #expect(Self.message(Self.resolve(["Delete"])).contains("disabled")) + #expect(Self.message(Self.resolve(["More"])).contains("opens a submenu")) + #expect(Self.message(Self.resolve(["Rename", "x"])).contains("has no submenu")) + } + + @Test + func `items list paths, flagging only disabled items and submenus`() { + let json = Tools.contextMenuJSON([ + ContextMenuRow(path: ["Rename"], enabled: true, submenu: false), + ContextMenuRow(path: ["Delete"], enabled: false, submenu: false), + ContextMenuRow(path: ["More"], enabled: true, submenu: true), + ]) + #expect(json.text == #"[{"path":["Rename"]},{"path":["Delete"],"enabled":false},{"path":["More"],"submenu":true}]"#) + } + + static func refusal(_ arguments: String) async -> String? { + do { + _ = try await Tools.click(JSON(parsing: arguments)) + return nil + } catch let error as ToolError { + return error.message + } catch { + return String(describing: error) + } + } + + @Test + func `foreground takes only left clicks, and menu_path only right clicks`() async { + #expect(await Self.refusal(#"{"snapshot":"s","point":{"x":0.5,"y":0.5},"kind":"right","foreground":true}"#)?.contains("foreground applies to single") == true) + #expect(await Self.refusal(#"{"snapshot":"s","point":{"x":0.5,"y":0.5},"menu_path":["Rename"]}"#)?.contains("set kind right") == true) + #expect(await Self.refusal(#"{"snapshot":"s","point":{"x":0.5,"y":0.5},"kind":"right","menu_path":[]}"#)?.contains("Menu path requires") == true) + #expect(await Self.refusal(#"{"snapshot":"s","point":{"x":0.5,"y":0.5},"foreground":"yes"}"#)?.contains("foreground to true or false") == true) + } +} diff --git a/native2/fixture/listfix.swift b/native2/fixture/listfix.swift new file mode 100644 index 0000000..4b38470 --- /dev/null +++ b/native2/fixture/listfix.swift @@ -0,0 +1,62 @@ +import AppKit +import SwiftUI +// The list fixture for #225 and #227: a SwiftUI List with Set selection whose handler, like many apps', acts only +// when NSApp.currentEvent is a mouse or key event, and rows with a context menu. It logs to stdout. Build it with +// `swiftc -O -parse-as-library native2/fixture/listfix.swift -o native2/.build/listfix`. + +func log(_ s: String) { + print(s) + fflush(stdout) +} + +/// Whether the event being handled is user input, as an app that ignores programmatic selection checks it. +func userInput() -> String { + guard let event = NSApp.currentEvent else { return "none" } + switch event.type { + case .leftMouseDown, .leftMouseUp, .rightMouseDown, .rightMouseUp, .keyDown: return "user(\(event.type.rawValue))" + default: return "other(\(event.type.rawValue))" + } +} + +final class Model: ObservableObject { + @Published var acted: [String] = [] +} + +struct Rows: View { + @State private var selection = Set() + @ObservedObject var model: Model + let rows = ["Alpha", "Bravo", "Charlie"] + + var body: some View { + VStack(alignment: .leading) { + List(rows, id: \.self, selection: $selection) { row in + Text(row) + .contextMenu { + Button("Rename \(row)") { act("rename \(row)") } + Button("Delete \(row)") { act("delete \(row)") } + Menu("More") { Button("Archive \(row)") { act("archive \(row)") } } + } + } + .onChange(of: selection) { _, new in + let how = userInput() + log("selection \(new.sorted()) event=\(how)") + if how.hasPrefix("user") { act("opened \(new.sorted().joined(separator: ","))") } + } + Text("Acted: " + (model.acted.last ?? "nothing")).padding(6) + } + .frame(width: 360, height: 260) + } + + func act(_ what: String) { + log("acted \(what)") + model.acted.append(what) + } +} + +@main +struct ListFixture: App { + @StateObject private var model = Model() + var body: some Scene { + Window("List Fixture", id: "list") { Rows(model: model) } + } +} diff --git a/src/tools.ts b/src/tools.ts index 9e4ce25..e346b5e 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -667,7 +667,7 @@ export function desktopTools({ place, app }: { place: string; app: string; wire? name: "desktop_click", op: "click", description: - `Click an element or point from a fresh desktop_inspect snapshot: kind single (default), double, right, middle or triple. Background, without moving the pointer; completed means delivered to that window, and a single click on a text field also reports whether it got focus. With modifiers (command, option, shift) it is foreground: the host activates the app and clicks with the real pointer and keys, then restores them, and the result is unknown, so inspect to check. A Control-click is kind right. Stale or unsupported targets are refused before input. ${expectNote}`, + `Click an element or point from a fresh desktop_inspect snapshot: kind single (default), double, right, middle or triple. Background, without moving the pointer; completed means delivered to that window, and a single click on a text field also reports whether it got focus. A background single click is an Accessibility press or selection, not a mouse event (in Chromium apps, a pointer click routed to the window where nothing is pressable), so an app that acts only on real user input may ignore it. With foreground true, or modifiers (command, option, shift), it is foreground: the host activates the app and clicks with the real pointer and keys, then restores them, and the result is unknown, so inspect to check. A Control-click is kind right. A right click lists the context_menu it opened; a background app's menu closes when other input arrives, so pass menu_path (exact titles) to choose an item in the same call. Stale or unsupported targets are refused before input. ${expectNote}`, parameters: { type: "object", required: ["snapshot"], @@ -683,6 +683,8 @@ export function desktopTools({ place, app }: { place: string; app: string; wire? maxItems: 3, uniqueItems: true, }, + ...foreground, + menu_path: menuPath, expect, }, },