feat(provider-webos): add LG webOS TV provider plugin

.fallowrc.json

@@ -11,6 +11,8 @@
     "packages/provider-revyl/test/package-smoke.mjs",
     "packages/provider-revyl/tsdown.config.ts",
     "packages/provider-revyl/src/plugin.ts",
+    "packages/provider-webos/test/package-smoke.mjs",
+    "packages/provider-webos/tsdown.config.ts",
     "vitest.mutation.config.ts",
     "src/sdk/index.ts",
     "src/sdk/io.ts",
@@ -98,6 +100,10 @@
     // through this workspace link to core source so typecheck needs no build; Fallow then credits
     // the source files rather than the package.
     "agent-device",
+    // Nothing in the root imports the webOS plugin or the tv-cdp package it bundles, but layering
+    // requires the root manifest to declare every workspace package.
+    "@agent-device/tv-cdp",
+    "@agent-device/webos",
     // Oxlint resolves the shared config's JS plugins dynamically from their package names.
     "@nkzw/eslint-plugin",
     "eslint-plugin-no-only-tests",

examples/sdk/tsconfig.json

@@ -61,6 +61,7 @@
     { "path": "../../packages/provider-limrun" },
     { "path": "../../packages/provider-revyl" },
     { "path": "../../packages/provider-testmu" },
+    { "path": "../../packages/provider-webos" },
     { "path": "../../packages/provider-webdriver" },
     { "path": "../../packages/provision-kit" },
     { "path": "../../packages/proxy" },
@@ -71,6 +72,7 @@
     { "path": "../../packages/selectors" },
     { "path": "../../packages/session-journal" },
     { "path": "../../packages/test-kit" },
+    { "path": "../../packages/tv-cdp" },
     { "path": "../../packages/xml" }
   ]
 }

package.json

@@ -336,6 +336,7 @@
     "@agent-device/selectors": "workspace:*",
     "@agent-device/session-journal": "workspace:*",
     "@agent-device/test-kit": "workspace:*",
+    "@agent-device/tv-cdp": "workspace:*",
     "@agent-device/xml": "workspace:*",
     "@arethetypeswrong/cli": "^0.18.5",
     "@chenglou/freerange": "^0.0.4",
@@ -375,6 +376,7 @@
     "yaml": "^2.9.0",
     "yauzl": "^3.4.0",
     "@agent-device/testmu": "workspace:*",
-    "@agent-device/revyl": "workspace:*"
+    "@agent-device/revyl": "workspace:*",
+    "@agent-device/webos": "workspace:*"
   }
 }

packages/capture-kit/src/snapshot-desktop-projection.ts

@@ -16,19 +16,19 @@ const INTERACTIVE_ROLE_TOKENS = [
   'switch',
 ] as const;
 
+/** `isInteractive` overrides the default test for producers whose every node carries a rect. */
 export function shapeDesktopSurfaceSnapshot(
   data: SnapshotResult,
   options: Pick<SnapshotOptions, 'depth' | 'interactiveOnly' | 'scope'>,
+  isInteractive: (node: RawSnapshotNode) => boolean = isInteractiveSnapshotNode,
 ): SnapshotResult {
   let nodes = data.nodes ?? [];
   if (options.scope) {
     nodes = scopeSnapshotNodes(nodes, options.scope, (range) =>
-      options.interactiveOnly
-        ? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode)
-        : true,
+      options.interactiveOnly ? nodes.slice(range.start, range.end).some(isInteractive) : true,
     );
   }
-  if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes);
+  if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes, isInteractive);
   if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth);
   return { ...data, nodes };
 }
@@ -46,12 +46,15 @@ export function scopeSnapshotNodes(
   return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0);
 }
 
-function filterInteractiveSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] {
+function filterInteractiveSnapshotNodes(
+  nodes: RawSnapshotNode[],
+  isInteractive: (node: RawSnapshotNode) => boolean,
+): RawSnapshotNode[] {
   if (nodes.length === 0) return nodes;
   const byIndex = new Map(nodes.map((node) => [node.index, node]));
   const keepIndexes = new Set<number>();
   for (const node of nodes) {
-    if (!isInteractiveSnapshotNode(node)) continue;
+    if (!isInteractive(node)) continue;
     let current: RawSnapshotNode | undefined = node;
     while (current) {
       if (keepIndexes.has(current.index)) break;

packages/capture-kit/src/snapshot-occlusion.test.ts

@@ -237,6 +237,42 @@ test('annotateSnapshotNodesCoveredByPolicy refuses without guessing a cover', ()
   assert.equal(result[0]?.coveredBy, undefined);
 });
 
+test('annotateSnapshotNodesCoveredByPolicy publishes the cover a backend hit test names', () => {
+  const target: RawSnapshotNode = {
+    index: 0,
+    type: 'button',
+    hittable: true,
+    label: 'Watch',
+    rect: { x: 136, y: 731, width: 300, height: 72 },
+  };
+  const card = {
+    type: 'button',
+    label: 'Top story',
+    rect: { x: 104, y: 208, width: 1114, height: 627 },
+  };
+
+  const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => card);
+
+  assert.equal(result?.interactionBlocked, 'covered');
+  assert.equal(result?.hittable, false);
+  assert.deepEqual(result?.coveredBy, { kind: 'button', label: 'Top story', rect: card.rect });
+});
+
+test('annotateSnapshotNodesCoveredByPolicy names no cover when the hit element is an anonymous box', () => {
+  const target: RawSnapshotNode = {
+    index: 0,
+    type: 'button',
+    hittable: true,
+    label: 'Watch',
+    rect: { x: 136, y: 731, width: 300, height: 72 },
+  };
+
+  const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => ({ type: 'group' }));
+
+  assert.equal(result?.interactionBlocked, 'covered');
+  assert.equal(result?.coveredBy, undefined);
+});
+
 test('annotateCoveredSnapshotNodes leaves an uncovered touch target unchanged', () => {
   const target: RawSnapshotNode = {
     index: 0,

packages/capture-kit/src/snapshot-occlusion.ts

@@ -152,7 +152,7 @@ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']);
  * no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`)
  * still describes itself with no label to lean on.
  */
-function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode | null {
+function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null {
   const label = node.label?.trim();
   const kind = formatRole(node.type ?? 'Element');
   if (!label && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null;
@@ -164,14 +164,24 @@ function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode | nul
   };
 }
 
-/** Applies backend-owned covered decisions through the same publication annotation contract. */
+/** What a backend hit test observed on top of a node, described like a scanned cover. */
+export type CoveringNodeEvidence = Pick<RawSnapshotNode, 'type' | 'label' | 'rect'>;
+
+/**
+ * Applies backend-owned covered decisions through the same publication annotation contract. A
+ * decision that carries the covering element publishes its description; `true` refuses without one.
+ */
 export function annotateSnapshotNodesCoveredByPolicy(
   nodes: RawSnapshotNode[],
-  isCovered: (node: RawSnapshotNode) => boolean,
+  isCovered: (node: RawSnapshotNode) => boolean | CoveringNodeEvidence,
 ): RawSnapshotNode[] {
-  const coveredPositions: Array<{ position: number; coveredBy?: undefined }> = [];
+  const coveredPositions: Array<{ position: number; coveredBy?: SnapshotCoveringNode }> = [];
   for (const [position, node] of nodes.entries()) {
-    if (isCandidateTouchNode(node) && isCovered(node)) coveredPositions.push({ position });
+    if (!isCandidateTouchNode(node)) continue;
+    const decision = isCovered(node);
+    if (!decision) continue;
+    const coveredBy = decision === true ? null : describeCoveringNode(decision);
+    coveredPositions.push(coveredBy ? { position, coveredBy } : { position });
   }
   return annotateCoveredPositions(nodes, coveredPositions);
 }

packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts

@@ -67,6 +67,7 @@ const DEFAULT_QUIET_MS_BY_PRODUCER = {
   'limrun-ios-tree': DEFAULT_STABLE_QUIET_MS,
   'revyl-ios-tree': DEFAULT_STABLE_QUIET_MS,
   'android-uiautomator': DEFAULT_STABLE_QUIET_MS,
+  'cdp-dom': DEFAULT_STABLE_QUIET_MS,
   'harmonyos-uitest': DEFAULT_STABLE_QUIET_MS,
   'macos-helper': DEFAULT_STABLE_QUIET_MS,
   'linux-atspi': DEFAULT_STABLE_QUIET_MS,

packages/kernel/src/snapshot.ts

@@ -587,7 +587,7 @@ export type SnapshotProvenance =
         | 'limrun-ios-tree'
         | 'revyl-ios-tree';
     }
-  | { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' }
+  | { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' | 'cdp-dom' }
   | { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' }
   | { backend: 'macos-helper'; producer: 'macos-helper' }
   | { backend: 'linux-atspi'; producer: 'linux-atspi' }

packages/provider-testmu/test/package-smoke.mjs

@@ -1,3 +1,4 @@
+export const provider = 'testmu';
 export const args = [
   '--platform',
   'android',

packages/provider-webos/README.md

@@ -0,0 +1,118 @@
+# @agent-device/webos
+
+Drive web apps on LG webOS TVs with [agent-device](https://agent-device.dev). The plugin launches
+the app with the webOS CLI, opens the `ares-inspect` tunnel, and drives the page over the Chrome
+DevTools Protocol: remote keys, pointer, text, snapshots, screenshots and console logs.
+
+Status: experimental. Verified on an LG 32LQ63006LA (webOS TV 22, dev build of a React Native Web
+app): `connect`, `open`, `snapshot -i`, `snapshot -s`, `tv-remote` (directions, select, back),
+`is focused`, `press @ref --settle`, `wait text`, `find`, `screenshot`, `logs`, `apps`,
+`appstate`, `close`, `disconnect`.
+
+## How it plugs in
+
+agent-device leases only iOS, Android and HarmonyOS devices, so a webOS TV enters as an
+**Android TV device that this plugin owns** (`platform: android`, `target: tv`, id
+`webos:<ares device>`). Every command on that device goes to this plugin's runtime, never to adb.
+
+## Requirements
+
+- webOS CLI (`ares-*`) on `PATH`, or the directory that holds it passed as `aresBin`; see LG's
+  [webOS CLI installation](https://webostv.developer.lge.com/develop/tools/cli-installation)
+- The TV in Developer Mode and registered with `ares-setup-device`; see LG's
+  [App Testing with Developer Mode App](https://webostv.developer.lge.com/develop/getting-started/developer-mode-app)
+- The app installed on the TV
+- The TV and this host on the same network
+
+## Install
+
+```sh
+agent-device plugins add @agent-device/webos
+agent-device daemon stop     # the next command starts a daemon with the plugin loaded
+```
+
+Plugin options: `aresBin` (directory with the `ares-*` CLIs) and `device` (default ares device
+name).
+
+To try a local build, pack the package and install the tarball into a plugin store installation:
+
+```sh
+pnpm --filter @agent-device/webos pack --out /tmp/agent-device-webos.tgz
+installation=$(uuidgen)
+npm install --prefix ~/.agent-device/plugins/$installation /tmp/agent-device-webos.tgz
+```
+
+Then register `{"plugins": {"@agent-device/webos": {"installation": "<installation>"}}}` in
+`~/.agent-device/config.json` and run `agent-device daemon stop`.
+
+## Use
+
+```sh
+agent-device daemon stop                              # the lease TTL is read when the daemon starts
+export AGENT_DEVICE_LEASE_TTL_MS=1800000 AGENT_DEVICE_LEASE_MAX_TTL_MS=1800000  # see Known limitations
+export AGENT_DEVICE_SESSION=checkout-tv                # one session name per agent
+agent-device connect webos [--device <ares device>]   # defaults to the ares default device
+agent-device open <appId>                             # ares-launch, then attach over CDP
+agent-device tv-remote press down
+agent-device snapshot -i
+agent-device screenshot ./tv.png
+agent-device logs start                               # then logs path
+agent-device close <appId>
+agent-device disconnect
+```
+
+The TV is a leased device, so every command from `connect` to `disconnect` names its session with
+`AGENT_DEVICE_SESSION`, `--session <name>` or a config session. Without one, the command fails with
+`REMOTE_SESSION_REQUIRED`.
+
+## Supported
+
+| Command | How |
+| --- | --- |
+| `open` / `close` | `ares-launch` / `ares-launch --close`, then the `ares-inspect` tunnel |
+| `tv-remote press up/down/left/right/select/back`, `back` | `Input.dispatchKeyEvent` with the LG remote codes (Back = 461) |
+| `appstate` | the attached app while its page is visible |
+| `apps` | `ares-install --listfull` |
+| `logs` | console calls, exceptions and browser log entries of the attached page, one line each |
+| `snapshot`, `find`, `is`, `get`, `wait` | DOM walk over `Runtime.evaluate`: visible interactive, labelled or text nodes, focus from `document.activeElement`; `-s`, `-d` and `-i` apply to the walk. A control whose centre hit-tests to another element is marked `[covered]` |
+| `screenshot` | `Page.captureScreenshot` |
+| `click`, `press <x> <y>`, `longpress`, `hover` | `Input.dispatchMouseEvent` (Magic Remote pointer), ending each move with a small wiggle that wakes the cursor |
+| `fill` | pointer press on the field, select its contents, then `Input.insertText`, or Backspace for an empty value |
+| `type` | `Input.insertText` into the focused element |
+| `scroll` | `mouseWheel` at the centre of the scroll container or viewport, by the requested distance |
+
+Not yet: system buttons (`home`, menu, volume, channel and media keys), `network dump`, `perf`,
+`install` and `uninstall`, recording, URLs and deep links in `open`, `power`.
+
+## Known limitations
+
+- **Idle leases expire after 60 s by default, and expiry ends the session.** agent-device gives a
+  direct-device provider lease its default TTL and beats it only while a command runs. When the
+  lease ends, by `disconnect` or by expiry, the plugin closes the page connection and the
+  `ares-inspect` tunnel. After a longer pause, the next command fails with `Lease is not active`
+  or `No active session` and needs a new `connect` and `open`. To avoid it, run
+  `agent-device daemon stop`, export `AGENT_DEVICE_LEASE_TTL_MS` and
+  `AGENT_DEVICE_LEASE_MAX_TTL_MS` (for example `1800000`, 30 min), and run the next command in
+  that shell. That command starts the daemon with both values.
+- **`screenshot` captures the app page, not the TV screen.** When the app is in the background,
+  for example after Home on the remote, the page is hidden and renders no frames, so `screenshot`
+  fails at once; `open <appId>` brings the app back. A whole-screen capture is not reachable on a retail TV: SSAP
+  `tv/executeOneShot` answers `401 insufficient permissions` to unsigned clients, LG rejects the
+  public signed manifest (`blacklisted certificate`), and the dev-mode shell (`prisoner`) gets no
+  answer from `luna-send-pub`.
+- **Console messages from before the attach are replayed without object previews**, so objects
+  in them show as `Object`. Messages logged after the attach show their properties.
+- **`open` on a running app can reload it** (webOS relaunch), so the first read may hit a document
+  without a body. `open` waits for a loaded document, and a snapshot of an unloaded page is empty
+  rather than an error. `open --relaunch` closes the app first, for a cold start.
+- **A second DevTools client on the same page hung in testing.** Close Chrome DevTools and other
+  inspector clients attached to the app before a session.
+- **Snapshots are DOM-derived.** Labels come from `aria-label`, `alt`, `title` or own text;
+  rects are CSS px, which match screenshot pixels only at `devicePixelRatio` 1 (1080p apps).
+  The walk does not enter iframes or shadow roots, so overlays rendered there (for example a
+  dev-server error overlay) appear in screenshots but not in snapshots.
+- **`click`, `press`, `fill` and `focus` refuse a control marked `[covered]`.** A tap at its
+  centre would land on the element on top, such as a card's own pressable area. `snapshot -i`
+  still lists it, and `tv-remote` reaches it through focus.
+- **System overlays are outside the page.** The LG menu, on-screen keyboard and other system
+  layers never appear in `screenshot` or `snapshot`.

packages/provider-webos/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@agent-device/webos",
+  "version": "0.21.36-dev",
+  "type": "module",
+  "description": "LG webOS TV web app plugin for agent-device.",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsdown --config tsdown.config.ts",
+    "prepack": "pnpm build"
+  },
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    },
+    "./agent-device": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    },
+    "./package.json": "./package.json"
+  },
+  "devDependencies": {
+    "@agent-device/host-kit": "workspace:*",
+    "@agent-device/kernel": "workspace:*",
+    "@agent-device/test-kit": "workspace:*",
+    "@agent-device/tv-cdp": "workspace:*",
+    "agent-device": "workspace:*"
+  },
+  "publishConfig": {
+    "exports": {
+      ".": {
+        "import": "./dist/agent-device.mjs"
+      },
+      "./agent-device": {
+        "import": "./dist/agent-device.mjs"
+      }
+    },
+    "access": "public"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/callstack/agent-device.git",
+    "directory": "packages/provider-webos"
+  },
+  "homepage": "https://agent-device.dev/docs/webos"
+}