diff --git a/.fallowrc.json b/.fallowrc.json index 40a51a2747..43f4763993 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -11,6 +11,8 @@ "packages/provider-revyl/test/package-smoke.mjs", "packages/provider-revyl/tsdown.config.ts", "packages/provider-revyl/src/plugin.ts", + "packages/provider-webos/test/package-smoke.mjs", + "packages/provider-webos/tsdown.config.ts", "vitest.mutation.config.ts", "src/sdk/index.ts", "src/sdk/io.ts", @@ -98,6 +100,10 @@ // through this workspace link to core source so typecheck needs no build; Fallow then credits // the source files rather than the package. "agent-device", + // Nothing in the root imports the webOS plugin or the tv-cdp package it bundles, but layering + // requires the root manifest to declare every workspace package. + "@agent-device/tv-cdp", + "@agent-device/webos", // Oxlint resolves the shared config's JS plugins dynamically from their package names. "@nkzw/eslint-plugin", "eslint-plugin-no-only-tests", diff --git a/examples/sdk/tsconfig.json b/examples/sdk/tsconfig.json index 3471e1fb7d..0a39ef5df1 100644 --- a/examples/sdk/tsconfig.json +++ b/examples/sdk/tsconfig.json @@ -61,6 +61,7 @@ { "path": "../../packages/provider-limrun" }, { "path": "../../packages/provider-revyl" }, { "path": "../../packages/provider-testmu" }, + { "path": "../../packages/provider-webos" }, { "path": "../../packages/provider-webdriver" }, { "path": "../../packages/provision-kit" }, { "path": "../../packages/proxy" }, @@ -71,6 +72,7 @@ { "path": "../../packages/selectors" }, { "path": "../../packages/session-journal" }, { "path": "../../packages/test-kit" }, + { "path": "../../packages/tv-cdp" }, { "path": "../../packages/xml" } ] } diff --git a/package.json b/package.json index 852742253a..7035902303 100644 --- a/package.json +++ b/package.json @@ -336,6 +336,7 @@ "@agent-device/selectors": "workspace:*", "@agent-device/session-journal": "workspace:*", "@agent-device/test-kit": "workspace:*", + "@agent-device/tv-cdp": "workspace:*", "@agent-device/xml": "workspace:*", "@arethetypeswrong/cli": "^0.18.5", "@chenglou/freerange": "^0.0.4", @@ -375,6 +376,7 @@ "yaml": "^2.9.0", "yauzl": "^3.4.0", "@agent-device/testmu": "workspace:*", - "@agent-device/revyl": "workspace:*" + "@agent-device/revyl": "workspace:*", + "@agent-device/webos": "workspace:*" } } diff --git a/packages/capture-kit/src/snapshot-desktop-projection.ts b/packages/capture-kit/src/snapshot-desktop-projection.ts index 0d5b0139ed..2765b39df5 100644 --- a/packages/capture-kit/src/snapshot-desktop-projection.ts +++ b/packages/capture-kit/src/snapshot-desktop-projection.ts @@ -16,19 +16,19 @@ const INTERACTIVE_ROLE_TOKENS = [ 'switch', ] as const; +/** `isInteractive` overrides the default test for producers whose every node carries a rect. */ export function shapeDesktopSurfaceSnapshot( data: SnapshotResult, options: Pick, + isInteractive: (node: RawSnapshotNode) => boolean = isInteractiveSnapshotNode, ): SnapshotResult { let nodes = data.nodes ?? []; if (options.scope) { nodes = scopeSnapshotNodes(nodes, options.scope, (range) => - options.interactiveOnly - ? nodes.slice(range.start, range.end).some(isInteractiveSnapshotNode) - : true, + options.interactiveOnly ? nodes.slice(range.start, range.end).some(isInteractive) : true, ); } - if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes); + if (options.interactiveOnly) nodes = filterInteractiveSnapshotNodes(nodes, isInteractive); if (typeof options.depth === 'number') nodes = filterSnapshotNodesByDepth(nodes, options.depth); return { ...data, nodes }; } @@ -46,12 +46,15 @@ export function scopeSnapshotNodes( return reindexSnapshotNodes(slice, slice[0]?.depth ?? 0); } -function filterInteractiveSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { +function filterInteractiveSnapshotNodes( + nodes: RawSnapshotNode[], + isInteractive: (node: RawSnapshotNode) => boolean, +): RawSnapshotNode[] { if (nodes.length === 0) return nodes; const byIndex = new Map(nodes.map((node) => [node.index, node])); const keepIndexes = new Set(); for (const node of nodes) { - if (!isInteractiveSnapshotNode(node)) continue; + if (!isInteractive(node)) continue; let current: RawSnapshotNode | undefined = node; while (current) { if (keepIndexes.has(current.index)) break; diff --git a/packages/capture-kit/src/snapshot-occlusion.test.ts b/packages/capture-kit/src/snapshot-occlusion.test.ts index 02bfe14a05..69a0589a72 100644 --- a/packages/capture-kit/src/snapshot-occlusion.test.ts +++ b/packages/capture-kit/src/snapshot-occlusion.test.ts @@ -237,6 +237,42 @@ test('annotateSnapshotNodesCoveredByPolicy refuses without guessing a cover', () assert.equal(result[0]?.coveredBy, undefined); }); +test('annotateSnapshotNodesCoveredByPolicy publishes the cover a backend hit test names', () => { + const target: RawSnapshotNode = { + index: 0, + type: 'button', + hittable: true, + label: 'Watch', + rect: { x: 136, y: 731, width: 300, height: 72 }, + }; + const card = { + type: 'button', + label: 'Top story', + rect: { x: 104, y: 208, width: 1114, height: 627 }, + }; + + const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => card); + + assert.equal(result?.interactionBlocked, 'covered'); + assert.equal(result?.hittable, false); + assert.deepEqual(result?.coveredBy, { kind: 'button', label: 'Top story', rect: card.rect }); +}); + +test('annotateSnapshotNodesCoveredByPolicy names no cover when the hit element is an anonymous box', () => { + const target: RawSnapshotNode = { + index: 0, + type: 'button', + hittable: true, + label: 'Watch', + rect: { x: 136, y: 731, width: 300, height: 72 }, + }; + + const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => ({ type: 'group' })); + + assert.equal(result?.interactionBlocked, 'covered'); + assert.equal(result?.coveredBy, undefined); +}); + test('annotateCoveredSnapshotNodes leaves an uncovered touch target unchanged', () => { const target: RawSnapshotNode = { index: 0, diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index dec0c4f148..7d9aef1526 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -152,7 +152,7 @@ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); * no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`) * still describes itself with no label to lean on. */ -function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode | null { +function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { const label = node.label?.trim(); const kind = formatRole(node.type ?? 'Element'); if (!label && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null; @@ -164,14 +164,24 @@ function describeCoveringNode(node: RawSnapshotNode): SnapshotCoveringNode | nul }; } -/** Applies backend-owned covered decisions through the same publication annotation contract. */ +/** What a backend hit test observed on top of a node, described like a scanned cover. */ +export type CoveringNodeEvidence = Pick; + +/** + * Applies backend-owned covered decisions through the same publication annotation contract. A + * decision that carries the covering element publishes its description; `true` refuses without one. + */ export function annotateSnapshotNodesCoveredByPolicy( nodes: RawSnapshotNode[], - isCovered: (node: RawSnapshotNode) => boolean, + isCovered: (node: RawSnapshotNode) => boolean | CoveringNodeEvidence, ): RawSnapshotNode[] { - const coveredPositions: Array<{ position: number; coveredBy?: undefined }> = []; + const coveredPositions: Array<{ position: number; coveredBy?: SnapshotCoveringNode }> = []; for (const [position, node] of nodes.entries()) { - if (isCandidateTouchNode(node) && isCovered(node)) coveredPositions.push({ position }); + if (!isCandidateTouchNode(node)) continue; + const decision = isCovered(node); + if (!decision) continue; + const coveredBy = decision === true ? null : describeCoveringNode(decision); + coveredPositions.push(coveredBy ? { position, coveredBy } : { position }); } return annotateCoveredPositions(nodes, coveredPositions); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index f080947a08..c3b5df9c38 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -67,6 +67,7 @@ const DEFAULT_QUIET_MS_BY_PRODUCER = { 'limrun-ios-tree': DEFAULT_STABLE_QUIET_MS, 'revyl-ios-tree': DEFAULT_STABLE_QUIET_MS, 'android-uiautomator': DEFAULT_STABLE_QUIET_MS, + 'cdp-dom': DEFAULT_STABLE_QUIET_MS, 'harmonyos-uitest': DEFAULT_STABLE_QUIET_MS, 'macos-helper': DEFAULT_STABLE_QUIET_MS, 'linux-atspi': DEFAULT_STABLE_QUIET_MS, diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 04b1725aa6..84cc2fc28a 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -587,7 +587,7 @@ export type SnapshotProvenance = | 'limrun-ios-tree' | 'revyl-ios-tree'; } - | { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' } + | { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' | 'cdp-dom' } | { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' } | { backend: 'macos-helper'; producer: 'macos-helper' } | { backend: 'linux-atspi'; producer: 'linux-atspi' } diff --git a/packages/provider-testmu/test/package-smoke.mjs b/packages/provider-testmu/test/package-smoke.mjs index fb29daa306..a1d49edeab 100644 --- a/packages/provider-testmu/test/package-smoke.mjs +++ b/packages/provider-testmu/test/package-smoke.mjs @@ -1,3 +1,4 @@ +export const provider = 'testmu'; export const args = [ '--platform', 'android', diff --git a/packages/provider-webos/README.md b/packages/provider-webos/README.md new file mode 100644 index 0000000000..bb5ab85466 --- /dev/null +++ b/packages/provider-webos/README.md @@ -0,0 +1,118 @@ +# @agent-device/webos + +Drive web apps on LG webOS TVs with [agent-device](https://agent-device.dev). The plugin launches +the app with the webOS CLI, opens the `ares-inspect` tunnel, and drives the page over the Chrome +DevTools Protocol: remote keys, pointer, text, snapshots, screenshots and console logs. + +Status: experimental. Verified on an LG 32LQ63006LA (webOS TV 22, dev build of a React Native Web +app): `connect`, `open`, `snapshot -i`, `snapshot -s`, `tv-remote` (directions, select, back), +`is focused`, `press @ref --settle`, `wait text`, `find`, `screenshot`, `logs`, `apps`, +`appstate`, `close`, `disconnect`. + +## How it plugs in + +agent-device leases only iOS, Android and HarmonyOS devices, so a webOS TV enters as an +**Android TV device that this plugin owns** (`platform: android`, `target: tv`, id +`webos:`). Every command on that device goes to this plugin's runtime, never to adb. + +## Requirements + +- webOS CLI (`ares-*`) on `PATH`, or the directory that holds it passed as `aresBin`; see LG's + [webOS CLI installation](https://webostv.developer.lge.com/develop/tools/cli-installation) +- The TV in Developer Mode and registered with `ares-setup-device`; see LG's + [App Testing with Developer Mode App](https://webostv.developer.lge.com/develop/getting-started/developer-mode-app) +- The app installed on the TV +- The TV and this host on the same network + +## Install + +```sh +agent-device plugins add @agent-device/webos +agent-device daemon stop # the next command starts a daemon with the plugin loaded +``` + +Plugin options: `aresBin` (directory with the `ares-*` CLIs) and `device` (default ares device +name). + +To try a local build, pack the package and install the tarball into a plugin store installation: + +```sh +pnpm --filter @agent-device/webos pack --out /tmp/agent-device-webos.tgz +installation=$(uuidgen) +npm install --prefix ~/.agent-device/plugins/$installation /tmp/agent-device-webos.tgz +``` + +Then register `{"plugins": {"@agent-device/webos": {"installation": ""}}}` in +`~/.agent-device/config.json` and run `agent-device daemon stop`. + +## Use + +```sh +agent-device daemon stop # the lease TTL is read when the daemon starts +export AGENT_DEVICE_LEASE_TTL_MS=1800000 AGENT_DEVICE_LEASE_MAX_TTL_MS=1800000 # see Known limitations +export AGENT_DEVICE_SESSION=checkout-tv # one session name per agent +agent-device connect webos [--device ] # defaults to the ares default device +agent-device open # ares-launch, then attach over CDP +agent-device tv-remote press down +agent-device snapshot -i +agent-device screenshot ./tv.png +agent-device logs start # then logs path +agent-device close +agent-device disconnect +``` + +The TV is a leased device, so every command from `connect` to `disconnect` names its session with +`AGENT_DEVICE_SESSION`, `--session ` or a config session. Without one, the command fails with +`REMOTE_SESSION_REQUIRED`. + +## Supported + +| Command | How | +| --- | --- | +| `open` / `close` | `ares-launch` / `ares-launch --close`, then the `ares-inspect` tunnel | +| `tv-remote press up/down/left/right/select/back`, `back` | `Input.dispatchKeyEvent` with the LG remote codes (Back = 461) | +| `appstate` | the attached app while its page is visible | +| `apps` | `ares-install --listfull` | +| `logs` | console calls, exceptions and browser log entries of the attached page, one line each | +| `snapshot`, `find`, `is`, `get`, `wait` | DOM walk over `Runtime.evaluate`: visible interactive, labelled or text nodes, focus from `document.activeElement`; `-s`, `-d` and `-i` apply to the walk. A control whose centre hit-tests to another element is marked `[covered]` | +| `screenshot` | `Page.captureScreenshot` | +| `click`, `press `, `longpress`, `hover` | `Input.dispatchMouseEvent` (Magic Remote pointer), ending each move with a small wiggle that wakes the cursor | +| `fill` | pointer press on the field, select its contents, then `Input.insertText`, or Backspace for an empty value | +| `type` | `Input.insertText` into the focused element | +| `scroll` | `mouseWheel` at the centre of the scroll container or viewport, by the requested distance | + +Not yet: system buttons (`home`, menu, volume, channel and media keys), `network dump`, `perf`, +`install` and `uninstall`, recording, URLs and deep links in `open`, `power`. + +## Known limitations + +- **Idle leases expire after 60 s by default, and expiry ends the session.** agent-device gives a + direct-device provider lease its default TTL and beats it only while a command runs. When the + lease ends, by `disconnect` or by expiry, the plugin closes the page connection and the + `ares-inspect` tunnel. After a longer pause, the next command fails with `Lease is not active` + or `No active session` and needs a new `connect` and `open`. To avoid it, run + `agent-device daemon stop`, export `AGENT_DEVICE_LEASE_TTL_MS` and + `AGENT_DEVICE_LEASE_MAX_TTL_MS` (for example `1800000`, 30 min), and run the next command in + that shell. That command starts the daemon with both values. +- **`screenshot` captures the app page, not the TV screen.** When the app is in the background, + for example after Home on the remote, the page is hidden and renders no frames, so `screenshot` + fails at once; `open ` brings the app back. A whole-screen capture is not reachable on a retail TV: SSAP + `tv/executeOneShot` answers `401 insufficient permissions` to unsigned clients, LG rejects the + public signed manifest (`blacklisted certificate`), and the dev-mode shell (`prisoner`) gets no + answer from `luna-send-pub`. +- **Console messages from before the attach are replayed without object previews**, so objects + in them show as `Object`. Messages logged after the attach show their properties. +- **`open` on a running app can reload it** (webOS relaunch), so the first read may hit a document + without a body. `open` waits for a loaded document, and a snapshot of an unloaded page is empty + rather than an error. `open --relaunch` closes the app first, for a cold start. +- **A second DevTools client on the same page hung in testing.** Close Chrome DevTools and other + inspector clients attached to the app before a session. +- **Snapshots are DOM-derived.** Labels come from `aria-label`, `alt`, `title` or own text; + rects are CSS px, which match screenshot pixels only at `devicePixelRatio` 1 (1080p apps). + The walk does not enter iframes or shadow roots, so overlays rendered there (for example a + dev-server error overlay) appear in screenshots but not in snapshots. +- **`click`, `press`, `fill` and `focus` refuse a control marked `[covered]`.** A tap at its + centre would land on the element on top, such as a card's own pressable area. `snapshot -i` + still lists it, and `tv-remote` reaches it through focus. +- **System overlays are outside the page.** The LG menu, on-screen keyboard and other system + layers never appear in `screenshot` or `snapshot`. diff --git a/packages/provider-webos/package.json b/packages/provider-webos/package.json new file mode 100644 index 0000000000..a08fad486a --- /dev/null +++ b/packages/provider-webos/package.json @@ -0,0 +1,49 @@ +{ + "name": "@agent-device/webos", + "version": "0.21.36-dev", + "type": "module", + "description": "LG webOS TV web app plugin for agent-device.", + "files": [ + "dist" + ], + "scripts": { + "build": "tsdown --config tsdown.config.ts", + "prepack": "pnpm build" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + }, + "./agent-device": { + "types": "./src/index.ts", + "default": "./src/index.ts" + }, + "./package.json": "./package.json" + }, + "devDependencies": { + "@agent-device/host-kit": "workspace:*", + "@agent-device/kernel": "workspace:*", + "@agent-device/test-kit": "workspace:*", + "@agent-device/tv-cdp": "workspace:*", + "agent-device": "workspace:*" + }, + "publishConfig": { + "exports": { + ".": { + "import": "./dist/agent-device.mjs" + }, + "./agent-device": { + "import": "./dist/agent-device.mjs" + } + }, + "access": "public" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/callstack/agent-device.git", + "directory": "packages/provider-webos" + }, + "homepage": "https://agent-device.dev/docs/webos" +} diff --git a/packages/provider-webos/src/ares.test.ts b/packages/provider-webos/src/ares.test.ts new file mode 100644 index 0000000000..21312b2fc2 --- /dev/null +++ b/packages/provider-webos/src/ares.test.ts @@ -0,0 +1,222 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { chmodSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import path from 'node:path'; +import { isRequestCanceledError } from '@agent-device/kernel/errors'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { afterEach, test, vi } from 'vitest'; + +import { createAres } from './ares.ts'; + +function fakeAresBin(outputs: Record) { + const bin = mkdtempForTestSync('ares-'); + for (const [name, output] of Object.entries(outputs)) { + const script = path.join(bin, name); + writeFileSync(script, `#!/bin/sh\ncat <<'OUTPUT'\n${output}\nOUTPUT\n`); + chmodSync(script, 0o755); + } + return bin; +} + +test('listDevices returns only TV profiles with their IP and default flag', async () => { + const bin = fakeAresBin({ + 'ares-setup-device': JSON.stringify([ + { profile: 'tv', name: 'living-room', default: true, deviceinfo: { ip: '192.0.2.10' } }, + { profile: 'ose', name: 'signage', default: false, deviceinfo: { ip: '192.0.2.20' } }, + ]), + }); + + const devices = await createAres(bin, process.env).listDevices(); + + assert.deepEqual(devices, [{ name: 'living-room', isDefault: true }]); +}); + +test('listApps reads each app id with its title from ares-install --listfull', async () => { + const bin = fakeAresBin({ + 'ares-install': [ + '[Info] Set target device : living-room', + 'id : com.sdk.ares.hostedapp', + 'title : Ares Hosted App', + 'version : 1.1.0', + '', + 'id : com.example.tvapp', + 'title : Example TV', + 'requiredACG', + '', + 'id : com.example.untitled', + ].join('\n'), + }); + + const apps = await createAres(bin, process.env).forDevice('living-room').listApps(); + + assert.deepEqual(apps, [ + { id: 'com.sdk.ares.hostedapp', name: 'Ares Hosted App' }, + { id: 'com.example.tvapp', name: 'Example TV' }, + { id: 'com.example.untitled', name: 'com.example.untitled' }, + ]); +}); + +test('a missing webOS CLI fails as TOOL_MISSING with a hint naming aresBin', async () => { + const bin = fakeAresBin({}); + + const listing = createAres(bin, process.env).listDevices(); + + await assert.rejects(listing, (error: { code?: string; details?: { hint?: string } }) => { + assert.equal(error.code, 'TOOL_MISSING'); + assert.match(error.details?.hint ?? '', /aresBin/); + return true; + }); +}); + +test('a failing ares command reports its stderr in a COMMAND_FAILED message', async () => { + const bin = mkdtempForTestSync('ares-'); + const script = path.join(bin, 'ares-launch'); + writeFileSync(script, `#!/bin/sh\necho 'ERR! Connection timed out' >&2\nexit 1\n`); + chmodSync(script, 0o755); + + const launch = createAres(bin, process.env).forDevice('living-room').launch('com.example.app'); + + await assert.rejects(launch, (error: { code?: string; message?: string }) => { + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'ares-launch failed: ERR! Connection timed out'); + return true; + }); +}); + +test('a cancelled ares-launch keeps its request-canceled reason', async () => { + const bin = mkdtempForTestSync('ares-'); + const script = path.join(bin, 'ares-launch'); + writeFileSync(script, '#!/bin/sh\nsleep 30\n'); + chmodSync(script, 0o755); + const cancel = new AbortController(); + + const launch = createAres(bin, process.env) + .forDevice('living-room') + .launch('com.example.app', cancel.signal); + cancel.abort(); + + await assert.rejects(launch, isRequestCanceledError); +}); + +const APP_ID = 'com.example.app'; +const cleanups: (() => Promise | void)[] = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +function writeScript(bin: string, name: string, body: string) { + const script = path.join(bin, name); + writeFileSync(script, `#!/bin/sh\n${body}\n`); + chmodSync(script, 0o755); +} + +// Serves the inspector target list the way the ares-inspect tunnel exposes it on localhost. +async function fakeInspector(status = 200) { + const server = createServer((_request, response) => { + response.statusCode = status; + response.end( + JSON.stringify([ + { + title: APP_ID, + url: `file:///media/developer/apps/usr/palm/applications/${APP_ID}/index.html`, + webSocketDebuggerUrl: 'ws://localhost/devtools/page/1', + }, + ]), + ); + }); + server.listen(0); + await once(server, 'listening'); + cleanups.push(() => void server.close()); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + return address.port; +} + +// Prints the inspector port like ares-inspect, records its pid, then stays up as the tunnel. One +// script serves every test, because macOS checks each new executable on its first run. +const TUNNEL_BIN = mkdtempForTestSync('ares-'); +writeScript( + TUNNEL_BIN, + 'ares-inspect', + 'echo $$ > "$PID_FILE"\necho "Application Debugging - http://localhost:$INSPECTOR_PORT"\nexec sleep 30', +); + +function tunnel(port: number) { + const pidFile = path.join(mkdtempForTestSync('ares-tunnel-'), 'ares-inspect.pid'); + const isRunning = () => { + try { + process.kill(Number(readFileSync(pidFile, 'utf8')), 0); + return true; + } catch { + return false; + } + }; + const ares = createAres(TUNNEL_BIN, { + ...process.env, + INSPECTOR_PORT: String(port), + PID_FILE: pidFile, + }); + return { tv: ares.forDevice('living-room'), isRunning }; +} + +test('inspect returns the app page on the port ares-inspect prints, and close ends the tunnel', async () => { + const { tv, isRunning } = tunnel(await fakeInspector()); + + const opened = await tv.inspect(APP_ID); + const ended = new Promise((resolve) => opened.onClose(() => resolve(undefined))); + opened.close(); + await ended; + + assert.equal(opened.target.webSocketDebuggerUrl, 'ws://localhost/devtools/page/1'); + assert.equal(isRunning(), false); +}); + +test('an ares-inspect that exits before the page appears reports its output', async () => { + const bin = mkdtempForTestSync('ares-'); + writeScript(bin, 'ares-inspect', "echo 'ERR! device unreachable'\nexit 1"); + + const inspecting = createAres(bin, process.env).forDevice('living-room').inspect(APP_ID); + + await assert.rejects(inspecting, (error: { code?: string; message?: string }) => { + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'ares-inspect exited: ERR! device unreachable'); + return true; + }); +}); + +test('a missing ares-inspect fails as TOOL_MISSING', async () => { + const inspecting = createAres(mkdtempForTestSync('ares-'), process.env) + .forDevice('living-room') + .inspect(APP_ID); + + await assert.rejects(inspecting, (error: { code?: string }) => error.code === 'TOOL_MISSING'); +}); + +test('no page within the timeout ends the tunnel and names the last inspector failure', async () => { + const port = await fakeInspector(500); + const { tv, isRunning } = tunnel(port); + + const inspecting = tv.inspect(APP_ID, undefined, 1000); + + await assert.rejects(inspecting, (error: { message?: string }) => { + assert.equal( + error.message, + `ares-inspect found no page for ${APP_ID} on living-room within 1000ms; the inspector last failed with: The inspector at localhost:${port} answered 500`, + ); + return true; + }); + await vi.waitFor(() => assert.equal(isRunning(), false), { timeout: 3000 }); +}); + +test('a cancelled inspect ends the tunnel and keeps its request-canceled reason', async () => { + const { tv, isRunning } = tunnel(await fakeInspector(500)); + const cancel = new AbortController(); + + const inspecting = tv.inspect(APP_ID, cancel.signal); + await vi.waitFor(() => assert.equal(isRunning(), true), { timeout: 3000 }); + cancel.abort(); + + await assert.rejects(inspecting, isRequestCanceledError); + await vi.waitFor(() => assert.equal(isRunning(), false), { timeout: 3000 }); +}); diff --git a/packages/provider-webos/src/ares.ts b/packages/provider-webos/src/ares.ts new file mode 100644 index 0000000000..26b78535d6 --- /dev/null +++ b/packages/provider-webos/src/ares.ts @@ -0,0 +1,152 @@ +import path from 'node:path'; + +import { runCmd, runCmdBackground } from '@agent-device/host-kit/command'; +import { Deadline, pollUntilSettled } from '@agent-device/host-kit/retry'; +import { AppError, asAppError, errorMessage } from '@agent-device/kernel/errors'; +import { listTargets, type PageTunnel } from '@agent-device/tv-cdp'; + +export type AresDevice = Readonly<{ + name: string; + isDefault: boolean; +}>; + +type AresSetupEntry = { + profile: string; + name: string; + default: boolean; +}; + +export type HostEnv = Readonly>; + +export function createAres(bin: string | undefined, hostEnv: HostEnv) { + const command = (name: string) => (bin ? path.resolve(bin, name) : name); + const env = bin + ? { ...hostEnv, PATH: [bin, hostEnv.PATH].filter(Boolean).join(path.delimiter) } + : hostEnv; + const ares = async (name: string, args: string[], signal?: AbortSignal) => { + try { + return (await runCmd(command(name), args, { env, timeoutMs: 60_000, signal })).stdout; + } catch (error) { + throw aresError(name, error); + } + }; + + const forDevice = (device: string) => ({ + async launch(appId: string, signal?: AbortSignal) { + await ares('ares-launch', ['-d', device, appId], signal); + }, + + async close(appId: string, signal?: AbortSignal) { + await ares('ares-launch', ['-d', device, '--close', appId], signal); + }, + + async listApps(): Promise<{ id: string; name: string }[]> { + return parseAppList(await ares('ares-install', ['-d', device, '--listfull'])); + }, + + // ares-inspect is a long-lived ssh tunnel that prints its local inspector port once it is up. + // The page target only appears once the app has loaded, so the port alone is not enough. + async inspect(appId: string, signal?: AbortSignal, timeoutMs = 30_000): Promise { + // The tunnel outlives this request, so the signal only cancels the wait for the page. + const { child, wait } = runCmdBackground( + command('ares-inspect'), + ['-d', device, '-a', appId], + { + env, + captureOutput: false, + }, + ); + let output = ''; + let collecting = true; + let spawnError: unknown; + let inspectorError: unknown; + const collect = (chunk: Buffer) => { + if (collecting) output += chunk; + }; + wait.catch((error: unknown) => { + if (child.pid === undefined) spawnError = error; + }); + child.stdout?.on('data', collect); + child.stderr?.on('data', collect); + const close = () => child.kill(); + const appTarget = async (port: number) => { + try { + const targets = await listTargets(port); + return targets.find((target) => target.url.includes(`/${appId}/`)); + } catch (error) { + inspectorError = error; + return undefined; + } + }; + + const { value: target } = await pollUntilSettled({ + deadline: Deadline.fromTimeoutMs(timeoutMs), + intervalMs: 500, + signal, + settled: (found) => found !== undefined, + probe: async () => { + if (spawnError) throw aresError('ares-inspect', spawnError); + if (child.exitCode !== null || child.signalCode !== null) { + throw new AppError('COMMAND_FAILED', `ares-inspect exited: ${output.trim()}`, { + hint: `Check that ${device} is reachable with ares-setup-device and in Developer Mode.`, + }); + } + const port = Number(output.match(/localhost:(\d+)/)?.[1]); + return port ? await appTarget(port) : undefined; + }, + }).catch((error: unknown) => { + close(); + throw error; + }); + if (target) { + collecting = false; + return { target, close, onClose: (listener) => child.once('exit', listener) }; + } + close(); + const lastError = inspectorError + ? `; the inspector last failed with: ${errorMessage(inspectorError)}` + : ''; + throw new AppError( + 'COMMAND_FAILED', + `ares-inspect found no page for ${appId} on ${device} within ${timeoutMs}ms${lastError}`, + { output: output.trim(), hint: `Check that ${appId} is installed and starts on the TV.` }, + ); + }, + }); + + return { + async listDevices(): Promise { + const devices: AresSetupEntry[] = JSON.parse(await ares('ares-setup-device', ['--listfull'])); + return devices + .filter((device) => device.profile === 'tv') + .map((device) => ({ + name: device.name, + isDefault: device.default, + })); + }, + forDevice, + }; +} + +function aresError(name: string, error: unknown): AppError { + const failure = asAppError(error, 'COMMAND_FAILED'); + if (failure.code === 'TOOL_MISSING') { + return new AppError('TOOL_MISSING', `${name} is not installed or not on PATH`, { + hint: 'Install the webOS CLI (npm install -g @webos-tools/cli), or set the plugin option aresBin to the directory that holds it.', + }); + } + if (failure.details?.processExitError !== true) return failure; + const stderr = typeof failure.details?.stderr === 'string' ? failure.details.stderr.trim() : ''; + return new AppError('COMMAND_FAILED', `${name} failed${stderr ? `: ${stderr}` : ''}`, { + hint: 'Check that the TV is on, in Developer Mode, and registered with ares-setup-device.', + }); +} + +// `ares-install --listfull` prints one `key : value` block per app, separated by blank lines. +function parseAppList(output: string) { + return output.split(/\n\s*\n/).flatMap((block) => { + const id = block.match(/^id : (.+)$/m)?.[1]?.trim(); + const title = block.match(/^title : (.+)$/m)?.[1]?.trim(); + return id ? [{ id, name: title ?? id }] : []; + }); +} diff --git a/packages/provider-webos/src/index.test.ts b/packages/provider-webos/src/index.test.ts new file mode 100644 index 0000000000..201d680b6d --- /dev/null +++ b/packages/provider-webos/src/index.test.ts @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; + +import webos from './index.ts'; + +test('the plugin connection leases an Android TV device and passes on the requested TV', () => { + const { connection } = webos(); + + assert.deepEqual(connection.resolve({ flags: { device: 'living-room' } }).profile, { + leaseProvider: 'webos', + platform: 'android', + target: 'tv', + device: 'living-room', + }); + assert.deepEqual(connection.resolve({ flags: {} }).profile, { + leaseProvider: 'webos', + platform: 'android', + target: 'tv', + }); +}); + +test('verify defers the TV and app to session open, naming the plugin default TV', async () => { + const verified = await webos({ device: 'living-room' }).connection.verify({ flags: {} }); + + assert.equal(verified.service, 'LG webOS'); + assert.deepEqual(verified.device, { status: 'deferred', name: 'living-room' }); + assert.deepEqual(verified.app, { status: 'deferred' }); +}); diff --git a/packages/provider-webos/src/index.ts b/packages/provider-webos/src/index.ts new file mode 100644 index 0000000000..7f354cfbc4 --- /dev/null +++ b/packages/provider-webos/src/index.ts @@ -0,0 +1,21 @@ +import type { AgentDevicePlugin } from 'agent-device/plugins'; +import { tvPluginConnection } from '@agent-device/tv-cdp/connection'; + +import { PROVIDER, type WebosOptions } from './provider.ts'; + +function webos(options: WebosOptions = {}) { + return { + apiVersion: 2, + provider: PROVIDER, + credentials: { variables: [] }, + connection: tvPluginConnection({ + provider: PROVIDER, + service: 'LG webOS', + verificationMessage: 'webOS TVs are read from ares-setup-device when a session opens.', + defaultDevice: options.device, + }), + setup: async (host) => (await import('./runtime.ts')).createWebosRuntime(options, host), + } satisfies AgentDevicePlugin; +} + +export { webos as default }; diff --git a/packages/provider-webos/src/keys.ts b/packages/provider-webos/src/keys.ts new file mode 100644 index 0000000000..68004999e2 --- /dev/null +++ b/packages/provider-webos/src/keys.ts @@ -0,0 +1,6 @@ +import { NAVIGATION_KEYS, type RemoteKeys } from '@agent-device/tv-cdp'; + +export const WEBOS_REMOTE_KEYS: RemoteKeys = { + ...NAVIGATION_KEYS, + back: { key: 'GoBack', code: 'BrowserBack', keyCode: 461 }, +}; diff --git a/packages/provider-webos/src/provider.ts b/packages/provider-webos/src/provider.ts new file mode 100644 index 0000000000..6cde4fb3a2 --- /dev/null +++ b/packages/provider-webos/src/provider.ts @@ -0,0 +1,8 @@ +export const PROVIDER = 'webos'; + +export type WebosOptions = Readonly<{ + /** Directory holding the ares-* CLIs when they are not on PATH (e.g. a project's node_modules/.bin). */ + aresBin?: string; + /** ares-setup-device entry to drive when the lease names none; defaults to the ares default device. */ + device?: string; +}>; diff --git a/packages/provider-webos/src/runtime.test.ts b/packages/provider-webos/src/runtime.test.ts new file mode 100644 index 0000000000..90593fa436 --- /dev/null +++ b/packages/provider-webos/src/runtime.test.ts @@ -0,0 +1,129 @@ +import assert from 'node:assert/strict'; +import { chmodSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { AppError } from '@agent-device/kernel/errors'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { afterEach, test } from 'vitest'; + +import { createWebosRuntime } from './runtime.ts'; + +const APP_ID = 'com.example.app'; +const shutdowns: (() => Promise)[] = []; +afterEach(async () => { + for (const shutdown of shutdowns.splice(0)) await shutdown(); + writeFileSync(CALLS, ''); +}); + +function lease(leaseId: string) { + return { + leaseId, + tenantId: 'tenant', + runId: 'run', + backend: 'android-instance' as const, + leaseProvider: 'webos', + createdAt: 0, + heartbeatAt: 0, + expiresAt: Number.MAX_SAFE_INTEGER, + }; +} + +// Two registered TVs, bedroom the ares default. Every ares call is recorded, and ares-inspect fails +// so open stops right after the launch. One script answers for every ares CLI through symlinks, +// because macOS checks each new executable on its first run. +const ARES_BIN = mkdtempForTestSync('ares-'); +const CALLS = path.join(ARES_BIN, 'calls.log'); +const TVS = [ + { profile: 'tv', name: 'living-room', default: false }, + { profile: 'tv', name: 'bedroom', default: true }, +]; +writeFileSync( + path.join(ARES_BIN, 'ares'), + `#!/bin/sh +name=\${0##*/} +echo "$name $*" >> ${CALLS} +case "$name" in + ares-setup-device) echo '${JSON.stringify(TVS)}' ;; + ares-inspect) echo 'ERR! no inspector'; exit 1 ;; +esac +`, +); +chmodSync(path.join(ARES_BIN, 'ares'), 0o755); +for (const name of ['ares-setup-device', 'ares-launch', 'ares-inspect']) { + symlinkSync('ares', path.join(ARES_BIN, name)); +} + +const recordedCalls = () => + readFileSync(CALLS, 'utf8') + .trim() + .split('\n') + .filter((line) => !line.startsWith('ares-setup-device')); + +async function webosRuntime(options: { device?: string } = {}) { + const { runtime } = await createWebosRuntime( + { aresBin: ARES_BIN, ...options }, + { + env: process.env, + createError: (code, message, details) => new AppError(code, message, details), + }, + ); + shutdowns.push(runtime.shutdown); + return runtime; +} + +test('connect leases the ares default TV when neither the flag nor the options name one', async () => { + const runtime = await webosRuntime(); + + const allocated = await runtime.leaseLifecycle.allocate(lease('lease-a')); + + assert.equal(allocated?.device.id, 'webos:bedroom'); +}); + +test('connect leases the TV the --device flag names over the plugin default', async () => { + const runtime = await webosRuntime({ device: 'bedroom' }); + + const allocated = await runtime.leaseLifecycle.allocate(lease('lease-a'), { + flags: { device: 'living-room' }, + }); + + assert.equal(allocated?.device.id, 'webos:living-room'); +}); + +test('connect to a TV ares does not know fails as DEVICE_NOT_FOUND and lists the known TVs', async () => { + const runtime = await webosRuntime({ device: 'attic' }); + + await assert.rejects( + runtime.leaseLifecycle.allocate(lease('lease-a')), + (error: unknown) => + error instanceof AppError && + error.code === 'DEVICE_NOT_FOUND' && + error.message === 'No webOS TV named attic in ares-setup-device' && + JSON.stringify(error.details?.available) === JSON.stringify(['living-room', 'bedroom']), + ); +}); + +test('open --relaunch closes the app, launches it, then opens the inspector on the leased TV', async () => { + const runtime = await webosRuntime(); + const allocated = await runtime.leaseLifecycle.allocate(lease('lease-a')); + assert.ok(allocated); + + await assert.rejects( + runtime.getInteractor(allocated.device)!.open(APP_ID, { terminateRunningApp: true }), + /ares-inspect exited: ERR! no inspector/, + ); + + assert.deepEqual(recordedCalls(), [ + `ares-launch -d bedroom --close ${APP_ID}`, + `ares-launch -d bedroom ${APP_ID}`, + `ares-inspect -d bedroom -a ${APP_ID}`, + ]); +}); + +test('close stops the app with ares-launch --close', async () => { + const runtime = await webosRuntime(); + const allocated = await runtime.leaseLifecycle.allocate(lease('lease-a')); + assert.ok(allocated); + + await runtime.getInteractor(allocated.device)!.close!(APP_ID); + + assert.deepEqual(recordedCalls(), [`ares-launch -d bedroom --close ${APP_ID}`]); +}); diff --git a/packages/provider-webos/src/runtime.ts b/packages/provider-webos/src/runtime.ts new file mode 100644 index 0000000000..3e255421cb --- /dev/null +++ b/packages/provider-webos/src/runtime.ts @@ -0,0 +1,45 @@ +import { createTvRuntime, type TvConnection, type TvRuntimeHost } from '@agent-device/tv-cdp'; + +import { createAres, type HostEnv } from './ares.ts'; +import { WEBOS_REMOTE_KEYS } from './keys.ts'; +import { PROVIDER, type WebosOptions } from './provider.ts'; + +type WebosRuntimeHost = TvRuntimeHost & Readonly<{ env: HostEnv }>; + +export function createWebosRuntime(options: WebosOptions, host: WebosRuntimeHost) { + const ares = createAres(options.aresBin, host.env); + + async function connectTv(flagDevice: string | undefined): Promise { + const requested = flagDevice ?? options.device; + const available = await ares.listDevices(); + const device = requested + ? available.find((candidate) => candidate.name === requested) + : (available.find((candidate) => candidate.isDefault) ?? available[0]); + if (!device) { + throw host.createError( + 'DEVICE_NOT_FOUND', + `No webOS TV ${requested ? `named ${requested} ` : ''}in ares-setup-device`, + { + available: available.map((candidate) => candidate.name), + hint: 'Add the TV with ares-setup-device, or pass --device .', + }, + ); + } + const tv = ares.forDevice(device.name); + return { + name: device.name, + launch: async (appId, { relaunch, signal }) => { + if (relaunch) await tv.close(appId, signal).catch(() => {}); + await tv.launch(appId, signal); + return await tv.inspect(appId, signal); + }, + close: (appId) => tv.close(appId), + listApps: tv.listApps, + }; + } + + return createTvRuntime( + { provider: PROVIDER, label: 'webOS', remoteKeys: WEBOS_REMOTE_KEYS, connectTv }, + host, + ); +} diff --git a/packages/provider-webos/test/package-smoke.mjs b/packages/provider-webos/test/package-smoke.mjs new file mode 100644 index 0000000000..84277d2fe7 --- /dev/null +++ b/packages/provider-webos/test/package-smoke.mjs @@ -0,0 +1,2 @@ +export const provider = 'webos'; +export const args = ['--device', 'living-room']; diff --git a/packages/provider-webos/tsconfig.json b/packages/provider-webos/tsconfig.json new file mode 100644 index 0000000000..7399c8aba4 --- /dev/null +++ b/packages/provider-webos/tsconfig.json @@ -0,0 +1,30 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, + "noEmit": false, + "emitDeclarationOnly": true, + "declaration": true, + "declarationDir": "./dist-types", + "tsBuildInfoFile": "./dist-types/index.tsbuildinfo", + "rootDir": ".", + "paths": { + "agent-device/plugins": ["./node_modules/agent-device/src/sdk/plugins.ts"] + } + }, + "include": ["src"], + "references": [ + { + "path": "../host-kit" + }, + { + "path": "../kernel" + }, + { + "path": "../test-kit" + }, + { + "path": "../tv-cdp" + } + ] +} diff --git a/packages/provider-webos/tsdown.config.ts b/packages/provider-webos/tsdown.config.ts new file mode 100644 index 0000000000..75de008ad3 --- /dev/null +++ b/packages/provider-webos/tsdown.config.ts @@ -0,0 +1,4 @@ +import { TV_PLUGIN_BUNDLE } from '@agent-device/tv-cdp/bundle'; +import { defineConfig } from 'tsdown'; + +export default defineConfig(TV_PLUGIN_BUNDLE); diff --git a/packages/tv-cdp/package.json b/packages/tv-cdp/package.json new file mode 100644 index 0000000000..60eabc77c7 --- /dev/null +++ b/packages/tv-cdp/package.json @@ -0,0 +1,30 @@ +{ + "name": "@agent-device/tv-cdp", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Private package shared by the TV web app plugins: drives an app page over the Chrome DevTools Protocol and serves it as a provider-owned Android TV device.", + "dependencies": { + "@agent-device/capture-kit": "workspace:*", + "@agent-device/contracts": "workspace:*", + "@agent-device/host-kit": "workspace:*", + "@agent-device/kernel": "workspace:*" + }, + "devDependencies": { + "@agent-device/test-kit": "workspace:*" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + }, + "./connection": { + "types": "./src/connection.ts", + "default": "./src/connection.ts" + }, + "./bundle": { + "types": "./src/bundle.ts", + "default": "./src/bundle.ts" + } + } +} diff --git a/packages/tv-cdp/src/app-log-descriptor.test.ts b/packages/tv-cdp/src/app-log-descriptor.test.ts new file mode 100644 index 0000000000..4ed4dd9955 --- /dev/null +++ b/packages/tv-cdp/src/app-log-descriptor.test.ts @@ -0,0 +1,33 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; + +import { appLogDescriptorCodec } from './app-log-descriptor.ts'; + +const codec = appLogDescriptorCodec('webos-cdp', 'webOS'); +const descriptor = { + transport: 'webos-cdp', + tv: 'living-room', + appBundleId: 'com.example.app', + outputPath: '/tmp/app.log', +}; + +test('an encoded app-log descriptor decodes back to the same TV and app', () => { + assert.deepEqual(codec.decode(codec.encode(descriptor)), { status: 'decoded', descriptor }); +}); + +test('a descriptor from another transport or with a missing field is invalid', () => { + const { tv: _tv, ...withoutTv } = descriptor; + const bodies: Parameters[0][] = [ + { ...descriptor, transport: 'tizen-cdp' }, + withoutTv, + { ...descriptor, appBundleId: 7 }, + { ...descriptor, outputPath: null }, + ]; + + for (const body of bodies) { + assert.deepEqual(codec.decode(body), { + status: 'invalid', + message: 'Invalid webOS app-log descriptor', + }); + } +}); diff --git a/packages/tv-cdp/src/app-log-descriptor.ts b/packages/tv-cdp/src/app-log-descriptor.ts new file mode 100644 index 0000000000..d39df16b5b --- /dev/null +++ b/packages/tv-cdp/src/app-log-descriptor.ts @@ -0,0 +1,35 @@ +import { APP_LOG_RESOURCE_KIND } from '@agent-device/contracts/app-log-runtime'; +import type { DurableDescriptorCodec } from '@agent-device/contracts/durable-resource-envelope'; + +type TvAppLogDescriptor = Readonly<{ + transport: string; + tv: string; + appBundleId: string; + outputPath: string; +}>; + +export function appLogDescriptorCodec( + transport: string, + label: string, +): DurableDescriptorCodec { + return Object.freeze({ + resourceKind: APP_LOG_RESOURCE_KIND, + version: 1, + encode: (descriptor) => ({ ...descriptor }), + decode: (body) => + body.transport === transport && + typeof body.tv === 'string' && + typeof body.appBundleId === 'string' && + typeof body.outputPath === 'string' + ? { + status: 'decoded', + descriptor: { + transport, + tv: body.tv, + appBundleId: body.appBundleId, + outputPath: body.outputPath, + }, + } + : { status: 'invalid', message: `Invalid ${label} app-log descriptor` }, + }); +} diff --git a/packages/tv-cdp/src/bundle.ts b/packages/tv-cdp/src/bundle.ts new file mode 100644 index 0000000000..1232c85248 --- /dev/null +++ b/packages/tv-cdp/src/bundle.ts @@ -0,0 +1,20 @@ +type BuildLog = Readonly<{ code?: string; message: string }>; + +/** tsdown options for a TV plugin: one self-contained ESM file with the workspace packages inside. */ +export const TV_PLUGIN_BUNDLE = { + entry: { 'agent-device': 'src/index.ts' }, + outDir: 'dist', + format: 'esm', + platform: 'node', + target: 'es2022', + minify: true, + dts: false, + hash: false, + deps: { alwaysBundle: [/^@agent-device\//] }, + inputOptions: { + onLog(level: string, log: BuildLog, handler: (level: string, log: BuildLog) => void) { + if (log.code === 'UNRESOLVED_IMPORT') throw new Error(log.message); + handler(level, log); + }, + }, +} as const; diff --git a/packages/tv-cdp/src/cdp.ts b/packages/tv-cdp/src/cdp.ts new file mode 100644 index 0000000000..8e38502c29 --- /dev/null +++ b/packages/tv-cdp/src/cdp.ts @@ -0,0 +1,122 @@ +import { AppError } from '@agent-device/kernel/errors'; + +import { createReplyWaiters } from './replies.ts'; + +const REQUEST_TIMEOUT_MS = 20_000; +const CDP_HINT = + 'Close Chrome DevTools and other inspector clients attached to the app, then open it again.'; + +export type CdpTarget = Readonly<{ + title: string; + url: string; + webSocketDebuggerUrl: string; +}>; + +export type CdpSession = Readonly<{ + send>(method: string, params?: Record): Promise; + evaluate(expression: string): Promise; + on(listener: (method: string, params: Record) => void): void; + onClose(listener: () => void): void; + close(): void; +}>; + +type CdpMessage = { + id?: number; + method?: string; + params?: Record; + result?: unknown; + error?: { message: string; code: number }; +}; + +export async function listTargets(port: number): Promise { + const response = await fetch(`http://localhost:${port}/json`, { + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) { + throw new AppError( + 'COMMAND_FAILED', + `The inspector at localhost:${port} answered ${response.status}`, + ); + } + return ((await response.json()) as CdpTarget[]).filter((target) => target.webSocketDebuggerUrl); +} + +export async function connectCdp(target: CdpTarget): Promise { + const socket = new WebSocket(target.webSocketDebuggerUrl); + await new Promise((resolve, reject) => { + const fail = (message: string) => { + clearTimeout(timer); + reject(new AppError('COMMAND_FAILED', message, { hint: CDP_HINT })); + }; + const timer = setTimeout(() => { + socket.close(); + fail(`The TV page at ${target.url} did not open within ${REQUEST_TIMEOUT_MS}ms`); + }, REQUEST_TIMEOUT_MS); + socket.onopen = (event) => { + clearTimeout(timer); + resolve(event); + }; + socket.onerror = () => fail(`Could not open the TV page at ${target.url}`); + }); + + let nextId = 0; + const replies = createReplyWaiters(); + const listeners: ((method: string, params: Record) => void)[] = []; + const closeListeners: (() => void)[] = []; + socket.onmessage = (event) => { + const message = JSON.parse(String(event.data)) as CdpMessage; + if (message.method) { + for (const listener of listeners) listener(message.method, message.params ?? {}); + } else if (message.id !== undefined) { + replies.deliver(String(message.id), message); + } + }; + socket.onclose = () => { + replies.failAll( + new AppError('COMMAND_FAILED', 'The TV page connection closed', { hint: CDP_HINT }), + ); + for (const listener of closeListeners) listener(); + }; + + const send = (method: string, params: Record = {}) => { + const id = ++nextId; + const reply = replies.wait( + String(id), + REQUEST_TIMEOUT_MS, + () => + new AppError('COMMAND_FAILED', `${method} got no answer within ${REQUEST_TIMEOUT_MS}ms`, { + hint: CDP_HINT, + }), + (message) => + message.error + ? { + error: new AppError('COMMAND_FAILED', `${method} failed: ${message.error.message}`, { + cdpErrorCode: message.error.code, + }), + } + : { value: message.result as T }, + ); + socket.send(JSON.stringify({ id, method, params })); + return reply; + }; + + return { + send, + async evaluate(expression: string) { + const { result, exceptionDetails } = await send<{ + result: { value?: T }; + exceptionDetails?: { exception?: { description?: string }; text: string }; + }>('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }); + if (exceptionDetails) { + throw new AppError( + 'COMMAND_FAILED', + `Page script failed: ${exceptionDetails.exception?.description ?? exceptionDetails.text}`, + ); + } + return result.value as T; + }, + on: (listener) => listeners.push(listener), + onClose: (listener) => closeListeners.push(listener), + close: () => socket.close(), + }; +} diff --git a/packages/tv-cdp/src/connection.ts b/packages/tv-cdp/src/connection.ts new file mode 100644 index 0000000000..cbc13c6a90 --- /dev/null +++ b/packages/tv-cdp/src/connection.ts @@ -0,0 +1,40 @@ +type ConnectionFlags = Readonly<{ flags: Readonly<{ device?: string }> }>; + +/** Plugin `connection` for a TV whose device is read from vendor tooling when a session opens. */ +export function tvPluginConnection(options: { + provider: string; + service: string; + verificationMessage: string; + defaultDevice?: string; +}) { + const { provider, service, verificationMessage, defaultDevice } = options; + return { + capabilities: { + leaseKind: 'direct-device-provider', + requiresAppAttachment: false, + requiresRemoteDaemon: false, + supportsArtifacts: false, + supportsDeferredAppSelection: true, + supportsDirectPortReverse: false, + usesCloudWebDriverLease: false, + }, + resolve: ({ flags }: ConnectionFlags) => ({ + profile: { + leaseProvider: provider, + platform: 'android' as const, + target: 'tv' as const, + ...(flags.device ? { device: flags.device } : {}), + }, + }), + verify: async ({ flags }: ConnectionFlags) => { + const device = flags.device ?? defaultDevice; + return { + provider, + service, + verificationMessage, + device: { status: 'deferred' as const, ...(device ? { name: device } : {}) }, + app: { status: 'deferred' as const }, + }; + }, + } as const; +} diff --git a/packages/tv-cdp/src/index.ts b/packages/tv-cdp/src/index.ts new file mode 100644 index 0000000000..e290209291 --- /dev/null +++ b/packages/tv-cdp/src/index.ts @@ -0,0 +1,12 @@ +import type { TvRuntimeHost } from './runtime.ts'; +import type { TvPlatform } from './tv.ts'; + +export { listTargets } from './cdp.ts'; +export { NAVIGATION_KEYS, type RemoteKeys } from './keys.ts'; +export type { TvRuntimeHost } from './runtime.ts'; +export type { PageTunnel, TvConnection } from './tv.ts'; + +// The runtime pulls in most of capture-kit, so it loads on first use to keep this entry light. +export async function createTvRuntime(platform: TvPlatform, host: TvRuntimeHost) { + return (await import('./runtime.ts')).createTvRuntime(platform, host); +} diff --git a/packages/tv-cdp/src/keys.test.ts b/packages/tv-cdp/src/keys.test.ts new file mode 100644 index 0000000000..f4c1cc44ef --- /dev/null +++ b/packages/tv-cdp/src/keys.test.ts @@ -0,0 +1,12 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; + +import { keyEvents, NAVIGATION_KEYS } from './keys.ts'; + +test('select sends Enter with text so a focused button receives a click', () => { + const [down, up] = keyEvents(NAVIGATION_KEYS.select); + + assert.equal(down!.type, 'keyDown'); + assert.equal(down!.text, '\r'); + assert.equal(up!.type, 'keyUp'); +}); diff --git a/packages/tv-cdp/src/keys.ts b/packages/tv-cdp/src/keys.ts new file mode 100644 index 0000000000..75e01b516e --- /dev/null +++ b/packages/tv-cdp/src/keys.ts @@ -0,0 +1,38 @@ +import type { TvRemoteButton } from '@agent-device/contracts/tv-remote'; + +export type KeyDefinition = Readonly<{ + key: string; + code: string; + keyCode: number; + text?: string; +}>; + +/** Remote buttons the page receives as key events over CDP. */ +export type RemoteKeys = Readonly>>; + +// Enter carries text because without it Chromium fires no keypress, so a focused