From e70b2bbcb774edd34518833cc57c9cca7f38d5b3 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Thu, 8 Oct 2026 20:03:17 +0200 Subject: [PATCH 1/6] fix(ios): restart a stale simulator accessibility bridge CoreSimulatorBridge can lose its accessibility connection after backboardd and SpringBoard restart, so describe-ui reports no data for every foreground app and annotations cannot highlight elements. Restart the bridge and retry once, at most every 10 seconds per device, and count restart outcomes. --- docs/architecture.md | 2 +- docs/telemetry.md | 2 ++ src/server/baguette.ts | 49 +++++++++++++++++++++++++++++++++- src/server/inspection-tools.ts | 2 +- src/server/plugin.ts | 2 +- src/server/telemetry.ts | 6 +++++ tests/plugin.test.ts | 28 +++++++++++++++++++ 7 files changed, 87 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d09ed741..5cdc534a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -48,7 +48,7 @@ An iOS JPEG decode error restarts capture through `mobile_stream_reset` with the iOS boot and shutdown calls run in order for each device and wait up to two minutes for the reported state. Boot skips the backend route if the device already runs or is booting. This avoids a second boot and Baguette's input repair on a running device. Shutdown closes that device's panel streams after the device stops. -On macOS 27 with Xcode 27, the bundled Baguette can list simulators and capture frames. Device Hub can stop taps, buttons, and keys from reaching an iOS 27 device. Connections await a fresh status check. Gestures use the last completed result while one expired check refreshes in the background, so the approximately 150–200 ms command no longer delays every second of input. A new block becomes visible when that refresh completes. When blocked, it shows a notice. Ask Codex to repair input with `mobile_repair_input`. The tool runs the bundled Baguette's `heal` command. Reconnect capture afterward to use new input handles. It restarts backboardd and SpringBoard without rebooting the device. Relaunching Device Hub can block input again. The panel automatically repairs a confirmed Device Hub block when connecting and reconnects if input becomes blocked later. Repair closes running apps. It limits automatic attempts to once per device per minute; a repeated block or failed repair remains visible in the bottom bar. Model input alone does not trigger automatic repair. Baguette's boot route also repairs input after boot. Do not run the repair to diagnose video. See [Baguette's Device Hub notes](https://github.com/tddworks/baguette/blob/main/docs/features/device-hub/README.md). +On macOS 27 with Xcode 27, the bundled Baguette can list simulators and capture frames. Device Hub can stop taps, buttons, and keys from reaching an iOS 27 device. Connections await a fresh status check. Gestures use the last completed result while one expired check refreshes in the background, so the approximately 150–200 ms command no longer delays every second of input. A new block becomes visible when that refresh completes. When blocked, it shows a notice. Ask Codex to repair input with `mobile_repair_input`. The tool runs the bundled Baguette's `heal` command. Reconnect capture afterward to use new input handles. It restarts backboardd and SpringBoard without rebooting the device. CoreSimulatorBridge can afterwards report no accessibility data for every foreground app until it restarts. UI reads that report no accessibility data restart the bridge and retry once, at most every 10 seconds per device. Relaunching Device Hub can block input again. The panel automatically repairs a confirmed Device Hub block when connecting and reconnects if input becomes blocked later. Repair closes running apps. It limits automatic attempts to once per device per minute; a repeated block or failed repair remains visible in the bottom bar. Model input alone does not trigger automatic repair. Baguette's boot route also repairs input after boot. Do not run the repair to diagnose video. See [Baguette's Device Hub notes](https://github.com/tddworks/baguette/blob/main/docs/features/device-hub/README.md). UI resource addresses include the release version so Codex can load new HTML after an update. The original simulator and workspace addresses and the old v1 through v6 simulator addresses still return the current UI. After updating, restart Codex once if it still uses an older MCP process. diff --git a/docs/telemetry.md b/docs/telemetry.md index 03d4a562..f3c825c9 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -2,6 +2,8 @@ [Back to README](../README.md) · [Contributing](../CONTRIBUTING.md) +Since 0.1.139, `ios.accessibility_bridge.restarts` counts restarts of the simulator's CoreSimulatorBridge after a describe-UI read reported no accessibility data, with a fixed `outcome` (`recovered`, `unavailable` or `failed`). Failed restarts also report a handled `simulator.accessibility_bridge` error. The existing `inspection.tool` errors, sampled MCP traces and `ui.annotations.inspection` timings cover the retried read. No device identifiers, accessibility content or command output are sent. + Since 0.1.134, physical iOS capture runs in the shared `ios-mirror-service` Node process. Existing native packet-processing, input acknowledgement, connection, resource, and crash instrumentation stays attached to the active capture. Native resource measurements now overlap that service's Node process rather than each chat's MCP process. The service uses centralized server initialization, its packaged development/release environment, and server-owned anonymous installation/process-session identity. UI decode/render timings and sampled MCP operations retain their existing boundaries. Subscriber keyframe requests no longer invalidate the shared native queue or release another panel’s touch. The native packet-processing and recovery measurements remain on this active path; the rebuilt addon retains matching debug symbols. `ios.mirror.shared.captures` and `ios.mirror.shared.subscribers` report active counts. `ios.mirror.shared.queue_dropped` counts frames discarded by subscriber queue overflow. Counts aggregate in 30-second windows and flush on service shutdown, using the simulator surface and physical iOS context. No per-frame events, device identifiers, subscription IDs, payloads, or paths are sent. The service clears its collection timer on shutdown and honors telemetry opt-out. diff --git a/src/server/baguette.ts b/src/server/baguette.ts index 0024d3a8..ca475940 100644 --- a/src/server/baguette.ts +++ b/src/server/baguette.ts @@ -14,6 +14,19 @@ import { definitionDeviceState, getDefinitionDiagnostic, parseDefinitionDiagnost import { SimulatorUnavailableError } from "./simulator-unavailable.ts"; import { ExpectedOperationError } from "../shared/error-reporting.ts"; import { baguetteEnvironment } from "./baguette-runtime.ts"; +import { captureServerError, recordAccessibilityBridgeRestart } from "./telemetry.ts"; + +type DescribedUi = { tree?: { label?: unknown } }; +const NO_ACCESSIBILITY_DATA = "no accessibility data"; +const BRIDGE_RESTART_INTERVAL = 10000; + +// CoreSimulatorBridge can lose its accessibility connection after backboardd and SpringBoard +// restart, then report no data for every foreground app until it restarts. +async function kickstartAccessibilityBridge(udid: string, signal: AbortSignal): Promise { + await promisify(execFile)("/usr/bin/xcrun", ["simctl", "spawn", udid, "launchctl", "kickstart", "-k", "user/foreground/com.apple.CoreSimulator.bridge"], { + timeout: 10000, maxBuffer: 64 * 1024, encoding: "utf8", signal, + }); +} export const definitionSchema = z.object({ identity: z.object({ udid: udidSchema, name: z.string(), model: z.string() }), @@ -36,10 +49,13 @@ export class Baguette { private diagnostics = ""; private disposed = false; private readonly embedded: boolean; + private readonly kickstartBridge: (udid: string, signal: AbortSignal) => Promise; + private readonly bridgeRestarts = new Map(); - constructor(baseUrl?: string) { + constructor(baseUrl?: string, kickstartBridge = kickstartAccessibilityBridge) { this.embedded = baseUrl == null; this.baseUrl = parseBaseUrl(baseUrl ?? "http://127.0.0.1:0"); + this.kickstartBridge = kickstartBridge; } async json(path: string, options: RequestInit = {}, timeout = 10000): Promise { @@ -63,6 +79,37 @@ export class Baguette { return payload; } + async describeUi(udid: string): Promise { + const path = `/simulators/${udidSchema.parse(udid)}/describe-ui.json`; + try { return await this.json(path) as DescribedUi; } + catch (error) { + if (!(error instanceof Error) || error.message !== NO_ACCESSIBILITY_DATA || !await this.restartAccessibilityBridge(udid)) throw error; + } + try { + const result = await this.json(path) as DescribedUi; + recordAccessibilityBridgeRestart("recovered"); + return result; + } catch (error) { + recordAccessibilityBridgeRestart("unavailable"); + throw error; + } + } + + // A foreground app with no data usually means a stale bridge. Restart it at most every 10 s per device. + private async restartAccessibilityBridge(udid: string): Promise { + if (Date.now() - (this.bridgeRestarts.get(udid) ?? -Infinity) < BRIDGE_RESTART_INTERVAL) return false; + this.bridgeRestarts.set(udid, Date.now()); + try { + await this.kickstartBridge(udid, this.lifecycle.signal); + return true; + } catch (error) { + if (this.lifecycle.signal.aborted) throw error; + recordAccessibilityBridgeRestart("failed"); + captureServerError(error, "simulator.accessibility_bridge"); + return false; + } + } + async status(signal?: AbortSignal): Promise { if (this.embedded && !this.child) { return { connected: false, managed: false, baseUrl: this.baseUrl.origin, devices: [], error: "The bundled simulator backend has not started." }; diff --git a/src/server/inspection-tools.ts b/src/server/inspection-tools.ts index 4d710d80..43c067ae 100644 --- a/src/server/inspection-tools.ts +++ b/src/server/inspection-tools.ts @@ -31,7 +31,7 @@ export function registerInspectionTools(server: McpServer, baguette: Baguette, a if (platform === "ios") { const device = await baguette.device(udidSchema.parse(deviceId), true); deviceName = device.name; - const response = await baguette.json(`/simulators/${deviceId}/describe-ui.json`); + const response = await baguette.describeUi(deviceId); native = response; appName = typeof response.tree?.label === "string" ? response.tree.label : undefined; } else { diff --git a/src/server/plugin.ts b/src/server/plugin.ts index 924b0676..4bf0fd3b 100644 --- a/src/server/plugin.ts +++ b/src/server/plugin.ts @@ -318,7 +318,7 @@ export async function createPlugin(html: string | (() => Promise), b inputSchema: deviceInput, annotations: read, }, guarded(async ({ udid }: { udid: string }) => { await baguette.device(udid, true); - const tree = await baguette.json(`/simulators/${udid}/describe-ui.json`); + const tree = await baguette.describeUi(udid); return result({ udid, tree }, JSON.stringify(tree)); })); diff --git a/src/server/telemetry.ts b/src/server/telemetry.ts index a2cd9cdd..171af7fb 100644 --- a/src/server/telemetry.ts +++ b/src/server/telemetry.ts @@ -26,6 +26,12 @@ export function recordPluginUpdate(operation: "check" | "install", outcome: "dis Sentry.metrics.gauge("plugin.update.duration", duration, { unit: "millisecond", attributes }); } +export function recordAccessibilityBridgeRestart(outcome: "recovered" | "unavailable" | "failed") { + if (process.env.MOBILE_DEV_TELEMETRY === "off") return; + const attributes = { component: "server", device_platform: "ios", device_kind: "simulator", outcome }; + Sentry.metrics.count("ios.accessibility_bridge.restarts", 1, { attributes }); +} + export function recordAndroidBackendStartup(duration: number, outcome: "ready" | "failed") { if (process.env.MOBILE_DEV_TELEMETRY === "off") return; const attributes = { component: "server", surface: "simulator", device_platform: "android", outcome }; diff --git a/tests/plugin.test.ts b/tests/plugin.test.ts index 7329e1e0..d7dad3e7 100644 --- a/tests/plugin.test.ts +++ b/tests/plugin.test.ts @@ -1,5 +1,7 @@ import { SENTRY_ORIGIN } from "../src/shared/telemetry.ts"; import { setImmediate } from "node:timers/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; import { SimulatorInputService } from "../src/server/simulator-input.ts"; import test from "node:test"; import assert from "node:assert/strict"; @@ -379,6 +381,32 @@ test("a reused backend survives disposal of the adapter", async t => { assert.equal((await fetch(`${fake.url}/simulators.json`)).status, 200); }); +test("a stale accessibility bridge restarts once before reading the UI again", async t => { + let now = 0; + t.mock.method(Date, "now", () => now); + let responses: unknown[] = []; + const server = createServer((_request, response) => response.end(JSON.stringify(responses.shift() ?? { ok: false, error: "no accessibility data" }))); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + t.after(() => server.close()); + const restarts: string[] = []; + const baguette = new Baguette(`http://127.0.0.1:${(server.address() as AddressInfo).port}`, async udid => { restarts.push(udid); }); + t.after(() => baguette.dispose()); + + responses = [{ ok: false, error: "no accessibility data" }, { ok: true, tree: { label: "HorseTinder" } }]; + assert.equal((await baguette.describeUi(UDID)).tree?.label, "HorseTinder"); + assert.deepEqual(restarts, [UDID]); + + // A screen with genuinely no data does not restart the bridge on every read. + now = 5000; + await assert.rejects(baguette.describeUi(UDID), /no accessibility data/); + assert.equal(restarts.length, 1); + + now = 20000; + responses = [{ ok: false, error: "describe failed" }]; + await assert.rejects(baguette.describeUi(UDID), /describe failed/); + assert.equal(restarts.length, 1); +}); + test("panel input reaches the native socket during a pending refresh and blocks after it reports Device Hub", { timeout: 2000 }, async t => { let now = 0; t.mock.method(Date, "now", () => now); From 77df530f0b4aa8bd34659ed33a31b554f42574d0 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Thu, 8 Oct 2026 20:03:33 +0200 Subject: [PATCH 2/6] fix(annotations): match Expo Go Metro targets by foreground bundle ID Target matching compared the accessibility app name with the last bundle ID segment, so Expo Go (host.exp.Exponent) never matched and annotations fell back to coarse accessibility elements. When no name matches on the device, look up the iOS simulator's foreground bundle ID and match it exactly. Split native snapshots by platform and read Android aliases in parallel with accessibility. --- docs/architecture.md | 2 +- src/server/inspection-tools.ts | 68 +++++++++++++++++++--------- src/server/metro-logs.ts | 2 +- src/server/react-native-inspector.ts | 25 ++++++---- tests/react-native-inspector.test.ts | 23 ++++++++++ 5 files changed, 88 insertions(+), 32 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 5cdc534a..00bfaf57 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -64,7 +64,7 @@ Send to chat shows only "Apply these annotations." Edit guidance stays in assist Screen annotations use the device's accessibility tree for native and React Native apps. Click an exposed element to add a note. Clicks and manual regions work while inspection loads; an open point note adopts the real element when its bounds arrive. If runtime inspection fails, the panel retries the native accessibility tool. The note popup can select an enclosing element when the tree includes one; flat Android snapshots offer elements whose reported bounds contain the selection. Drag to mark a region when the app does not expose a view. Region annotations carry user-selected bounds, not inferred component names. React Native development apps can also supply runtime component names and measured native bounds through the MCP server. The panel makes no browser connection to Metro. The server uses an existing Metro server at `http://127.0.0.1:8081`, requires a unique device/app match and support for multiple debuggers, and closes its connection after each snapshot. It does not start Metro or take over another debugger. The `mobile_inspect_ui` tool accepts `metroUrl` and `targetId` for explicit targets. -The React Native adapter reads the DevTools hook to locate mounted components, then uses the documented native element `getBoundingClientRect()` API. Native screen containers with empty DOM bounds fall back to `measureInWindow()` callbacks through a temporary CDP binding. The server removes that binding and its global function before closing the connection. Logical component bounds cover their measured native children. +The React Native adapter reads the DevTools hook to locate mounted components, then uses the documented native element `getBoundingClientRect()` API. Native screen containers with empty DOM bounds fall back to `measureInWindow()` callbacks through a temporary CDP binding. The server removes that binding and its global function before closing the connection. Logical component bounds cover their measured native children. Automatic inspection picks the Metro target on the selected device whose bundle ID ends with the accessibility app name. When that finds none, such as Expo Go (`host.exp.Exponent`), iOS simulators look up the foreground app's bundle ID and match it exactly. A backgrounded React Native app never stands in for another foreground app. Traversal uses a work queue so deeply nested navigator wrappers do not cut off visible rows. Native-stack screens marked inactive through `activityState` and `aria-hidden` are excluded, along with disconnected native elements. Decorative elements marked only `aria-hidden` remain selectable. Text labels use string children when available, and text outlines follow the component's layout bounds. diff --git a/src/server/inspection-tools.ts b/src/server/inspection-tools.ts index 43c067ae..beb770b9 100644 --- a/src/server/inspection-tools.ts +++ b/src/server/inspection-tools.ts @@ -6,6 +6,8 @@ import { z } from "zod"; import type { Baguette } from "./baguette.ts"; import type { ServeEmu } from "./serve-emu.ts"; import { inspectReactNative } from "./react-native-inspector.ts"; +import type { InspectorRequest } from "./react-native-inspector.ts"; +import { readDeviceApps } from "./device-apps/sources.ts"; import { captureServerError } from "./telemetry.ts"; import { errorMessage, udidSchema } from "../shared/protocol.ts"; import { screenComponents } from "../shared/screen-annotations.ts"; @@ -13,6 +15,47 @@ import { adbPath } from "./native-logs.ts"; const execute = promisify(execFile); +type NativeSnapshot = { + native: unknown; + app: Pick; +}; + +async function iosSnapshot(baguette: Baguette, deviceId: string, signal: AbortSignal): Promise { + const udid = udidSchema.parse(deviceId); + const device = await baguette.device(udid, true); + const response = await baguette.describeUi(udid); + return { + native: response, + app: { + deviceName: device.name, + appName: typeof response.tree?.label === "string" ? response.tree.label : undefined, + resolveForegroundAppId: async () => (await readDeviceApps(udid, signal, "ios", "simulator")).foregroundApp?.bundleId ?? undefined, + }, + }; +} + +async function androidSnapshot(android: ServeEmu, deviceId: string, deviceName: string, screenWidth: number): Promise { + const [response, deviceAliases] = await Promise.all([android.accessibility(deviceId), androidAliases(deviceId)]); + const packages = new Set((response.tree.nodes ?? []).map((item: { packageName?: string }) => item.packageName).filter((name: unknown): name is string => typeof name === "string" && !name.startsWith("com.android.") && name !== "android")); + return { + native: screenComponents(response.tree, screenWidth / response.screen.width), + app: { deviceName, deviceAliases, appId: packages.size === 1 ? [...packages][0] : undefined }, + }; +} + +// Metro names Android targets by model and release rather than the backend's device name. +async function androidAliases(deviceId: string): Promise { + try { + const adb = await adbPath(); + const properties = await Promise.all(["ro.product.model", "ro.build.version.release", "ro.build.version.sdk"].map(property => + execute(adb, ["-s", deviceId, "shell", "getprop", property], { timeout: 3000, maxBuffer: 4096 }).then(result => result.stdout.trim()))); + return properties.every(Boolean) ? [properties[0], `${properties[0]} - ${properties[1]} - API ${properties[2]}`] : []; + } catch { + // The backend's device name remains available for matching. + return []; + } +} + export function registerInspectionTools(server: McpServer, baguette: Baguette, android: ServeEmu) { const controller = new AbortController(); registerAppTool(server, "mobile_inspect_ui", { @@ -26,29 +69,12 @@ export function registerInspectionTools(server: McpServer, baguette: Baguette, a _meta: { ui: { visibility: ["app", "model"] } }, }, async ({ platform, deviceId, deviceName, screenWidth, metroUrl, targetId }) => { try { - let deviceAliases: string[] = []; - let native: unknown, appName: string | undefined, appId: string | undefined; - if (platform === "ios") { - const device = await baguette.device(udidSchema.parse(deviceId), true); - deviceName = device.name; - const response = await baguette.describeUi(deviceId); - native = response; - appName = typeof response.tree?.label === "string" ? response.tree.label : undefined; - } else { - const response = await android.accessibility(deviceId); - native = screenComponents(response.tree, screenWidth / response.screen.width); - const packages = new Set((response.tree.nodes ?? []).map((item: { packageName?: string }) => item.packageName).filter((name: unknown): name is string => typeof name === "string" && !name.startsWith("com.android.") && name !== "android")); - if (packages.size === 1) appId = [...packages][0]; - try { - const adb = await adbPath(); - const properties = await Promise.all(["ro.product.model", "ro.build.version.release", "ro.build.version.sdk"].map(property => - execute(adb, ["-s", deviceId, "shell", "getprop", property], { timeout: 3000, maxBuffer: 4096 }).then(result => result.stdout.trim()))); - if (properties.every(Boolean)) deviceAliases = [properties[0], `${properties[0]} - ${properties[1]} - API ${properties[2]}`]; - } catch { /* The backend's device name remains available for matching. */ } - } + const { native, app } = platform === "ios" + ? await iosSnapshot(baguette, deviceId, controller.signal) + : await androidSnapshot(android, deviceId, deviceName, screenWidth); let runtime: Awaited>; try { - runtime = await inspectReactNative({ url: metroUrl, targetId, deviceName, deviceAliases, appName, appId, platform, screenWidth }, controller.signal); + runtime = await inspectReactNative({ ...app, url: metroUrl, targetId, platform, screenWidth }, controller.signal); } catch { // Metro is optional. Native apps, absent servers and reloads retain AX selection. runtime = { available: false, reason: "inspector-unavailable" }; diff --git a/src/server/metro-logs.ts b/src/server/metro-logs.ts index 41fa82c5..e3442d04 100644 --- a/src/server/metro-logs.ts +++ b/src/server/metro-logs.ts @@ -5,7 +5,7 @@ import { parseBaseUrl, errorMessage } from "../shared/protocol.ts"; import { parseMetroEvent } from "./log-parsers.ts"; import type { LogSink, StopLogSource } from "./native-logs.ts"; -type InspectorTarget = MetroTarget & { webSocketDebuggerUrl: string; supportsMultipleDebuggers: boolean }; +export type InspectorTarget = MetroTarget & { webSocketDebuggerUrl: string; supportsMultipleDebuggers: boolean }; export async function metroTargets(origin: string, signal?: AbortSignal): Promise { const base = parseBaseUrl(origin, "Metro URL"); diff --git a/src/server/react-native-inspector.ts b/src/server/react-native-inspector.ts index 4a5769ca..88e9e0d0 100644 --- a/src/server/react-native-inspector.ts +++ b/src/server/react-native-inspector.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { WebSocket } from "ws"; import { z } from "zod"; import { metroTargets } from "./metro-logs.ts"; +import type { InspectorTarget } from "./metro-logs.ts"; import { collectReactNativeTree } from "./react-native-snapshot.js"; import type { ScreenBounds, ReactElementContext } from "../shared/screen-annotations.ts"; import { resolveReactNativeSources } from "./react-native-source.ts"; @@ -14,19 +15,25 @@ const stackIds = z.array(z.number().int().min(0).max(511)).max(12).optional(); const node: z.ZodType = z.lazy(() => z.object({ source: z.literal("react-native"), role: z.string().max(256), label: z.string().max(256).optional(), identifier: z.string().max(256).optional(), frame, nodeId: z.string().max(256).optional(), parentId: z.string().max(256).optional(), depth: z.number().int().min(0).max(10000).optional(), children: z.array(node).max(3000).default([]), react: react.optional(), creationStackIds: stackIds, ownerStackIds: stackIds })); const creationFrame = z.object({ url: z.number().int().min(0).max(31), line: z.number().int().positive(), column: z.number().int().nonnegative(), methodName: z.string().max(256) }); const snapshot = z.object({ available: z.boolean(), tree: z.array(node).max(3000).optional(), windowWidth: z.number().nonnegative().finite().optional(), truncated: z.boolean().optional(), sourceUrls: z.array(z.string().max(2048)).max(32).default([]), sourceStacks: z.array(z.array(creationFrame).max(3)).max(512).default([]) }); -export type InspectorRequest = { url?: string; targetId?: string; deviceName: string; deviceAliases?: string[]; appName?: string; appId?: string; platform: "ios" | "android"; screenWidth: number }; +export type InspectorRequest = { url?: string; targetId?: string; deviceName: string; deviceAliases?: string[]; appName?: string; appId?: string; resolveForegroundAppId?: () => Promise; platform: "ios" | "android"; screenWidth: number }; const normalized = (value: string) => value.toLowerCase().replace(/[^a-z0-9]/g, ""); +// Never pick another app merely because it has an inspector on this device. +async function appTargets(targets: InspectorTarget[], request: InspectorRequest): Promise { + const names = [request.deviceName, ...(request.deviceAliases ?? [])].map(normalized); + const onDevice = targets.filter(target => target.deviceName && names.includes(normalized(target.deviceName))); + if (request.appId) return onDevice.filter(target => target.appId === request.appId); + const app = normalized(request.appName ?? ""); + const named = onDevice.filter(target => app && target.appId && normalized(target.appId.split(".").at(-1)!) === app); + if (named.length || !onDevice.length || !request.resolveForegroundAppId) return named; + // Display names can differ from bundle IDs, such as Expo Go and host.exp.Exponent. + const appId = await request.resolveForegroundAppId().catch(() => undefined); + return appId ? onDevice.filter(target => target.appId === appId) : []; +} + export async function inspectReactNative(request: InspectorRequest, signal?: AbortSignal) { const targets = await metroTargets(request.url ?? "http://127.0.0.1:8081", signal); - const matches = targets.filter(target => { - if (request.targetId) return target.id === request.targetId; - if (!target.deviceName || ![request.deviceName, ...(request.deviceAliases ?? [])].some(name => normalized(target.deviceName!) === normalized(name))) return false; - if (request.appId) return target.appId === request.appId; - // Never pick another app merely because it has an inspector on this device. - const app = normalized(request.appName ?? ""); - return !!app && !!target.appId && normalized(target.appId.split(".").at(-1)!) === app; - }); + const matches = request.targetId ? targets.filter(target => target.id === request.targetId) : await appTargets(targets, request); if (matches.length !== 1) return { available: false as const, reason: matches.length > 1 ? "ambiguous-target" : "no-matching-target" }; const target = matches[0]; if (!target.supportsMultipleDebuggers) return { available: false as const, reason: "exclusive-debugger" }; diff --git a/tests/react-native-inspector.test.ts b/tests/react-native-inspector.test.ts index cde8c9a9..4f47ab72 100644 --- a/tests/react-native-inspector.test.ts +++ b/tests/react-native-inspector.test.ts @@ -100,6 +100,29 @@ test("automatic inspection refuses ambiguous apps, wrong devices and exclusive d assert.equal(server.calls(), 0); }); +test("a display name that differs from the bundle ID matches the foreground app on the same device", async t => { + const server = await backend(t); + const expo = { ...server.target, id: "expo", appId: "host.exp.Exponent" }; + const otherDevice = { ...expo, id: "android", deviceName: "Pixel 9" }; + server.setTargets([expo, otherDevice]); + let lookups = 0; + const resolveForegroundAppId = async () => { lookups++; return "host.exp.Exponent"; }; + assert.equal((await inspectReactNative({ ...request, appName: "Expo Go", url: server.origin })).available, false); + assert.equal((await inspectReactNative({ ...request, appName: "Expo Go", resolveForegroundAppId, url: server.origin })).available, true); + assert.equal(lookups, 1); + + // A backgrounded React Native app never stands in for a different foreground app. + const native = async () => "com.example.native"; + assert.equal((await inspectReactNative({ ...request, appName: "Expo Go", resolveForegroundAppId: native, url: server.origin })).available, false); + const failing = async () => { throw new Error("foreground unavailable"); }; + assert.equal((await inspectReactNative({ ...request, appName: "Expo Go", resolveForegroundAppId: failing, url: server.origin })).available, false); + + // Name matches skip the lookup. + server.setTargets([server.target]); + assert.equal((await inspectReactNative({ ...request, resolveForegroundAppId, url: server.origin })).available, true); + assert.equal(lookups, 1); +}); + test("Android runtime bounds convert from DIPs to screen pixels", async t => { const server = await backend(t); const result = await inspectReactNative({ ...request, url: server.origin, platform: "android", appId: server.target.appId, screenWidth: 1200 }); From add1c80ae3df3ff0b3ded13b276617d062966309 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Thu, 8 Oct 2026 20:03:52 +0200 Subject: [PATCH 3/6] fix(annotations): skip React Native elements hidden under later content Hit testing preferred the smallest element at a point regardless of paint order, so a card stack's slightly inset next card won over the visible card and highlights appeared offset. Image, text or SVG painted later in another branch now hides earlier React Native elements at that point. Index each component list once for ancestor lookups. --- docs/architecture.md | 2 +- src/shared/screen-annotations.ts | 37 ++++++++++++++++++++++++++------ tests/screen-annotations.test.ts | 26 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 00bfaf57..61b1470c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -64,7 +64,7 @@ Send to chat shows only "Apply these annotations." Edit guidance stays in assist Screen annotations use the device's accessibility tree for native and React Native apps. Click an exposed element to add a note. Clicks and manual regions work while inspection loads; an open point note adopts the real element when its bounds arrive. If runtime inspection fails, the panel retries the native accessibility tool. The note popup can select an enclosing element when the tree includes one; flat Android snapshots offer elements whose reported bounds contain the selection. Drag to mark a region when the app does not expose a view. Region annotations carry user-selected bounds, not inferred component names. React Native development apps can also supply runtime component names and measured native bounds through the MCP server. The panel makes no browser connection to Metro. The server uses an existing Metro server at `http://127.0.0.1:8081`, requires a unique device/app match and support for multiple debuggers, and closes its connection after each snapshot. It does not start Metro or take over another debugger. The `mobile_inspect_ui` tool accepts `metroUrl` and `targetId` for explicit targets. -The React Native adapter reads the DevTools hook to locate mounted components, then uses the documented native element `getBoundingClientRect()` API. Native screen containers with empty DOM bounds fall back to `measureInWindow()` callbacks through a temporary CDP binding. The server removes that binding and its global function before closing the connection. Logical component bounds cover their measured native children. Automatic inspection picks the Metro target on the selected device whose bundle ID ends with the accessibility app name. When that finds none, such as Expo Go (`host.exp.Exponent`), iOS simulators look up the foreground app's bundle ID and match it exactly. A backgrounded React Native app never stands in for another foreground app. +The React Native adapter reads the DevTools hook to locate mounted components, then uses the documented native element `getBoundingClientRect()` API. Native screen containers with empty DOM bounds fall back to `measureInWindow()` callbacks through a temporary CDP binding. The server removes that binding and its global function before closing the connection. Logical component bounds cover their measured native children. Hit testing follows paint order: image, text or SVG content rendered later in another branch hides earlier React Native elements at that point, so a card stack's next card stays unselectable under the top card. A later plain container does not hide content, because the snapshot has no background information. Automatic inspection picks the Metro target on the selected device whose bundle ID ends with the accessibility app name. When that finds none, such as Expo Go (`host.exp.Exponent`), iOS simulators look up the foreground app's bundle ID and match it exactly. A backgrounded React Native app never stands in for another foreground app. Traversal uses a work queue so deeply nested navigator wrappers do not cut off visible rows. Native-stack screens marked inactive through `activityState` and `aria-hidden` are excluded, along with disconnected native elements. Decorative elements marked only `aria-hidden` remain selectable. Text labels use string children when available, and text outlines follow the component's layout bounds. diff --git a/src/shared/screen-annotations.ts b/src/shared/screen-annotations.ts index 00493d3f..b98becb0 100644 --- a/src/shared/screen-annotations.ts +++ b/src/shared/screen-annotations.ts @@ -77,18 +77,41 @@ export function componentAt(components: ScreenComponent[], point: ScreenPoint, s return componentsAt(components, point, screen)[0]; } +// Hover hit tests run per pointer move, so index each component list once. +type ComponentIndex = { order: Map; byId: Map }; +const componentIndexes = new WeakMap(); +function componentIndex(components: ScreenComponent[]): ComponentIndex { + let index = componentIndexes.get(components); + if (!index) { + index = { order: new Map(components.map((item, position) => [item, position])), byId: new Map(components.flatMap(item => item.nodeId ? [[item.nodeId, item] as const] : [])) }; + componentIndexes.set(components, index); + } + return index; +} + +function ancestorIds(component: ScreenComponent, byId: Map): Set { + const ids = new Set(); + for (let parentId = component.parentId; parentId && !ids.has(parentId); parentId = byId.get(parentId)?.parentId) ids.add(parentId); + return ids; +} + +// React Native paints later branches over earlier ones. Image, text or SVG drawn later in another +// branch hides earlier elements at that point, such as the next card under a card stack. +const PAINTED_CONTENT = /Image|Text|Svg/; +function unoccluded(hits: ScreenComponent[], { order, byId }: ComponentIndex): ScreenComponent[] { + const covers = hits.filter(item => item.source === "react-native" && PAINTED_CONTENT.test(item.role ?? "")).map(cover => ({ order: order.get(cover)!, ancestors: ancestorIds(cover, byId) })); + return hits.filter(item => item.source !== "react-native" || !item.nodeId + || !covers.some(cover => cover.order > order.get(item)! && !cover.ancestors.has(item.nodeId!))); +} + export function componentsAt(components: ScreenComponent[], point: ScreenPoint, screen?: { width: number; height: number }): ScreenComponent[] { - const hits = components.filter(({ bounds: b }) => (!screen || b.width < screen.width * .95 || b.height < screen.height * .95) && point.x >= b.x && point.x < b.x + b.width && point.y >= b.y && point.y < b.y + b.height) + const index = componentIndex(components); + const hits = unoccluded(components.filter(({ bounds: b }) => (!screen || b.width < screen.width * .95 || b.height < screen.height * .95) && point.x >= b.x && point.x < b.x + b.width && point.y >= b.y && point.y < b.y + b.height), index) .sort((a, b) => Number(b.source === "react-native") - Number(a.source === "react-native") || a.bounds.width * a.bounds.height - b.bounds.width * b.bounds.height || b.depth - a.depth); const selected = hits[0]; if (!selected) return []; // Nested trees supply real ancestors. Flat Android snapshots supply only bounds. - const ancestors = new Set(); - let parentId = selected.parentId; - while (parentId && !ancestors.has(parentId)) { - ancestors.add(parentId); - parentId = components.find(item => item.nodeId === parentId)?.parentId; - } + const ancestors = ancestorIds(selected, index.byId); return hits.filter(item => item === selected || (selected.parentId ? ancestors.has(item.nodeId ?? "") : item.bounds.x <= selected.bounds.x && item.bounds.y <= selected.bounds.y && item.bounds.x + item.bounds.width >= selected.bounds.x + selected.bounds.width diff --git a/tests/screen-annotations.test.ts b/tests/screen-annotations.test.ts index 25b37d1c..6e80b3a7 100644 --- a/tests/screen-annotations.test.ts +++ b/tests/screen-annotations.test.ts @@ -95,6 +95,32 @@ test("screen-wide containers never win and smaller controls beat deeper containe assert.equal(componentAt(components, { x: 350, y: 500 }, screen), undefined); }); +test("a card painted on top hides the slightly smaller next card under a stack", () => { + const card = (key: string, inset: number) => ({ source: "react-native", role: "Animated(View)", react: { component: "Animated(View)", owners: [], key }, frame: { x: inset, y: inset, width: 400 - inset * 2, height: 600 - inset * 2 }, children: [ + { source: "react-native", role: "RCTImageView", frame: { x: inset, y: inset, width: 400 - inset * 2, height: 600 - inset * 2 }, children: [] }, + { source: "react-native", role: "RCTView", frame: { x: 20 + inset, y: 500, width: 90, height: 30 }, children: [ + { source: "react-native", role: "RCTText", label: `${key} chip`, frame: { x: 30 + inset, y: 505, width: 70, height: 20 }, children: [] }, + ] }, + ] }); + // The next card renders first and is inset, so its elements are smaller and offset. + const components = screenComponents([{ source: "react-native", role: "View", frame: { x: 0, y: 0, width: 400, height: 600 }, children: [card("preview", 8), card("active", 0)] }]); + const screen = { width: 402, height: 874 }; + assert.equal(componentAt(components, { x: 45, y: 515 }, screen)?.label, "active chip"); + assert.equal(componentAt(components, { x: 25, y: 515 }, screen)?.role, "RCTView"); + const enclosing = componentsAt(components, { x: 45, y: 515 }, screen); + assert.deepEqual(enclosing.map(item => item.role), ["RCTText", "RCTView", "Animated(View)", "View"]); + assert.equal(enclosing[2].react?.key, "active"); + assert.equal(componentAt(components, { x: 200, y: 300 }, screen)?.role, "RCTImageView"); + assert.ok(componentsAt(components, { x: 200, y: 300 }, screen).every(item => item.react?.key !== "preview")); + + // A later transparent container does not hide content beneath it. + const overlay = screenComponents([{ source: "react-native", role: "View", frame: { x: 0, y: 0, width: 400, height: 600 }, children: [ + { source: "react-native", role: "RCTText", label: "Under overlay", frame: { x: 10, y: 10, width: 100, height: 20 }, children: [] }, + { source: "react-native", role: "RCTView", frame: { x: 0, y: 0, width: 200, height: 200 }, children: [] }, + ] }]); + assert.equal(componentAt(overlay, { x: 20, y: 15 }, screen)?.label, "Under overlay"); +}); + test("a sparse accessibility tree supports explicit regions without pixel guesses", async () => { const f = fixture(); f.store.readTree = async () => ({ role: "AXApplication", frame: { x: 0, y: 0, width: 393, height: 852 } }); From c0717a750f1a9410b60cc92ae9a97415447b0e96 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Thu, 8 Oct 2026 20:04:25 +0200 Subject: [PATCH 4/6] fix(annotations): map bounds with the accessibility screen size iOS bezel definitions can describe a smaller screen cutout than the device, such as 400x872 points for a 402x874 iPhone, which stretched highlights toward the bottom right. Inspection now returns the accessibility root's screen size through a typed tree result, and the store rescales the capture and pending pointer state to it. Drags keep their latest point when inspection finishes mid-drag. --- docs/architecture.md | 2 +- src/server/inspection-tools.ts | 9 +++-- src/shared/screen-annotations.ts | 21 ++++++++--- src/ui/screen-annotations.ts | 31 +++++++++++++--- src/ui/simulator-panel.ts | 18 +++++----- tests/screen-annotations-ui.test.ts | 2 +- tests/screen-annotations.test.ts | 55 +++++++++++++++++++++++------ tests/ui-telemetry.test.ts | 4 +-- 8 files changed, 106 insertions(+), 36 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 61b1470c..bc019cc4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -64,7 +64,7 @@ Send to chat shows only "Apply these annotations." Edit guidance stays in assist Screen annotations use the device's accessibility tree for native and React Native apps. Click an exposed element to add a note. Clicks and manual regions work while inspection loads; an open point note adopts the real element when its bounds arrive. If runtime inspection fails, the panel retries the native accessibility tool. The note popup can select an enclosing element when the tree includes one; flat Android snapshots offer elements whose reported bounds contain the selection. Drag to mark a region when the app does not expose a view. Region annotations carry user-selected bounds, not inferred component names. React Native development apps can also supply runtime component names and measured native bounds through the MCP server. The panel makes no browser connection to Metro. The server uses an existing Metro server at `http://127.0.0.1:8081`, requires a unique device/app match and support for multiple debuggers, and closes its connection after each snapshot. It does not start Metro or take over another debugger. The `mobile_inspect_ui` tool accepts `metroUrl` and `targetId` for explicit targets. -The React Native adapter reads the DevTools hook to locate mounted components, then uses the documented native element `getBoundingClientRect()` API. Native screen containers with empty DOM bounds fall back to `measureInWindow()` callbacks through a temporary CDP binding. The server removes that binding and its global function before closing the connection. Logical component bounds cover their measured native children. Hit testing follows paint order: image, text or SVG content rendered later in another branch hides earlier React Native elements at that point, so a card stack's next card stays unselectable under the top card. A later plain container does not hide content, because the snapshot has no background information. Automatic inspection picks the Metro target on the selected device whose bundle ID ends with the accessibility app name. When that finds none, such as Expo Go (`host.exp.Exponent`), iOS simulators look up the foreground app's bundle ID and match it exactly. A backgrounded React Native app never stands in for another foreground app. +The React Native adapter reads the DevTools hook to locate mounted components, then uses the documented native element `getBoundingClientRect()` API. Native screen containers with empty DOM bounds fall back to `measureInWindow()` callbacks through a temporary CDP binding. The server removes that binding and its global function before closing the connection. Logical component bounds cover their measured native children. Hit testing follows paint order: image, text or SVG content rendered later in another branch hides earlier React Native elements at that point, so a card stack's next card stays unselectable under the top card. A later plain container does not hide content, because the snapshot has no background information. iOS bezel definitions can describe a screen cutout slightly smaller than the device, such as 400×872 points for a 402×874 iPhone. Inspection returns the accessibility root's screen size, and the panel maps element bounds and annotation points with it instead. Automatic inspection picks the Metro target on the selected device whose bundle ID ends with the accessibility app name. When that finds none, such as Expo Go (`host.exp.Exponent`), iOS simulators look up the foreground app's bundle ID and match it exactly. A backgrounded React Native app never stands in for another foreground app. Traversal uses a work queue so deeply nested navigator wrappers do not cut off visible rows. Native-stack screens marked inactive through `activityState` and `aria-hidden` are excluded, along with disconnected native elements. Decorative elements marked only `aria-hidden` remain selectable. Text labels use string children when available, and text outlines follow the component's layout bounds. diff --git a/src/server/inspection-tools.ts b/src/server/inspection-tools.ts index beb770b9..274dac9b 100644 --- a/src/server/inspection-tools.ts +++ b/src/server/inspection-tools.ts @@ -10,7 +10,8 @@ import type { InspectorRequest } from "./react-native-inspector.ts"; import { readDeviceApps } from "./device-apps/sources.ts"; import { captureServerError } from "./telemetry.ts"; import { errorMessage, udidSchema } from "../shared/protocol.ts"; -import { screenComponents } from "../shared/screen-annotations.ts"; +import { accessibilityScreen, screenComponents } from "../shared/screen-annotations.ts"; +import type { ScreenSize } from "../shared/screen-annotations.ts"; import { adbPath } from "./native-logs.ts"; const execute = promisify(execFile); @@ -18,6 +19,7 @@ const execute = promisify(execFile); type NativeSnapshot = { native: unknown; app: Pick; + screen?: ScreenSize; }; async function iosSnapshot(baguette: Baguette, deviceId: string, signal: AbortSignal): Promise { @@ -26,6 +28,7 @@ async function iosSnapshot(baguette: Baguette, deviceId: string, signal: AbortSi const response = await baguette.describeUi(udid); return { native: response, + screen: accessibilityScreen(response.tree), app: { deviceName: device.name, appName: typeof response.tree?.label === "string" ? response.tree.label : undefined, @@ -69,7 +72,7 @@ export function registerInspectionTools(server: McpServer, baguette: Baguette, a _meta: { ui: { visibility: ["app", "model"] } }, }, async ({ platform, deviceId, deviceName, screenWidth, metroUrl, targetId }) => { try { - const { native, app } = platform === "ios" + const { native, app, screen } = platform === "ios" ? await iosSnapshot(baguette, deviceId, controller.signal) : await androidSnapshot(android, deviceId, deviceName, screenWidth); let runtime: Awaited>; @@ -80,7 +83,7 @@ export function registerInspectionTools(server: McpServer, baguette: Baguette, a runtime = { available: false, reason: "inspector-unavailable" }; } // Deep React trees exceed host JSON decoder limits. Keep ancestry as IDs on flat records. - const data = { tree: screenComponents(runtime.available ? [runtime.tree, native] : native), runtime: { available: runtime.available, truncated: runtime.available ? runtime.truncated : false } }; + const data = { tree: screenComponents(runtime.available ? [runtime.tree, native] : native), runtime: { available: runtime.available, truncated: runtime.available ? runtime.truncated : false }, ...(screen && { screen }) }; return { content: [{ type: "text", text: JSON.stringify(data) }], structuredContent: data }; } catch (error) { captureServerError(error, "inspection.tool"); diff --git a/src/shared/screen-annotations.ts b/src/shared/screen-annotations.ts index b98becb0..6b92001d 100644 --- a/src/shared/screen-annotations.ts +++ b/src/shared/screen-annotations.ts @@ -1,7 +1,8 @@ import type { SimulatorDevice } from "./protocol.ts"; export type ScreenPoint = { x: number; y: number }; -export type ScreenBounds = ScreenPoint & { width: number; height: number }; +export type ScreenSize = { width: number; height: number }; +export type ScreenBounds = ScreenPoint & ScreenSize; export type ReactElementContext = { component: string; owners: string[]; key?: string; sourceKind?: "element" | "owner"; source?: { file: string; line: number; column?: number; functionName?: string } }; export type ScreenComponent = { name: string; bounds: ScreenBounds; role?: string; identifier?: string; label?: string; value?: string; depth: number; source?: "accessibility" | "screen" | "react-native"; nodeId?: string; parentId?: string; react?: ReactElementContext }; export type ScreenSelectionContext = { ancestors: ScreenComponent[]; siblings: ScreenComponent[]; siblingCount: number; children: ScreenComponent[]; childCount: number; instance?: { index: number; total: number } }; @@ -11,7 +12,7 @@ export type ScreenAnnotation = { text: string; simulator: SimulatorDevice; point: ScreenPoint; - screen: { width: number; height: number; units: "points" | "pixels" }; + screen: ScreenSize & { units: "points" | "pixels" }; component: ScreenComponent; selection?: ScreenSelectionContext; nearbyText?: string[]; @@ -46,6 +47,18 @@ function bounds(value: unknown): ScreenBounds | undefined { return { x: x as number, y: y as number, width: width as number, height: height as number }; } +export function screenSize(value: unknown): ScreenSize | undefined { + const size = record(value), width = size?.width, height = size?.height; + if (typeof width === "number" && typeof height === "number" && Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) return { width, height }; +} + +// iOS accessibility roots span the full screen in points. Bezel definitions can describe a +// slightly smaller screen cutout, such as 400×872 for a 402×874 device. +export function accessibilityScreen(tree: unknown): ScreenSize | undefined { + const root = record(tree), frame = bounds(root?.frame); + if (frame?.x === 0 && frame.y === 0 && /^(AX)?Application$/i.test(String(root?.role ?? ""))) return screenSize(frame); +} + // Accept backend trees and normalized, flat MCP snapshots with explicit parents. export function screenComponents(tree: unknown, scale = 1): ScreenComponent[] { const components: ScreenComponent[] = []; @@ -73,7 +86,7 @@ export function screenComponents(tree: unknown, scale = 1): ScreenComponent[] { return components; } -export function componentAt(components: ScreenComponent[], point: ScreenPoint, screen?: { width: number; height: number }): ScreenComponent | undefined { +export function componentAt(components: ScreenComponent[], point: ScreenPoint, screen?: ScreenSize): ScreenComponent | undefined { return componentsAt(components, point, screen)[0]; } @@ -104,7 +117,7 @@ function unoccluded(hits: ScreenComponent[], { order, byId }: ComponentIndex): S || !covers.some(cover => cover.order > order.get(item)! && !cover.ancestors.has(item.nodeId!))); } -export function componentsAt(components: ScreenComponent[], point: ScreenPoint, screen?: { width: number; height: number }): ScreenComponent[] { +export function componentsAt(components: ScreenComponent[], point: ScreenPoint, screen?: ScreenSize): ScreenComponent[] { const index = componentIndex(components); const hits = unoccluded(components.filter(({ bounds: b }) => (!screen || b.width < screen.width * .95 || b.height < screen.height * .95) && point.x >= b.x && point.x < b.x + b.width && point.y >= b.y && point.y < b.y + b.height), index) .sort((a, b) => Number(b.source === "react-native") - Number(a.source === "react-native") || a.bounds.width * a.bounds.height - b.bounds.width * b.bounds.height || b.depth - a.depth); diff --git a/src/ui/screen-annotations.ts b/src/ui/screen-annotations.ts index efcb893f..adaa7615 100644 --- a/src/ui/screen-annotations.ts +++ b/src/ui/screen-annotations.ts @@ -1,10 +1,11 @@ import { componentAt, componentsAt, formatAnnotationMessage, screenComponents, screenSelectionContext } from "../shared/screen-annotations.ts"; -import type { ScreenAnnotation, ScreenBounds, ScreenComponent, ScreenPoint, ScreenSelectionContext } from "../shared/screen-annotations.ts"; +import type { ScreenAnnotation, ScreenBounds, ScreenComponent, ScreenPoint, ScreenSelectionContext, ScreenSize } from "../shared/screen-annotations.ts"; import { recordUiTiming, countUiEvent } from "./telemetry.ts"; import type { SimulatorDevice } from "../shared/protocol.ts"; import type { PanelContext } from "./model-context.ts"; type Capture = { screenshot: ScreenAnnotation["screenshot"]; screen: ScreenAnnotation["screen"] }; +export type InspectedTree = { tree: unknown; screen?: ScreenSize }; type Draft = Capture & { id: string; number: number; component: ScreenComponent; point: ScreenPoint; text: string; nearbyText?: string[]; selection?: ScreenSelectionContext }; export class ScreenAnnotationsStore { private state = { @@ -26,7 +27,7 @@ export class ScreenAnnotationsStore { private revision = 0; private nextNumber = 1; capture?: () => Capture; - readTree?: (simulator: SimulatorDevice) => Promise; + readTree?: (simulator: SimulatorDevice) => Promise; subscribe = (listener: () => void) => { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; }; getSnapshot = () => this.state; get messageText() { return formatAnnotationMessage(this.state.annotations); } @@ -76,15 +77,16 @@ export class ScreenAnnotationsStore { this.hoverPoint = undefined; this.cancelSelection(); this.update({ selecting: true, loading: true, capture, hovered: undefined, draft: undefined, candidates: [], error: "", status: "" }); - const tree = await this.readTree(this.simulator); + const { tree, screen } = await this.readTree(this.simulator); if (revision === this.revision) { + this.adoptScreen(screen); const startedAt = performance.now(); this.components = screenComponents(tree); recordUiTiming("ui.annotations.tree_processing", performance.now() - startedAt); if (this.components.some(component => component.react?.source)) countUiEvent("ui.annotations.source_available"); const draft = this.state.draft; if (draft?.component.source === "screen" && !draft.component.role) { - const candidates = componentsAt(this.components, draft.point, capture.screen); + const candidates = componentsAt(this.components, draft.point, draft.screen); if (candidates.length) this.update({ draft: { ...draft, component: candidates[0], selection: this.selectionContext(candidates[0]), nearbyText: this.nearbyText(candidates[0]) }, candidates }); } } @@ -92,6 +94,25 @@ export class ScreenAnnotationsStore { if (revision === this.revision) this.update({ error: `Could not read component names. You can still annotate a screen region. ${error instanceof Error ? error.message : String(error)}` }); } finally { if (revision === this.revision) { this.update({ loading: false }); this.hover(this.hoverPoint); } } } + // Bezel definitions can describe a smaller screen cutout than the accessibility screen that + // element bounds use, such as 400×872 for a 402×874 iPhone. Move this capture's pointer state to it. + private adoptScreen(size?: ScreenSize) { + const capture = this.state.capture; + if (!size || !capture || capture.screen.units !== "points" || (size.width === capture.screen.width && size.height === capture.screen.height)) return; + const scaleX = size.width / capture.screen.width, scaleY = size.height / capture.screen.height; + const point = (value: ScreenPoint) => ({ x: value.x * scaleX, y: value.y * scaleY }); + const bounds = (value: ScreenBounds) => ({ ...point(value), width: value.width * scaleX, height: value.height * scaleY }); + const screen = { ...capture.screen, ...size }; + const draft = this.state.draft; + if (this.dragStart) this.dragStart = point(this.dragStart); + if (this.hoverPoint) this.hoverPoint = point(this.hoverPoint); + this.update({ + capture: { ...capture, screen }, + selectionBounds: this.state.selectionBounds && bounds(this.state.selectionBounds), + ...(draft?.screenshot.id === capture.screenshot.id && { draft: { ...draft, screen, point: point(draft.point), + component: draft.component.source === "screen" ? { ...draft.component, bounds: bounds(draft.component.bounds) } : draft.component } }), + }); + } exit() { if (this.state.busy) return; this.revision++; @@ -100,8 +121,8 @@ export class ScreenAnnotationsStore { this.update({ selecting: false, loading: false, draft: undefined, hovered: undefined, candidates: [] }); } hover(point?: ScreenPoint) { - if (this.dragStart && point) { this.moveSelection(point); return; } this.hoverPoint = point; + if (this.dragStart && point) { this.moveSelection(point); return; } if (!this.state.selecting || this.state.draft) return; const hovered = point ? componentAt(this.components, point, this.state.capture?.screen) : undefined; if (hovered !== this.state.hovered) this.update({ hovered }); diff --git a/src/ui/simulator-panel.ts b/src/ui/simulator-panel.ts index 4811226d..5fd4b912 100644 --- a/src/ui/simulator-panel.ts +++ b/src/ui/simulator-panel.ts @@ -14,7 +14,7 @@ import { captureUiError, countUiEvent, recordUiTiming } from "./telemetry.ts"; import type { PanelContext } from "./model-context.ts"; import { getScreenAnnotations } from "./screen-annotations.ts"; import type { ScreenAnnotation } from "../shared/screen-annotations.ts"; -import { screenComponents } from "../shared/screen-annotations.ts"; +import { accessibilityScreen, screenComponents, screenSize } from "../shared/screen-annotations.ts"; import { AndroidVideo } from "./android-video.ts"; import { PhysicalIosVideo } from "./ios-mirror-video.ts"; import { iosVideoBatchSchema } from "../shared/ios-video.ts"; @@ -241,25 +241,25 @@ export function createSimulatorPanel( } annotations.capture = captureScreen; annotations.readTree = async simulator => { - if (simulator.kind === "physical" && simulator.platform === "ios") return []; + if (simulator.kind === "physical" && simulator.platform === "ios") return { tree: [] }; const screenWidth = annotations.getSnapshot().capture?.screen.width ?? points.width; const startedAt = performance.now(); try { try { const result = await call("mobile_inspect_ui", { platform, deviceId: simulator.udid, deviceName: simulator.name, screenWidth }, { timeout: 10000 }); - if (!result.structuredContent?.tree) throw new Error("Component inspection returned no tree."); - const runtime = result.structuredContent.runtime; + const content = result.structuredContent; + if (!content?.tree) throw new Error("Component inspection returned no tree."); + const runtime = content.runtime; if (runtime && typeof runtime === "object" && "available" in runtime && runtime.available === true) countUiEvent("ui.annotations.runtime_available"); if (runtime && typeof runtime === "object" && "truncated" in runtime && runtime.truncated === true) countUiEvent("ui.annotations.inspection_truncated"); - return result.structuredContent.tree; + return { tree: content.tree, screen: screenSize(content.screen) }; } catch { // Hosts with an older tool list and failed inspectors can still read AX. countUiEvent("ui.annotations.inspection_fallback"); const result = await call(platform === "android" ? "mobile_android_describe_ui" : "mobile_describe_ui", platform === "android" ? { deviceId: simulator.udid } : { udid: simulator.udid }, { timeout: 5000 }); - const screen = result.structuredContent?.screen as { width?: number } | undefined; - if (platform === "android" && screen?.width && Number.isFinite(screen.width) && screen.width > 0) - return screenComponents(result.structuredContent?.tree, screenWidth / screen.width); - return result.structuredContent?.tree; + const screen = screenSize(result.structuredContent?.screen); + if (platform === "android" && screen) return { tree: screenComponents(result.structuredContent?.tree, screenWidth / screen.width) }; + return { tree: result.structuredContent?.tree, screen: accessibilityScreen(result.structuredContent?.tree) }; } } finally { recordUiTiming("ui.annotations.inspection", performance.now() - startedAt); } }; diff --git a/tests/screen-annotations-ui.test.ts b/tests/screen-annotations-ui.test.ts index f62e7424..80f63388 100644 --- a/tests/screen-annotations-ui.test.ts +++ b/tests/screen-annotations-ui.test.ts @@ -39,7 +39,7 @@ test("the simulator toolbar opens notes, saves blue markers, and lets users edit const store = getScreenAnnotations(dom.window.document.querySelector('[data-element="stage"]')); const context = new PanelContext({ getHostCapabilities: () => ({ updateModelContext: { image: {} }, message: { text: {} } }), async sendMessage() { return { isError: true }; } } as unknown as App, { modelContext: { getCurrent: () => undefined, update: async () => ({ updateId: "annotation-update" }) } } as unknown as OpenAIExtensions); store.capture = () => ({ screenshot: { id: "capture", data: "AA==", capturedAt: "2026-10-01T10:00:00Z" }, screen: { width: 390, height: 844, units: "points" } }); - store.readTree = async () => ({ label: "Continue", identifier: "continue", role: "AXButton", frame: { x: 10, y: 20, width: 100, height: 40 } }); + store.readTree = async () => ({ tree: { label: "Continue", identifier: "continue", role: "AXButton", frame: { x: 10, y: 20, width: 100, height: 40 } } }); await act(async () => { store.connect(context); store.configure({ udid: "iphone", name: "iPhone", state: "Booted", runtime: "iOS", platform: "ios" }, false); store.setViewport({ x: 16, y: 16, width: 150, height: 300, stageWidth: 640, stageHeight: 360 }); diff --git a/tests/screen-annotations.test.ts b/tests/screen-annotations.test.ts index 6e80b3a7..a53207f1 100644 --- a/tests/screen-annotations.test.ts +++ b/tests/screen-annotations.test.ts @@ -1,7 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { componentAt, componentsAt, screenComponents, formatAnnotationContext, screenSelectionContext, ANNOTATION_EDIT_GUIDANCE } from "../src/shared/screen-annotations.ts"; +import { accessibilityScreen, componentAt, componentsAt, screenComponents, screenSize, formatAnnotationContext, screenSelectionContext, ANNOTATION_EDIT_GUIDANCE } from "../src/shared/screen-annotations.ts"; import { ScreenAnnotationsStore } from "../src/ui/screen-annotations.ts"; +import type { InspectedTree } from "../src/ui/screen-annotations.ts"; import type { ScreenAnnotation } from "../src/shared/screen-annotations.ts"; import { PanelContext } from "../src/ui/model-context.ts"; import { App } from "@modelcontextprotocol/ext-apps"; @@ -39,7 +40,7 @@ function fixture(messageApp?: App) { } } as unknown as OpenAIExtensions); const store = new ScreenAnnotationsStore(); store.connect(context); store.configure(simulator, false); store.capture = () => capture; - store.readTree = async () => ({ elements: [{ ...component, label: component.name, frame: component.bounds }] }); + store.readTree = async () => ({ tree: { elements: [{ ...component, label: component.name, frame: component.bounds }] } }); return { context, store, updates, messages, clear() { current = null; context.hostChanged(); }, remove(id: string) { if (current) current = { ...current, updateId: "removed", content: current.content?.filter(item => item._meta?.["mobile-dev/annotationId"] !== id) }; context.hostChanged(); }, @@ -123,7 +124,7 @@ test("a card painted on top hides the slightly smaller next card under a stack", test("a sparse accessibility tree supports explicit regions without pixel guesses", async () => { const f = fixture(); - f.store.readTree = async () => ({ role: "AXApplication", frame: { x: 0, y: 0, width: 393, height: 852 } }); + f.store.readTree = async () => ({ tree: { role: "AXApplication", frame: { x: 0, y: 0, width: 393, height: 852 } } }); await f.store.toggle(); f.store.hover({ x: 150, y: 280 }); assert.equal(f.store.getSnapshot().hovered, undefined); @@ -141,7 +142,7 @@ test("a sparse accessibility tree supports explicit regions without pixel guesse test("pending inspection accepts clicks and upgrades the open note when elements arrive", async () => { const f = fixture(); - let release!: (tree: unknown) => void; + let release!: (result: InspectedTree) => void; f.store.readTree = () => new Promise(resolve => { release = resolve; }); const reading = f.store.toggle(); assert.equal(f.store.getSnapshot().loading, true); @@ -149,13 +150,45 @@ test("pending inspection accepts clicks and upgrades the open note when elements f.store.endSelection({ x: 50, y: 40 }); assert.equal(f.store.getSnapshot().draft?.component.name, "Screen point"); f.store.setText("Make this wider"); - release({ label: "Continue", role: "AXButton", frame: component.bounds }); + release({ tree: { label: "Continue", role: "AXButton", frame: component.bounds } }); await reading; assert.equal(f.store.getSnapshot().draft?.component.name, "Continue"); assert.equal(f.store.getSnapshot().draft?.text, "Make this wider"); f.store.dispose(); }); +test("the accessibility screen size replaces a smaller bezel cutout and rescales pending pointer state", async () => { + const f = fixture(); + // The bezel cutout reports 400×872 for a 402×874 device; element bounds use the real screen. + f.store.capture = () => ({ ...capture, screen: { width: 400, height: 872, units: "points" } }); + let release!: (result: InspectedTree) => void; + f.store.readTree = () => new Promise(resolve => { release = resolve; }); + const reading = f.store.toggle(); + f.store.hover({ x: 200, y: 436 }); + assert.equal(f.store.beginSelection({ x: 100, y: 218 }), true); + f.store.hover({ x: 300, y: 654 }); + release({ tree: [{ label: "Info", role: "AXButton", frame: { x: 334.3, y: 550.7, width: 32.7, height: 32.3 } }], screen: { width: 402, height: 874 } }); + await reading; + const state = f.store.getSnapshot(); + assert.deepEqual(state.capture?.screen, { width: 402, height: 874, units: "points" }); + assert.deepEqual(Object.values(state.selectionBounds!).map(value => Math.round(value * 10) / 10), [100.5, 218.5, 201, 437]); + f.store.cancelSelection(); + f.store.hover({ x: 350, y: 567 }); + assert.equal(f.store.getSnapshot().hovered?.label, "Info"); + + // Pixel captures keep their size, and the MCP boundary rejects invalid sizes. + f.store.exit(); + f.store.capture = () => ({ ...capture, screen: { width: 1080, height: 2400, units: "pixels" } }); + f.store.readTree = async () => ({ tree: [], screen: { width: 402, height: 874 } }); + await f.store.toggle(); + assert.equal(f.store.getSnapshot().capture?.screen.width, 1080); + assert.equal(screenSize({ width: -1, height: 874 }), undefined); + assert.equal(screenSize({ width: 402, height: "874" }), undefined); + assert.deepEqual(accessibilityScreen({ role: "AXApplication", frame: { x: 0, y: 0, width: 402, height: 874 } }), { width: 402, height: 874 }); + assert.equal(accessibilityScreen({ role: "AXWindow", frame: { x: 0, y: 0, width: 402, height: 874 } }), undefined); + f.store.dispose(); +}); + test("pending or failed inspection keeps manual region selection working", async () => { const f = fixture(); let reject!: (error: Error) => void; @@ -181,7 +214,7 @@ test("real parents remain selectable without choosing overlapping siblings", asy assert.equal(candidates[0].parentId, candidates[1].nodeId); assert.equal(componentAt(items, { x: 300, y: 130 }, capture.screen)?.name, "Card"); const f = fixture(); - f.store.readTree = async () => tree; + f.store.readTree = async () => ({ tree }); await f.store.toggle(); f.store.select({ x: 40, y: 125 }); assert.deepEqual(f.store.getSnapshot().draft?.selection?.ancestors.map(item => item.name), ["Card"]); @@ -204,7 +237,7 @@ test("a child deletion identifies one repeated instance and keeps enclosing rows { source: "react-native", role: "RCTText", label: "Subtitle", nodeId: `subtitle-${index}`, frame: { x: 30, y: 140 + index * 90, width: 150, height: 20 }, react: { component: "Text", owners: ["HomeScreen", "Row"], source } }, ], })); - f.store.readTree = async () => rows; + f.store.readTree = async () => ({ tree: rows }); await f.store.toggle(); f.store.select({ x: 40, y: 235 }); f.store.setText("Remove this subtitle"); await f.store.save(); const saved = f.context.screenAnnotations[0]; @@ -261,7 +294,7 @@ test("flat MCP snapshots retain element names, depths and real parents", async ( const snapshot = JSON.parse(JSON.stringify(items)); assert.deepEqual(screenComponents(snapshot), items); const f = fixture(); - f.store.readTree = async () => snapshot; + f.store.readTree = async () => ({ tree: snapshot }); await f.store.toggle(); f.store.hover({ x: 300, y: 130 }); assert.equal(f.store.getSnapshot().hovered?.name, "Card"); @@ -275,7 +308,7 @@ test("flat MCP snapshots retain element names, depths and real parents", async ( test("a saved and sent note retains its source location, owners, testID and nearby text", async () => { const react = { component: "Text", owners: ["HomeScreen", "CardRow", "Text"], source: { file: "/project/src/HomeScreen.tsx", line: 49, column: 11, functionName: "HomeScreen.renderItem" } }; const f = fixture(); - f.store.readTree = async () => [{ ...component, source: "react-native", react, label: "Continue" }, { name: "Go to the next step", label: "Go to the next step", role: "Text", bounds: { x: 10, y: 65, width: 100, height: 20 } }]; + f.store.readTree = async () => ({ tree: [{ ...component, source: "react-native", react, label: "Continue" }, { name: "Go to the next step", label: "Go to the next step", role: "Text", bounds: { x: 10, y: 65, width: 100, height: 20 } }] }); await f.store.toggle(); f.store.select({ x: 50, y: 40 }); f.store.setText("Make this larger"); await f.store.save(); assert.deepEqual(f.context.screenAnnotations[0].component.react, react); assert.deepEqual(f.context.screenAnnotations[0].nearbyText, ["Go to the next step"]); @@ -400,9 +433,9 @@ test("selecting supports hover, multiple notes, edits, regions, and Escape witho f.store.edit(f.context.screenAnnotations[0]); f.store.setText("Make it blue"); await f.store.save(); assert.equal(f.context.screenAnnotations.length, 2); assert.equal(f.context.screenAnnotations.at(-1)?.text, "Make it blue"); f.store.exit(); - let release!: (value: unknown) => void; + let release!: (result: InspectedTree) => void; f.store.readTree = () => new Promise(resolve => { release = resolve; }); - const reading = f.store.toggle(); f.store.exit(); release({ elements: [component] }); await reading; + const reading = f.store.toggle(); f.store.exit(); release({ tree: { elements: [component] } }); await reading; assert.equal(f.store.getSnapshot().selecting, false); assert.equal(f.store.getSnapshot().loading, false); f.store.dispose(); }); diff --git a/tests/ui-telemetry.test.ts b/tests/ui-telemetry.test.ts index 847fd34d..c953b899 100644 --- a/tests/ui-telemetry.test.ts +++ b/tests/ui-telemetry.test.ts @@ -128,9 +128,9 @@ test("browser telemetry labels surface measurements, propagates traces, and flus store.connect(context); store.configure({ udid: "PRIVATE_DEVICE", name: "PRIVATE_DEVICE_NAME", runtime: "iOS 26", state: "Booted" }, false); store.capture = () => ({ screenshot: { id: "PRIVATE_CAPTURE", data: "PRIVATE_IMAGE", capturedAt: "2026-10-01T10:00:00Z" }, screen: { width: 402, height: 874, units: "points" } }); - store.readTree = async () => [{ source: "react-native", role: "RCTText", label: "PRIVATE_LABEL", bounds: { x: 10, y: 20, width: 100, height: 40 }, react: { + store.readTree = async () => ({ tree: [{ source: "react-native", role: "RCTText", label: "PRIVATE_LABEL", bounds: { x: 10, y: 20, width: 100, height: 40 }, react: { component: "PRIVATE_COMPONENT", owners: ["PRIVATE_OWNER"], key: "PRIVATE_REACT_KEY", source: { file: "/Users/alice/private.tsx", line: 49, column: 11 }, - } }]; + } }] }); await store.toggle(); store.select({ x: 50, y: 40 }); store.setText("PRIVATE_NOTE"); await store.save(); await store.send(); store.select({ x: 50, y: 40 }); store.setText("PRIVATE_NOTE_AGAIN"); await store.save(); for (const failure of ["timeout", "composer", "unexpected", ""]) { sendFailure = failure; await store.send(); } From 4d25d62ae66b3bf7062700128b14d8a5ddbc41a1 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Thu, 8 Oct 2026 20:04:32 +0200 Subject: [PATCH 5/6] fix(ui): open only iOS when both platforms have active devices Opening the panel with a booted iOS simulator and a running Android device showed both platforms side by side. Open iOS by default, matching the documented behavior, and start from iOS before devices load. Android still opens alone when it has the only active device, and the platform select can show both. --- docs/devices.md | 2 +- src/ui/components/workspace.tsx | 2 +- src/ui/device-layout.ts | 10 ++++------ src/ui/workspace-app.tsx | 2 +- tests/device-layout.test.ts | 4 ++-- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/devices.md b/docs/devices.md index 480c27d8..fb30d844 100644 --- a/docs/devices.md +++ b/docs/devices.md @@ -14,7 +14,7 @@ Android logs need `adb` from an installed Android SDK. The reader checks `ANDROI ## Device panel -Open a new chat after installing. Open Mobile Dev in the sidebar or call `mobile_open_workspace` for the fullscreen view. Call `mobile_open_simulator` for the panel beside a chat. iOS opens by default. Enable Android from the toolbar to show both panels side by side. Each panel has a device dropdown, Home, App Switcher, and Screenshot. Pick a device in each panel. Use the iOS and Android toggles to show either, both, or neither simulator. Selecting a device boots it if needed, then connects its screen. A selected running device connects automatically. The panel uses Apple’s device bezel and screen mask from the installed DeviceKit assets, with a simple frame as a fallback when assets are unavailable. Use the settings button at the bottom right for appearance, text size, location, and the device frame. iOS also offers contrast; Android offers rotation. The menu shows only settings supported by the bundled backend. Click the screen to type or drag. Closing the panel closes its stream. +Open a new chat after installing. Open Mobile Dev in the sidebar or call `mobile_open_workspace` for the fullscreen view. Call `mobile_open_simulator` for the panel beside a chat. iOS opens by default, including when an iOS device and an Android device are both running. Android opens instead when it has the only running device. Enable both platforms from the toolbar to show the panels side by side. Each panel has a device dropdown, Home, App Switcher, and Screenshot. Pick a device in each panel. Use the iOS and Android toggles to show either, both, or neither simulator. Selecting a device boots it if needed, then connects its screen. A selected running device connects automatically. The panel uses Apple’s device bezel and screen mask from the installed DeviceKit assets, with a simple frame as a fallback when assets are unavailable. Use the settings button at the bottom right for appearance, text size, location, and the device frame. iOS also offers contrast; Android offers rotation. The menu shows only settings supported by the bundled backend. Click the screen to type or drag. Closing the panel closes its stream. Use Select in the simulator toolbar to pause the screen. Hover to outline a component, then click to add a note. React Native development apps can supply runtime elements when accessibility omits a view. Drag to mark a region when neither source exposes it. Saved notes leave numbered blue bubbles. Notes attach text and available element details to your next chat message. The captured screen stays local for editing; annotations never attach screenshots. Click a bubble to edit or remove a note, or use Send to chat to send all notes for that device. If chat is unavailable, the panel keeps the notes and retries when you return. A sent or cleared batch starts again at 1. diff --git a/src/ui/components/workspace.tsx b/src/ui/components/workspace.tsx index e603a0eb..4fe6c9a3 100644 --- a/src/ui/components/workspace.tsx +++ b/src/ui/components/workspace.tsx @@ -23,7 +23,7 @@ export type { DeviceLayout } from "../device-layout.ts"; export function Workspace({ logs, performance, recordingController, updates, automaticLayout, onLayout }: { logs: LogsPanel; performance: PerformancePanel; recordingController: RecordingController; updates?: PluginUpdateController; automaticLayout?: DeviceLayout; onLayout: (layout: DeviceLayout) => void }) { const [tool, setTool] = useState<"logs" | "performance">("logs"); const [savedVisible, setSavedVisible] = useState(false); - const [layout, setLayout] = useState(automaticLayout ?? "both"); + const [layout, setLayout] = useState(automaticLayout ?? "ios"); const manualLayout = useRef(false); const recordingState = useSyncExternalStore(recordingController.subscribe, recordingController.getSnapshot); useEffect(() => { diff --git a/src/ui/device-layout.ts b/src/ui/device-layout.ts index f58fa829..a1e2d58e 100644 --- a/src/ui/device-layout.ts +++ b/src/ui/device-layout.ts @@ -7,11 +7,9 @@ export function isDeviceActive(device?: SimulatorDevice): boolean { return device?.state === "Booted"; } +// Open one platform by default. iOS wins when both have an active device; the +// platform select can still show both. export function activeDeviceLayout(ios?: SimulatorDevice, android?: SimulatorDevice): DeviceLayout { - const iosActive = isDeviceActive(ios); - const androidActive = isDeviceActive(android); - if (iosActive && androidActive) return "both"; - if (iosActive) return "ios"; - if (androidActive) return "android"; - return "none"; + if (isDeviceActive(ios)) return "ios"; + return isDeviceActive(android) ? "android" : "none"; } diff --git a/src/ui/workspace-app.tsx b/src/ui/workspace-app.tsx index 8c06d686..06b6fa84 100644 --- a/src/ui/workspace-app.tsx +++ b/src/ui/workspace-app.tsx @@ -49,7 +49,7 @@ const panels = (["ios", "android"] as const).map(platform => { }); const [ios, android] = panels; logsPanel.setLayout(document.documentElement.dataset.view === "workspace"); -changeLayout("both"); +changeLayout("ios"); function updateSelection() { const visible = panels.filter(panel => !panel.root.hidden); diff --git a/tests/device-layout.test.ts b/tests/device-layout.test.ts index 50cdebed..104b27f0 100644 --- a/tests/device-layout.test.ts +++ b/tests/device-layout.test.ts @@ -11,12 +11,12 @@ const iosPhone: SimulatorDevice = { }; const androidPhone: SimulatorDevice = { udid: "phone", name: "Pixel", model: "Pixel", state: "Booted", runtime: "Android", platform: "android", kind: "physical", transportType: "wired" }; -test("only active platforms are visible for simulators, emulators, and physical phones", () => { +test("one active platform opens by default, preferring iOS when both have active devices", () => { for (const ios of [iosSimulator, iosPhone]) { assert.equal(activeDeviceLayout(ios), "ios"); for (const android of [androidEmulator, androidPhone]) { assert.equal(activeDeviceLayout(undefined, android), "android"); - assert.equal(activeDeviceLayout(ios, android), "both"); + assert.equal(activeDeviceLayout(ios, android), "ios"); } } }); From 03f5af8746896d20dafcd96cdd8da026cc3fef1d Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Thu, 8 Oct 2026 20:04:33 +0200 Subject: [PATCH 6/6] chore(release): bump version to 0.1.139 --- .codex-plugin/plugin.json | 2 +- package-lock.json | 4 ++-- package.json | 2 +- src/shared/version.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 3bb237eb..03e5e16f 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "mobile-dev", - "version": "0.1.134", + "version": "0.1.139", "description": "Keep a simulator beside the chat while building, running, or debugging iOS and Android apps. Stream devices, read native and Metro logs, render interactive CPU, memory, and FPS recordings, and control apps with Mobile Dev tools.", "author": { "name": "Callstack" diff --git a/package-lock.json b/package-lock.json index 4cd00eb4..6db92a24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codex-mobile-dev-plugin", - "version": "0.1.134", + "version": "0.1.139", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codex-mobile-dev-plugin", - "version": "0.1.134", + "version": "0.1.139", "dependencies": { "@base-ui/react": "^1.8.0", "@fontsource-variable/inter": "^5.3.0", diff --git a/package.json b/package.json index dfd92368..31761a0c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codex-mobile-dev-plugin", - "version": "0.1.134", + "version": "0.1.139", "private": true, "type": "module", "engines": { diff --git a/src/shared/version.ts b/src/shared/version.ts index f8654781..84777a96 100644 --- a/src/shared/version.ts +++ b/src/shared/version.ts @@ -1 +1 @@ -export const PLUGIN_VERSION = "0.1.134"; +export const PLUGIN_VERSION = "0.1.139";