From 8f3f212d2e3453c1b9e591943ced12193a798759 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sat, 3 Oct 2026 22:13:02 +0900 Subject: [PATCH 01/32] feat: add native Xcode 27 iOS Simulator support Integrate Xcode's native MCP interaction sessions and simctl with the shared device driver, standalone CLI, and embedded SDK.\n\nPreserve Android defaults, native accessibility hit points, and cancellation-safe device leases. Add read-only setup checks, usage documentation, and hermetic regression coverage. --- README.md | 14 +- README_CN.md | 13 +- artemis/agents/checker/checker.py | 2 + artemis/agents/diagnoser/diagnoser.py | 4 + artemis/agents/explorer/screen_index.py | 10 +- artemis/agents/flash/runner.py | 6 +- artemis/agents/operator/operator.py | 4 +- artemis/agents/operator/prompts.py | 8 + artemis/agents/platform_guidance.py | 29 + artemis/context.py | 13 + artemis/controllers/unified_controller.py | 24 + artemis/drivers/factory.py | 10 + artemis/drivers/ios/__init__.py | 12 + artemis/drivers/ios/bridge.py | 156 +++++ artemis/drivers/ios/hierarchy.py | 95 +++ artemis/drivers/ios/xcode_driver.py | 427 +++++++++++++ artemis/interfaces/cli/commands/run.py | 62 +- artemis/mcp/action_session.py | 9 +- artemis/mcp/actuators/ios.py | 79 +++ artemis/sdk/agent.py | 166 ++++- artemis/sdk/builders/agent_config_builder.py | 4 + artemis/tools/index.py | 9 +- artemis/tools/mobile/launch_app.py | 43 +- artemis/tools/mobile/read_hierarchy.py | 8 +- artemis/utils/visualization.py | 55 +- docs/ios.md | 190 ++++++ pyright-core.json | 2 + scripts/setup_ios_env.sh | 67 ++ tests/unit/drivers/test_ios_bridge.py | 353 +++++++++++ tests/unit/drivers/test_ios_driver.py | 614 ++++++++++++++++++ tests/unit/test_cli.py | 1 + tests/unit/test_ios_integration.py | 623 +++++++++++++++++++ tests/unit/test_ios_observation.py | 103 +++ third_party/mobile_use/context.py | 1 + third_party/mobile_use/sdk/agent.py | 16 +- 35 files changed, 3194 insertions(+), 38 deletions(-) create mode 100644 artemis/agents/platform_guidance.py create mode 100644 artemis/drivers/ios/__init__.py create mode 100644 artemis/drivers/ios/bridge.py create mode 100644 artemis/drivers/ios/hierarchy.py create mode 100644 artemis/drivers/ios/xcode_driver.py create mode 100644 artemis/mcp/actuators/ios.py create mode 100644 docs/ios.md create mode 100755 scripts/setup_ios_env.sh create mode 100644 tests/unit/drivers/test_ios_bridge.py create mode 100644 tests/unit/drivers/test_ios_driver.py create mode 100644 tests/unit/test_ios_integration.py create mode 100644 tests/unit/test_ios_observation.py diff --git a/README.md b/README.md index 8683bcce..f63feec3 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,18 @@ cd artemis > **Tip**: Opens `http://localhost:8000` in your default browser with a device connection wizard, live screen mirroring, prompt sandbox, and execution replays. You can also run directly from CLI: `uv run artemis run "Open Settings, find Battery and tell me current level" --profile flash`. +### iOS Simulator (macOS, Xcode 27+) + +Standalone CLI and embedded Python SDK tasks can target iOS simulators through +Xcode's native MCP device-interaction tools. See the [iOS setup and usage guide](./docs/ios.md) +for prerequisites, Xcode access approval, and current limits: + +```bash +bash scripts/setup_ios_env.sh +uv run artemis run "Open Settings and inspect the General screen" \ + --platform ios --standalone --device-serial --profile flash +``` +
@@ -300,7 +312,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [ ] **iOS Platform Expansion**: Extending multimodal perception and mobile automation to iOS devices and simulators. +- [ ] **iOS Platform Expansion**: [Standalone iOS Simulator support](./docs/ios.md) is available with Xcode 27+; physical devices and web/daemon integration remain planned. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index 67f461c5..fce858bd 100644 --- a/README_CN.md +++ b/README_CN.md @@ -106,6 +106,17 @@ cd artemis > **提示**：启动后将自动在默认浏览器中打开 Web 控制台（`http://localhost:8000`），提供设备连接向导、实时投屏、任务演练与状态回放面板。你也可以通过命令行直接运行：`uv run artemis run "打开系统设置，找到电池选项并告诉我当前电量" --profile flash`。 +### iOS 模拟器（macOS，Xcode 27+） + +独立 CLI 和嵌入式 Python SDK 可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器。 +安装要求、Xcode 权限审批和当前限制见 [iOS 使用指南（英文）](./docs/ios.md)： + +```bash +bash scripts/setup_ios_env.sh +uv run artemis run "打开系统设置并查看通用页面" \ + --platform ios --standalone --device-serial --profile flash +``` +
@@ -298,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 路线图 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [ ] **iOS 跨平台支持**：将视觉感知与自动化执行引擎拓展至 iOS 真机与模拟器。 +- [ ] **iOS 跨平台支持**：[独立模式 iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+；真机和 Web/Daemon 集成仍在规划中。 - [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。 diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index 52dff2ca..3c081177 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -255,6 +255,8 @@ def _load_prompts() -> dict[str, str]: def probes_enabled(ctx: ArtemisContext) -> bool: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + return False setup = getattr(ctx, "execution_setup", None) return not (setup and getattr(setup, "disable_device_probes", False)) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 13d80317..133893ad 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -72,6 +72,8 @@ def __init__(self, ctx: ArtemisContext): self._llm = None def _is_device_online(self) -> bool: + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": + return getattr(self.ctx, "_active_driver", None) is not None try: if self.ctx.adb_client is None: return False @@ -111,6 +113,8 @@ def _build_traced_tools(self) -> list: *get_history_tools(self.ctx), get_submit_answer_tool(self.ctx), ] + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": + all_tools = [t for t in all_tools if t.name != "run_adb_command"] if not self.is_device_online: logger.info( "Diagnoser running in OFFLINE environment: stripping adb short" diff --git a/artemis/agents/explorer/screen_index.py b/artemis/agents/explorer/screen_index.py index 837082a2..a91b0b2f 100644 --- a/artemis/agents/explorer/screen_index.py +++ b/artemis/agents/explorer/screen_index.py @@ -24,7 +24,7 @@ import re from typing import Any, Literal -from artemis.utils.visualization import parse_bounds +from artemis.utils.visualization import get_hit_point, parse_bounds Bounds = tuple[int, int, int, int] ElementSource = Literal["xml", "ocr"] @@ -84,6 +84,8 @@ class ScreenElement: @property def center(self) -> tuple[int, int]: + if (point := get_hit_point(self.node)) is not None: + return point left, top, right, bottom = self.bounds return (left + right) // 2, (top + bottom) // 2 @@ -173,8 +175,12 @@ def add(element: ScreenElement) -> None: for node in fused_xml or []: if not isinstance(node, dict): continue + if "hit_point" in node and get_hit_point(node, width, height) is None: + node = {key: value for key, value in node.items() if key != "hit_point"} class_name = node.get("class") or node.get("className") - resource_id = node.get("resource-id") or node.get("resourceId") + resource_id = ( + node.get("resource-id") or node.get("resourceId") or node.get("resource_id") + ) interactive = any(_is_truthy_attr(node.get(key)) for key in INTERACTION_KEYS) for ocr in node.get("ocr_elements") or []: diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 3ea3f686..a875f3ce 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -296,7 +296,11 @@ def _render_system_prompt(self, tools_declaration: list) -> str: prompt_path = Path(__file__).parent / "flash_runner.md" prompt_template = prompt_path.read_text(encoding="utf-8") available_tools = frozenset(t.name for t in tools_declaration) - return Template(prompt_template).render(goal=self.goal, available_tools=available_tools) + from artemis.agents.platform_guidance import device_action_guidance + + return device_action_guidance(self.ctx) + Template(prompt_template).render( + goal=self.goal, available_tools=available_tools + ) # ------------------------------------------------------------------ # Per-turn helpers (observe / think) diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index d2453627..49bd3abf 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -919,7 +919,9 @@ async def __call__(self, state: State): # The analyzer is always available: the "output truncated, use # analyze_task_output" hint arrives mid-turn, and tools are bound once per # turn, so a conditional mount would be one turn late. - if not any(t.name == "analyze_task_output" for t in all_tools): + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) != "ios" and not any( + t.name == "analyze_task_output" for t in all_tools + ): all_tools.append(analyze_task_output_wrapper.tool_fn_getter(self.ctx)) traced_tools = [trace_langchain_tool(t, self.ctx) for t in all_tools] diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de23..d7eb3dc2 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -203,6 +203,8 @@ def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]: # video_recording_tools_enabled); the prompt must not advertise it when the # tool is not actually available this run. available = set(OPERATOR_PROMPT_TOOLSET) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + available.difference_update({"run_adb_command", "manage_task", "analyze_task_output"}) setup = getattr(ctx, "execution_setup", None) if not (setup and getattr(setup, "video_recording_tools_enabled", False)): available.discard("video_analyzer") @@ -316,6 +318,9 @@ def render_transcript_static_system( available = resolve_operator_prompt_tools(ctx) static_template = apply_operator_prompt_contract(static_template, available_tools=available) + from artemis.agents.platform_guidance import device_action_guidance + + static_template = device_action_guidance(ctx) + static_template return Template(static_template).render( initial_goal=state.initial_goal, subgoals_status="", @@ -354,6 +359,9 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte available = resolve_operator_prompt_tools(ctx) prompt_template = apply_operator_prompt_contract(prompt_template, available_tools=available) + from artemis.agents.platform_guidance import device_action_guidance + + prompt_template = device_action_guidance(ctx) + prompt_template plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.") diff --git a/artemis/agents/platform_guidance.py b/artemis/agents/platform_guidance.py new file mode 100644 index 00000000..e44dcb8c --- /dev/null +++ b/artemis/agents/platform_guidance.py @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared platform constraints for the Flash and Pro device-action prompts.""" + + +def device_action_guidance(ctx) -> str: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) != "ios": + return "" + return ( + "Target platform: iOS Simulator. For input_text, explicitly set clear_exist=false; " + "type into an empty field or at its existing cursor. Whole-field clearing, Android " + "keycodes, Back/Delete keys, ADB commands, video analysis and app locking are " + "unavailable. Navigate with visible iOS controls; press_key supports enter, home, " + "power, volume_up, volume_down, and app_switch. manage_app accepts installed iOS " + "display names or bundle identifiers. Action coordinates use the screenshot's " + "normalized 0-1000 space.\n\n" + ) diff --git a/artemis/context.py b/artemis/context.py index 57360c8c..c9738688 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -198,6 +198,17 @@ class ArtemisContext(DeviceClientAccessors, BaseModel): async def __aenter__(self) -> ArtemisContext: return self + async def disconnect_driver(self) -> None: + """Release the cached driver, including partially initialized transports.""" + driver = self._active_driver + self._active_driver = None + self._mobile_controller = None + if driver is not None: + try: + await driver.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug(f"Device driver disconnect failed: {exc}", exc_info=True) + async def __aexit__(self, exc_type, exc_val, exc_tb): # Close the in-process action session before draining background tasks so its # owner task exits cleanly rather than being cancelled below. @@ -283,6 +294,8 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except Exception as exc: logger.debug(f"DataEngine shutdown failed; skipped: {exc}", exc_info=True) + await self.disconnect_driver() + from artemis.data_engine.engine import DataEngine diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index d0dc1d03..818d4caa 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -56,6 +56,21 @@ def __init__(self, ctx: ArtemisContext): super().__init__(ctx, get_driver(ctx)) self._segment_cache: dict[tuple[str, int, float, float], VideoRecordingResult] = {} + async def open_url(self, url: str) -> bool: + if self.ctx.device.mobile_platform == "ios": + return await self._driver.open_url(url) + return await super().open_url(url) + + async def erase_text(self, nb_chars: int | None = None) -> bool: + if self.ctx.device.mobile_platform == "ios": + if nb_chars is not None and nb_chars > 0: + for _ in range(nb_chars): + if not await self._driver.press_key("delete"): + return False + return True + return await self._driver.input_text("", clear_existing=True) + return await super().erase_text(nb_chars) + @staticmethod async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process: kwargs: dict[str, Any] = { @@ -399,6 +414,11 @@ async def start_video_recording( ) -> VideoRecordingResult: """Start screen recording on Android device using scrcpy.""" self._segment_cache.clear() + if self.ctx.device.mobile_platform == "ios": + return VideoRecordingResult( + success=False, + message="iOS video analysis is not supported by the recording controller.", + ) device_id = self._get_device_id() # Check mock driver first @@ -519,6 +539,10 @@ async def start_video_recording( async def stop_video_recording(self) -> VideoRecordingResult: """Stop scrcpy recording and return the converted MP4 video file.""" self._segment_cache.clear() + if self.ctx.device.mobile_platform == "ios": + return VideoRecordingResult( + success=False, message="No iOS controller recording is active." + ) device_id = self._get_device_id() # Check mock driver first diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index d6283a1c..8b024595 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -33,6 +33,16 @@ def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: """Instantiates the appropriate BaseDeviceDriver based on the runtime context.""" + mobile_platform = getattr(ctx.device, "mobile_platform", "android") + if mobile_platform == "ios": + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS Simulator support is local only; cloud mode targets Android.") + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + return XcodeSimulatorDriver(device_id=ctx.device.device_id) + if mobile_platform not in {"android", "mock"}: + raise ValueError(f"Unsupported mobile platform: {mobile_platform}") + # 1. Cloud mode check. Cloud devices are reached through the gateway's # RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there # because the Accessibility Helper needs a local adb forward. diff --git a/artemis/drivers/ios/__init__.py b/artemis/drivers/ios/__init__.py new file mode 100644 index 00000000..1c23238d --- /dev/null +++ b/artemis/drivers/ios/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native Xcode simulator drivers (loaded only when iOS is selected).""" diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py new file mode 100644 index 00000000..2d06366a --- /dev/null +++ b/artemis/drivers/ios/bridge.py @@ -0,0 +1,156 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Task-safe access to Xcode's MCP bridge using Artemis's existing MCP SDK. + +The SDK handles initialization, notifications and response IDs. A single owner +task enters and exits the SDK's AnyIO contexts, and serializes device commands. +A timeout retires the bridge; a possibly executed input is never retried. +""" + +import asyncio +import json +import os +from typing import Any + +import anyio +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.shared.exceptions import McpError + + +_ERRORS = ( + OSError, + ValueError, + RuntimeError, + McpError, + ExceptionGroup, + anyio.EndOfStream, + anyio.ClosedResourceError, + anyio.BrokenResourceError, +) + + +class XcodeBridge: + """Own one native bridge subprocess for one simulator driver.""" + + def __init__(self, timeout_seconds: float = 120.0): + self.timeout_seconds = timeout_seconds + self._owner: asyncio.Task[None] | None = None + self._ready: asyncio.Future[set[str]] | None = None + self._requests: asyncio.Queue[Any] = asyncio.Queue() + self._active: asyncio.Future[Any] | None = None + self.tools: set[str] = set() + + @property + def connected(self) -> bool: + return self._owner is not None and not self._owner.done() + + async def start(self) -> None: + if self._owner is not None and not self._owner.done(): + return + self._ready = asyncio.get_running_loop().create_future() + self._owner = asyncio.create_task(self._run(), name="artemis-xcode-bridge") + try: + self.tools = await asyncio.wait_for(asyncio.shield(self._ready), self.timeout_seconds) + except (*_ERRORS, TimeoutError, asyncio.CancelledError): + await self.close() + raise + + async def _run(self) -> None: + try: + # Preserve DEVELOPER_DIR/MCP_XCODE_PID and the user's approved configuration. + params = StdioServerParameters( + command="xcrun", args=["mcpbridge"], env=dict(os.environ) + ) + async with stdio_client(params) as (reader, writer): + async with ClientSession(reader, writer) as session: + await session.initialize() + names: set[str] = set() + cursor = None + while True: + page = await session.list_tools(cursor=cursor) + names.update(tool.name for tool in page.tools) + cursor = page.nextCursor + if not cursor: + break + if self._ready is not None and not self._ready.done(): + self._ready.set_result(names) + while True: + request = await self._requests.get() + if request is None: + return + future, name, arguments = request + if future.cancelled(): + continue + self._active = future + result = await session.call_tool(name, arguments) + if not future.done(): + future.set_result(result) + self._active = None + except _ERRORS as exc: + if self._ready is not None and not self._ready.done(): + self._ready.set_exception(RuntimeError(f"Cannot connect to Xcode MCP: {exc}")) + if self._active is not None and not self._active.done(): + self._active.set_exception(RuntimeError(f"Xcode MCP connection failed: {exc}")) + finally: + error = RuntimeError( + "Xcode MCP bridge has closed. Reconnect the driver before retrying." + ) + if self._ready is not None and not self._ready.done(): + self._ready.set_exception(error) + if self._active is not None and not self._active.done(): + self._active.set_exception(error) + self._active = None + while not self._requests.empty(): + request = self._requests.get_nowait() + if request is not None and not request[0].done(): + request[0].set_exception(error) + + async def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + if self._owner is None or self._owner.done(): + raise RuntimeError("Xcode MCP bridge is not connected.") + future = asyncio.get_running_loop().create_future() + self._requests.put_nowait((future, name, arguments)) + try: + result = await asyncio.wait_for(future, self.timeout_seconds) + except (TimeoutError, asyncio.CancelledError): + await self.close() + raise + if result.isError: + message = " ".join(block.text for block in result.content if block.type == "text") + raise RuntimeError(f"Xcode tool {name} failed: {message}") + if isinstance(result.structuredContent, dict): + return result.structuredContent + # Xcode also supports older clients with JSON text content. + for block in result.content: + if block.type == "text": + try: + data = json.loads(block.text) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + return data + raise RuntimeError(f"Xcode tool {name} returned no structured result.") + + async def close(self) -> None: + owner, self._owner = self._owner, None + if owner is None: + return + if not owner.done(): + owner.cancel() + try: + await owner + except asyncio.CancelledError: + pass + if self._ready is not None and self._ready.done() and not self._ready.cancelled(): + self._ready.exception() # consume an initialization error on cancellation + self.tools.clear() diff --git a/artemis/drivers/ios/hierarchy.py b/artemis/drivers/ios/hierarchy.py new file mode 100644 index 00000000..b0cf4dc2 --- /dev/null +++ b/artemis/drivers/ios/hierarchy.py @@ -0,0 +1,95 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Convert Xcode's textual accessibility frames and hit points to screen pixels.""" + +import re +from typing import Any + +_NUMBER = r"(-?\d+(?:\.\d+)?)" +_FRAME = re.compile( + r"\{\{\s*" + + _NUMBER + + r",\s*" + + _NUMBER + + r"\},\s*\{\s*" + + _NUMBER + + r",\s*" + + _NUMBER + + r"\}\}" +) +_HIT = re.compile(r"hitPoint:\s*\{\s*" + _NUMBER + r",\s*" + _NUMBER + r"\}") +_BUNDLE = re.compile(r"Application bundle identifier:\s*(\S+)") + + +def application_bundle(hierarchy: str) -> str | None: + bundles = set(_BUNDLE.findall(hierarchy)) + return next(iter(bundles)) if len(bundles) == 1 else None + + +def parse_hierarchy( + hierarchy: str, width: int, height: int +) -> tuple[list[dict[str, Any]], tuple[float, float]]: + """Determine scale from the screen window; never assume a Retina factor. + + UIKit hierarchy geometry is in the native interaction coordinate space. + Artemis observations/actions use full screenshot pixels. Missing or rotated + geometry is an error rather than silently tapping with an unverified scale. + """ + lines = hierarchy.splitlines() + windows = [] + for line in lines: + match = _FRAME.search(line) + if match and re.match(r"\s*(?:UI)?Window\b", line): + x, y, w, h = map(float, match.groups()) + if x == 0 and y == 0 and w > 0 and h > 0: + windows.append((w, h)) + matching = [(w, h) for w, h in windows if abs(width / w - height / h) < 0.05] + if windows and not matching: + raise RuntimeError( + "Xcode hierarchy has no screen window matching the screenshot. Recapture before interacting." + ) + # Xcode 27 exports the full screenshot at logical screen dimensions. An + # inaccessible custom canvas can have no AX Window; visual targeting still + # works in this documented native coordinate space. + w, h = max(matching, key=lambda size: size[0] * size[1]) if matching else (width, height) + scale = width / w, height / h + elements = [] + for line in lines: + match = _FRAME.search(line) + if not match: + continue + x, y, w, h = map(float, match.groups()) + if w <= 0 or h <= 0: + continue + left, top = round(x * scale[0]), round(y * scale[1]) + right, bottom = round((x + w) * scale[0]), round((y + h) * scale[1]) + if right <= 0 or bottom <= 0 or left >= width or top >= height: + continue + label = re.search(r"label:\s*'((?:\\.|[^'])*)'", line) + quoted = re.search(r'"([^"\n]*)"', line[: match.start()]) + identifier = re.search(r"identifier:\s*'((?:\\.|[^'])*)'", line) + hit = _HIT.search(line) + activation = re.search(r"activationBundleId:\s*(\S+)", line) + element: dict[str, Any] = { + "text": label.group(1) if label else quoted.group(1) if quoted else "", + "resource_id": identifier.group(1) if identifier else "", + "class": line.strip().split(",", 1)[0].split(" ", 1)[0], + "bounds": f"[{left},{top}][{right},{bottom}]", + "parsed_bounds": {"left": left, "top": top, "right": right, "bottom": bottom}, + } + if hit: + hx, hy = map(float, hit.groups()) + element["hit_point"] = [round(hx * scale[0]), round(hy * scale[1])] + if activation: + element["activation_bundle_id"] = activation.group(1) + elements.append(element) + return elements, scale diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py new file mode 100644 index 00000000..d9caf37a --- /dev/null +++ b/artemis/drivers/ios/xcode_driver.py @@ -0,0 +1,427 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator support using Xcode 27's native MCP and simctl tools.""" + +import asyncio +import base64 +from io import BytesIO +import json +from pathlib import Path +import plistlib +import re +import sys +from typing import Any, Literal +from uuid import uuid4 + +from PIL import Image + +from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection +from artemis.drivers.ios.bridge import XcodeBridge +from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy +from third_party.mobile_use.controllers.types import ElementQuery +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + + +async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: + """Run argv directly, reporting native errors and reaping cancelled children.""" + process = await asyncio.create_subprocess_exec( + "xcrun", + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except (TimeoutError, asyncio.CancelledError): + if process.returncode is None: + process.kill() + await process.communicate() + raise + if process.returncode: + raise RuntimeError( + f"xcrun {' '.join(arguments[:3])} failed: {stderr.decode(errors='replace').strip()}" + ) + return stdout + + +class XcodeSimulatorDriver(BaseDeviceDriver): + """One explicitly selected iOS simulator and one native interaction session. + + Android remains the default platform. This driver neither enables Xcode + permissions nor controls physical devices. Unsupported Android operations + report an error instead of returning a synthetic success. + """ + + def __init__(self, device_id: str = "booted", width: int = 0, height: int = 0): + self._device_id = device_id + self._width, self._height = width, height + self._scale: tuple[float, float] | None = None + self._session_key: str | None = None + self._bridge = XcodeBridge() + self._connect_lock = asyncio.Lock() + self._operation_lock = asyncio.Lock() + + @property + def device_id(self) -> str: + return self._device_id + + @property + def screen_size(self) -> tuple[int, int]: + return self._width, self._height + + async def resolve_device(self) -> str: + """Read-only validation and UDID selection, safe before the execution lease.""" + async with self._connect_lock: + await self._resolve_device() + return self._device_id + + async def _resolve_device(self) -> dict[str, Any]: + if sys.platform != "darwin": + raise RuntimeError("iOS Simulator support requires macOS and Xcode 27 or later.") + version = (await run_xcrun("xcodebuild", "-version")).decode() + match = re.search(r"Xcode\s+(\d+)", version) + if not match or int(match.group(1)) < 27: + raise RuntimeError( + "Native iOS interaction requires Xcode 27 or later. Set DEVELOPER_DIR to select it." + ) + devices = json.loads(await run_xcrun("simctl", "list", "devices", "--json")) + available = [ + device + for runtime, entries in devices.get("devices", {}).items() + if ".iOS-" in runtime + for device in entries + if device.get("isAvailable") + ] + if self._device_id == "booted": + candidates = [device for device in available if device.get("state") == "Booted"] + if len(candidates) != 1: + raise ValueError( + "Select --device-serial ; 'booted' requires exactly one booted iOS simulator." + ) + else: + candidates = [ + device + for device in available + if device.get("udid", "").lower() == self._device_id.lower() + ] + if len(candidates) != 1: + raise ValueError(f"Unavailable iOS simulator UDID: {self._device_id}") + self._device_id = candidates[0]["udid"] + return candidates[0] + + async def connect(self) -> None: + async with self._connect_lock: + if self._session_key: + return + candidate = await self._resolve_device() + if candidate.get("state") == "Shutdown": + await run_xcrun("simctl", "boot", self._device_id) + await run_xcrun("simctl", "bootstatus", self._device_id, "-b", timeout=180.0) + await self._bridge.start() + required = { + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", + } + if not required.issubset(self._bridge.tools): + await self._bridge.close() + raise RuntimeError( + "Xcode's native device interaction tools are unavailable. Select Xcode 27 and approve Artemis's access in Xcode." + ) + try: + session = await self._bridge.call( + "DeviceInteractionStartSession", + { + "deviceIdentifier": self._device_id, + "sessionIdentifier": f"Artemis Simulator {uuid4().hex[:8]}", + }, + ) + self._session_key = session.get("interactionSessionKey") + if not self._session_key: + raise RuntimeError("Xcode did not return a device interaction session key.") + if ( + not session.get("deviceIsSimulator") + or session.get("deviceUUID", "").lower() != self._device_id.lower() + ): + raise RuntimeError("Xcode selected a different device; refusing to interact.") + await self.get_screen_data(skip_settling=True) + except (OSError, ValueError, RuntimeError, TimeoutError, asyncio.CancelledError): + try: + await self.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: + logger.warning( + f"Could not release the Xcode session after a connection failure: {cleanup_error}", + ) + raise + + async def disconnect(self) -> None: + async with self._operation_lock: + key, self._session_key = self._session_key, None + try: + if key: + # A timeout retires the old connection; cleanup may create + # a fresh bridge solely to close the known native session. + if not self._bridge.connected: + await self._bridge.start() + await self._bridge.call( + "DeviceInteractionEndSession", {"interactionSessionKey": key} + ) + finally: + self._scale = None + await self._bridge.close() + + async def _synthesize(self, command: str = "", activation: str | None = None) -> dict[str, Any]: + self._require_connected() + arguments = {"interactSessionKey": self._session_key, "interactionCommand": command} + if activation: + arguments["activationBundleId"] = activation + return await self._bridge.call("DeviceInteractionSynthesize", arguments) + + def _require_connected(self) -> None: + if not self._session_key or self._device_id == "booted": + raise RuntimeError("Connect the iOS simulator driver before interacting.") + + async def _capture(self) -> ScreenData: + result = await self._synthesize() + image_path = result.get("screenshotPath") + if not image_path: + raise RuntimeError("Xcode returned no screenshot; no observation is available.") + screenshot = await asyncio.to_thread(Path(image_path).read_bytes) + with Image.open(BytesIO(screenshot)) as image: + self._width, self._height = image.size + hierarchy_path = result.get("hierarchyPath") + if not hierarchy_path: + self._scale = None + raise RuntimeError( + "Xcode returned no accessibility hierarchy. Recapture after the UI settles." + ) + hierarchy = await asyncio.to_thread(Path(hierarchy_path).read_text, encoding="utf-8") + self._scale = None + elements, self._scale = parse_hierarchy(hierarchy, self._width, self._height) + return ScreenData( + screenshot_bytes=screenshot, + screenshot_base64=base64.b64encode(screenshot).decode("ascii"), + ui_elements=elements, + width=self._width, + height=self._height, + platform="ios", + ) + + async def get_screen_data(self, skip_settling: bool = False) -> ScreenData: + # Xcode captures after animations settle; an extra Android delay is unnecessary. + async with self._operation_lock: + return await self._capture() + + def _point(self, x: int, y: int) -> str: + if self._scale is None: + raise RuntimeError("Capture an iOS screen before coordinate interaction.") + if not 0 <= x < self._width or not 0 <= y < self._height: + raise ValueError("iOS input coordinates are outside the current screenshot.") + return f"{x / self._scale[0]:.4f} {y / self._scale[1]:.4f}" + + @staticmethod + def _activation_for(data: ScreenData, x: int, y: int) -> str | None: + """Native overlapping-app elements require activation before input.""" + candidates = [] + for element in data.ui_elements: + bundle = element.get("activation_bundle_id") + bounds = element.get("parsed_bounds") + if not bundle or not isinstance(bounds, dict): + continue + if bounds["left"] <= x < bounds["right"] and bounds["top"] <= y < bounds["bottom"]: + area = (bounds["right"] - bounds["left"]) * (bounds["bottom"] - bounds["top"]) + candidates.append((area, bundle)) + if not candidates: + return None + minimum = min(area for area, _ in candidates) + bundles = {bundle for area, bundle in candidates if area == minimum} + if len(bundles) != 1: + raise ValueError( + "The target overlaps multiple iOS applications; activate the intended app first." + ) + return bundles.pop() + + async def tap( + self, x: int, y: int, duration_ms: int = 100, times: int = 1, delay_ms: int = 100 + ) -> bool: + if times < 1 or duration_ms < 0 or delay_ms < 0: + raise ValueError("Tap count must be positive and durations nonnegative.") + async with self._operation_lock: + previous_size = self.screen_size + data = await self._capture() + if self.screen_size != previous_size: + raise ValueError( + "The iOS screen changed orientation or size. Observe it again before tapping." + ) + command = f"t {self._point(x, y)} {duration_ms / 1000:.3f}" + for index in range(times): + await self._synthesize(command, activation=self._activation_for(data, x, y)) + if index < times - 1: + await asyncio.sleep(delay_ms / 1000) + return True + + async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool: + return await self.tap(x, y, duration_ms=duration_ms) + + async def swipe( + self, start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 800 + ) -> bool: + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + previous_size = self.screen_size + data = await self._capture() + if self.screen_size != previous_size: + raise ValueError( + "The iOS screen changed orientation or size. Observe it again before swiping." + ) + await self._synthesize( + f"t {self._point(start_x, start_y)} f {self._point(end_x, end_y)} {duration_ms / 1000:.3f}", + activation=self._activation_for(data, start_x, start_y), + ) + return True + + async def swipe_direction( + self, + direction: SwipeDirection | Literal["up", "down", "left", "right"], + duration_ms: int = 800, + ) -> bool: + direction = SwipeDirection(direction).value + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + data = await self._capture() + w, h = self.screen_size + points = { + "up": (w // 2, h * 3 // 4, w // 2, h // 4), + "down": (w // 2, h // 4, w // 2, h * 3 // 4), + "left": (w * 3 // 4, h // 2, w // 4, h // 2), + "right": (w // 4, h // 2, w * 3 // 4, h // 2), + } + sx, sy, ex, ey = points[direction] + await self._synthesize( + f"t {self._point(sx, sy)} f {self._point(ex, ey)} {duration_ms / 1000:.3f}", + activation=self._activation_for(data, sx, sy), + ) + return True + + async def input_text(self, text: str, clear_existing: bool = True) -> bool: + if clear_existing: + raise NotImplementedError( + "Xcode has no verified replace-text operation. Use clear_existing=False to append, or clear the field through its UI." + ) + # Encode every code point: literal native escape sequences cannot become + # unintended control characters, and newlines/Unicode retain their value. + escaped = "".join(f"\\u{{{ord(char):04X}}}" for char in text) + async with self._operation_lock: + await self._synthesize(f"sender keyboard kbd {escaped}") + return True + + async def press_key(self, key: KeyCode | str | int) -> bool: + key = key.value if isinstance(key, KeyCode) else str(key).lower() + commands = { + "home": "b h", + "power": "b p", + "volume_up": "b u", + "volume_down": "b d", + "app_switch": "b h b h", + "enter": r"sender keyboard kbd \u{000A}", + } + if key not in commands: + raise NotImplementedError(f"Key {key!r} is not supported by the native iOS driver.") + async with self._operation_lock: + await self._synthesize(commands[key]) + return True + + async def launch_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "launch", self._device_id, package_name) + return True + + async def stop_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "terminate", self._device_id, package_name) + return True + + async def install_app(self, app_path: Path) -> str: + self._require_connected() + path = app_path.expanduser().resolve() + if path.suffix != ".app" or not path.is_dir(): + raise ValueError("iOS installation requires a simulator-built .app directory.") + with (path / "Info.plist").open("rb") as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + if not isinstance(bundle, str) or not bundle: + raise ValueError("The .app has no CFBundleIdentifier in Info.plist.") + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "install", self._device_id, str(path), timeout=120) + return bundle + + async def list_apps(self) -> dict[str, str]: + async with self._operation_lock: + self._require_connected() + data = plistlib.loads(await run_xcrun("simctl", "listapps", self._device_id)) + return { + bundle: info.get("CFBundleDisplayName") or info.get("CFBundleName") or bundle + for bundle, info in data.items() + } + + async def open_url(self, url: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "openurl", self._device_id, url) + return True + + async def get_current_package(self) -> str | None: + async with self._operation_lock: + result = await self._synthesize() + path = result.get("hierarchyPath") + if not path: + return None + hierarchy = await asyncio.to_thread(Path(path).read_text, encoding="utf-8") + return application_bundle(hierarchy) + + async def find_element( + self, query: ElementQuery, screen_data: ScreenData | None = None + ) -> tuple[dict[str, Any] | None, list[int] | None, str | None]: + element, center, error = await super().find_element(query, screen_data) + if element is not None: + center = element.get("hit_point", center) + return element, center, error + + async def tap_element( + self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 + ) -> bool: + async with self._operation_lock: + data = await self._capture() + element, center, error = await self.find_element(query, data) + if error or element is None or center is None: + return False + await self._synthesize( + f"t {self._point(*center)} {duration_ms / 1000 if long_press else 0.1:.3f}", + activation=element.get("activation_bundle_id"), + ) + return True + + async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: + raise NotImplementedError("Android shell commands are unavailable on iOS Simulator.") + + async def start_video_recording(self, output_dir: Path | None = None) -> None: + raise NotImplementedError("Video recording is not yet integrated for iOS Simulator.") + + async def stop_video_recording(self) -> str | None: + return None diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 89dca25a..65e4196c 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -20,6 +20,7 @@ from langchain_core.callbacks.base import Callbacks from artemis.config import checker_overrides_for_level, settings +from artemis.context import DevicePlatform from artemis.utils.startup_progress import publish_startup_progress from third_party.mobile_use.main import ( GoalArgument, @@ -63,6 +64,7 @@ async def execute_task( explorer_flash_mode: str | None = None, explorer_pro_mode: str | None = None, verification_level: str | None = None, + platform: DevicePlatform = DevicePlatform.ANDROID, ) -> None: """Executes a single mobile automation task end-to-end. @@ -84,6 +86,9 @@ async def execute_task( verification_level: Coarse Checker preset ('off', 'final', 'checkpoints', 'strict'); applied before the explicit ``enable_checker`` switch. """ + platform = DevicePlatform(platform) + if platform == DevicePlatform.IOS and os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS Simulator support is local only; cloud mode targets Android.") effective_sid = ( session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") ) @@ -134,10 +139,15 @@ async def execute_task( pro_mode=explorer_pro_mode, ) - target_serial = ( - device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") - ) - if not target_serial: + target_serial = device_serial + if platform == DevicePlatform.IOS: + target_serial = target_serial or "booted" + config.with_video_recording_tools(enabled=False) + else: + target_serial = ( + target_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") + ) + if not target_serial and platform == DevicePlatform.ANDROID: try: from artemis.runtime import device_pool @@ -146,9 +156,7 @@ async def execute_task( target_serial = None if target_serial: - from artemis.context import DevicePlatform - - config.for_device(DevicePlatform.ANDROID, target_serial) + config.for_device(platform, target_serial) if graph_config_callbacks: config.with_graph_config_callbacks(graph_config_callbacks) @@ -176,6 +184,15 @@ def run_command( help="Execution profile ('flash' for fast reactive, 'pro' for full graph).", ), ] = "pro", + platform: Annotated[ + DevicePlatform, + typer.Option( + "--platform", + metavar="PLATFORM", + rich_help_panel="Platform", + help="Target mobile platform: Android (default) or a local iOS simulator.", + ), + ] = DevicePlatform.ANDROID, locked_app_package: Annotated[ str | None, typer.Option( @@ -192,7 +209,7 @@ def run_command( str | None, typer.Option( "--app-path", - help="Local APK path to install before starting the task.", + help="Local Android APK or iOS simulator .app directory to install before the task.", ), ] = None, enable_planner_validation: Annotated[ @@ -278,7 +295,7 @@ def run_command( typer.Option( "--device-serial", "-s", - help="Target specific Android device by serial number (e.g. emulator-5554).", + help="Android serial or iOS simulator UDID; iOS defaults to the single 'booted' simulator.", ), ] = None, session_id: Annotated[ @@ -296,8 +313,7 @@ def run_command( ), ] = False, ) -> None: - """Run an autonomous UI automation task on the connected Android device.""" - ensure_video_recording_available(with_video_recording_tools) + """Run an autonomous UI automation task on an Android device or iOS simulator.""" console = Console() @@ -307,6 +323,26 @@ def run_command( ) is_standalone = standalone or os.environ.get("ARTEMIS_STANDALONE") == "1" + if platform == DevicePlatform.IOS: + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise typer.BadParameter( + "iOS Simulator support is local only; cloud mode targets Android." + ) + if is_worker: + raise typer.BadParameter( + "The daemon and device queue support Android only; run iOS tasks locally." + ) + if locked_app_package: + raise typer.BadParameter("--locked-app is unavailable for iOS simulator tasks.") + if with_video_recording_tools: + raise typer.BadParameter( + "iOS video analysis is unavailable; omit --with-video-recording-tools." + ) + is_standalone = True + console.print("[dim]Running the iOS simulator task in standalone mode.[/dim]") + else: + ensure_video_recording_available(with_video_recording_tools) + # All platforms route through unified Artemis Daemon unless specifically configured as standalone if not is_worker and not is_standalone: try: @@ -390,7 +426,8 @@ def on_status(sess_info): f"[yellow]Daemon routing notice: {exc}. Falling back to local execution...[/yellow]" ) - display_local_device_status(console, host=settings.ADB_HOST, port=settings.ADB_PORT) + if platform == DevicePlatform.ANDROID: + display_local_device_status(console, host=settings.ADB_HOST, port=settings.ADB_PORT) cancelled = False original_sigterm = None @@ -427,6 +464,7 @@ def on_status(sess_info): explorer_flash_mode=explorer_flash_mode, explorer_pro_mode=explorer_pro_mode, verification_level=verification_level, + platform=platform, ) ) except (KeyboardInterrupt, asyncio.CancelledError): diff --git a/artemis/mcp/action_session.py b/artemis/mcp/action_session.py index 06e512ca..f5f58ca0 100644 --- a/artemis/mcp/action_session.py +++ b/artemis/mcp/action_session.py @@ -233,7 +233,14 @@ async def get_action_session(ctx: Any, actuator: Any = None) -> ActionSession: from artemis.mcp.action_server import build_action_server from artemis.mcp.actuators.adb import AdbActuator - actuator = actuator or getattr(ctx, "actuator", None) or AdbActuator(ctx) + actuator = actuator or getattr(ctx, "actuator", None) + if actuator is None: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + from artemis.mcp.actuators.ios import IosActuator + + actuator = IosActuator(ctx) + else: + actuator = AdbActuator(ctx) server = build_action_server(actuator) session = ActionSession(server) await session.start() diff --git a/artemis/mcp/actuators/ios.py b/artemis/mcp/actuators/ios.py new file mode 100644 index 00000000..57b0eef6 --- /dev/null +++ b/artemis/mcp/actuators/ios.py @@ -0,0 +1,79 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS actuator sharing Artemis's normalized action contract.""" + +from artemis.mcp.action_types import ActionCode, ActionResult +from artemis.mcp.action_manifest import DEVICE_ACTIONS +from artemis.mcp.actuators.adb import AdbActuator, ensure_focus_at_coords + + +class IosActuator(AdbActuator): + """Reuse controller actions while keeping Android shell conventions out of iOS.""" + + def capabilities(self) -> frozenset[str]: + return DEVICE_ACTIONS - {"erase_one_char", "focus_and_clear_text"} + + def _dims(self) -> tuple[int, int]: + # The simulator may rotate after SDK initialization. Its most recent + # screenshot dimensions own the coordinate space for the next action. + return self.controller.driver.screen_size + + async def input_text( + self, text: str, target: tuple[int, int] | None = None, clear_exist: bool = True + ) -> ActionResult: + if clear_exist: + return ActionResult.failure( + "input_text", + "Whole-field clearing is unavailable on iOS. Focus an empty field or place " + "the cursor manually, then set clear_exist=false.", + code=ActionCode.UNSUPPORTED, + ) + if target: + error = await ensure_focus_at_coords(self.controller, *self._to_px(*target)) + if error: + return ActionResult.failure("input_text", error) + success = await self.controller.type_text(text, clear_existing=clear_exist) + if not success: + return ActionResult.failure("input_text", "Failed to type text on the iOS simulator.") + return ActionResult.success("input_text", f"Typed '{text}'.") + + async def press_key(self, key: str) -> ActionResult: + if await self.controller.press_key(key): + return ActionResult.success("press_key", f"Pressed key '{key}'.") + return ActionResult.failure( + "press_key", f"iOS key '{key}' is unsupported or failed.", code=ActionCode.UNSUPPORTED + ) + + async def manage_app(self, action: str, app_name: str) -> ActionResult: + if action.lower() not in {"launch", "stop"}: + return ActionResult.failure( + "manage_app", f"Invalid manage_app action: {action}", code=ActionCode.INVALID_ARGS + ) + from artemis.tools.mobile.launch_app import find_package + + bundle_id = await find_package(self.ctx, app_name, use_fallback=False) + if not bundle_id: + return ActionResult.failure( + "manage_app", + f"Installed iOS app not found: {app_name}", + code=ActionCode.PACKAGE_NOT_FOUND, + ) + if action.lower() == "launch": + success = await self.controller.launch_app(bundle_id) + else: + success = await self.controller.terminate_app(bundle_id) + if not success: + return ActionResult.failure("manage_app", f"Failed to {action} iOS app '{bundle_id}'.") + return ActionResult.success("manage_app", f"Dispatched {action} for iOS app '{bundle_id}'.") diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index c0c0da08..e28395d9 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -13,16 +13,20 @@ # limitations under the License. import asyncio +import contextlib import inspect import os import re +import threading import uuid +from io import BytesIO from pathlib import Path from dotenv import load_dotenv from google import genai from langchain_google_genai import ChatGoogleGenerativeAI +from PIL import Image from artemis.clients.screen_client_factory import ( describe_backend, @@ -36,17 +40,18 @@ ) from artemis.context import ( ArtemisContext, + DeviceContext, DevicePlatform, ExecutionSetup, ) from artemis.data_engine.engine import DataEngine from artemis.graph.state import State -from artemis.runtime import trace_store +from artemis.runtime import DeviceExecutionLock, trace_store from artemis.runtime.cancel_requests import watch_for_cancel_request from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix from artemis.sdk.types.agent import AgentConfig from artemis.utils.startup_progress import publish_startup_progress -from third_party.mobile_use.sdk.types.exceptions import AgentError +from third_party.mobile_use.sdk.types.exceptions import AgentError, AgentNotInitializedError from third_party.mobile_use.sdk.types.task import Task from third_party.mobile_use.sdk.agent import AgentBase, TOutput from third_party.mobile_use.utils.logger import get_logger @@ -113,7 +118,7 @@ def __init__( updates = {} if target_dev: updates["device_id"] = target_dev - updates["device_platform"] = DevicePlatform.ANDROID + updates["device_platform"] = config.device_platform or DevicePlatform.ANDROID if concurrency_mode: updates["concurrency_mode"] = str(concurrency_mode).strip().lower() if max_concurrency is not None: @@ -125,6 +130,141 @@ def __init__( self._tmp_traces_dir = Path(settings.TRACES_PATH) self._initialized = False self._task_lock = asyncio.Lock() + self._ios_driver = None + self._adb_client = None + self._ui_adb_client = None + + async def _init_internal( + self, + api_key: str | None = None, + retry_count: int = 5, + retry_wait_seconds: int = 5, + ): + if self._config.device_platform != DevicePlatform.IOS: + return await super()._init_internal(api_key, retry_count, retry_wait_seconds) + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise AgentError("iOS Simulator support is local only; cloud mode targets Android.") + if self._initialized: + return True + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + publish_startup_progress( + "device_check", "Checking the iOS simulator", session_id=self._session_id + ) + driver = XcodeSimulatorDriver(device_id=self._config.device_id or "booted") + self._ios_driver = driver + try: + # Resolve the simulator without booting it or opening a native UI + # session. Mutating setup waits for run_task's execution lease. + await driver.resolve_device() + except (OSError, ValueError, RuntimeError, TimeoutError, asyncio.CancelledError): + await driver.disconnect() + self._ios_driver = None + raise + width, height = driver.screen_size + self._device_context = DeviceContext( + host_platform="DARWIN", + mobile_platform=DevicePlatform.IOS, + device_id=driver.device_id, + device_width=width, + device_height=height, + ) + # Android recording and read-only ADB probes have no iOS equivalent yet. + self._config = self._config.model_copy( + update={"video_recording_tools_enabled": False, "disable_device_probes": True} + ) + publish_startup_progress( + "device_ready", "iOS simulator selected", session_id=self._session_id + ) + asyncio.create_task(self._prewarm_llm_connections(api_key)) + self._initialized = True + return True + + @contextlib.asynccontextmanager + async def _ios_operation(self): + """Serialize one public iOS SDK call behind the device execution lease. + + Called from ``run_task`` itself (internal app installation), the helper + reuses the lease and native session the task already holds. Called + publicly, it acquires the same FIFO device lease a task would, then + opens a short-lived native session that is always closed on exit. + """ + driver = self._ios_driver + if not self._initialized or driver is None: + raise AgentNotInitializedError() + if asyncio.current_task() is getattr(self, "_current_task", None): + yield driver + return + async with self._task_lock: + if not self._initialized or self._ios_driver is not driver: + raise AgentNotInitializedError() + device_lock = DeviceExecutionLock( + driver.device_id, + description="Artemis iOS device operation", + concurrency_mode=getattr(self._config, "concurrency_mode", "per_device"), + max_concurrency=getattr(self._config, "max_concurrency", None), + session_id=self._session_id, + ingress="sdk", + ) + queue_cancel_event = threading.Event() + acquire_task = asyncio.create_task( + asyncio.to_thread(device_lock.acquire, cancel_event=queue_cancel_event) + ) + connect_attempted = False + try: + try: + await asyncio.shield(acquire_task) + except asyncio.CancelledError: + queue_cancel_event.set() + try: + await asyncio.shield(acquire_task) + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + # Draining the lock acquisition after cancellation is best effort. + logger.debug( + f"Device lock acquisition drain after cancel failed: {exc}", + exc_info=True, + ) + raise + connect_attempted = True + await driver.connect() + ( + self._device_context.device_width, + self._device_context.device_height, + ) = driver.screen_size + yield driver + finally: + try: + if connect_attempted: + try: + await driver.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug( + f"iOS driver disconnect after SDK operation failed: {exc}", + exc_info=True, + ) + finally: + await asyncio.to_thread(device_lock.release) + + async def _install_app_internal(self, app_path: str | Path) -> str | None: + if self._config.device_platform != DevicePlatform.IOS: + return await super()._install_app_internal(app_path) + async with self._ios_operation() as driver: + return await driver.install_app(Path(app_path)) + + async def _prepare_app_lock(self, task: Task, context: ArtemisContext): + if context.device.mobile_platform == DevicePlatform.IOS and task.request.locked_app_package: + raise AgentError( + "iOS app locking is unavailable because foreground ownership cannot be verified." + ) + return await super()._prepare_app_lock(task, context) + + async def get_screenshot(self): + if self._config.device_platform != DevicePlatform.IOS: + return await super().get_screenshot() + async with self._ios_operation() as driver: + data = await driver.get_screen_data() + with Image.open(BytesIO(data.screenshot_bytes)) as image: + return image.copy() async def _prewarm_llm_connections(self, api_key: str | None = None): """Pre-warms the HTTP2/gRPC connection pools for both Native GenAI and LangChain clients in the background.""" @@ -200,6 +340,10 @@ def _on_cancel() -> None: logger.debug(f"[{task_name}] Cancel watcher stopped: {exc}") async def clean(self, force: bool = False): + driver = getattr(self, "_ios_driver", None) + if driver is not None: + await driver.disconnect() + self._ios_driver = None if not self._initialized and not force: return @@ -210,6 +354,14 @@ async def clean(self, force: bool = False): async def _ensure_device_unlocked(self) -> None: """Reject secure keyguard instead of allowing an agent to guess credentials.""" + driver = getattr(self, "_ios_driver", None) + if driver is not None: + # A previous task may have closed its native transport. + await driver.connect() + self._device_context.device_width, self._device_context.device_height = ( + driver.screen_size + ) + return if self._adb_client is None: raise AgentError("ADB client is not initialized.") @@ -234,6 +386,8 @@ async def _ensure_device_unlocked(self) -> None: async def _prepare_device_environment(self, context: ArtemisContext): """Prepare device environment flags (like forcing Web Accessibility) before the task runs.""" + if context.device.mobile_platform == DevicePlatform.IOS: + return if not self._config.force_web_accessibility: logger.info( "Forcing web accessibility is disabled in AgentConfig. Skipping" @@ -282,6 +436,12 @@ async def _prepare_device_environment(self, context: ArtemisContext): def _prepare_tracing(self, task: Task, context: ArtemisContext): """Prepare tracing and data engine setup.""" + driver = getattr(self, "_ios_driver", None) + if driver is not None: + from artemis.mcp.actuators.ios import IosActuator + + context._active_driver = driver + context.actuator = IosActuator(context) task_name = self._prepare_trace_paths(task) context.execution_setup = ExecutionSetup( diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 16893864..c982a3d8 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -76,6 +76,10 @@ def for_device_serial(self, device_serial: str) -> "AgentConfigBuilder": """Target a specific Android device by its ADB serial number.""" return self.for_device(DevicePlatform.ANDROID, device_serial) + def for_ios_simulator(self, device_id: str = "booted") -> "AgentConfigBuilder": + """Target an iOS simulator UDID, or the single booted simulator.""" + return self.for_device(DevicePlatform.IOS, device_id) + def with_concurrency_mode(self, mode: str) -> "AgentConfigBuilder": """Configure concurrency mode: 'global' (1 task globally) or 'per_device' (1 task per device).""" self._concurrency_mode = str(mode).strip().lower() diff --git a/artemis/tools/index.py b/artemis/tools/index.py index cf1762ce..6d71ecb6 100644 --- a/artemis/tools/index.py +++ b/artemis/tools/index.py @@ -27,9 +27,16 @@ def get_tools_from_wrappers( wrappers: list[ToolWrapper], ) -> list[BaseTool]: """Instantiate the available tools of ``wrappers``, traced into the data engine.""" - return build_tools_from_wrappers( + tools = build_tools_from_wrappers( ctx, wrappers, wrap_tool=lambda t: trace_langchain_tool(t, ctx) ) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + tools = [ + t + for t in tools + if t.name not in {"run_adb_command", "manage_task", "analyze_task_output"} + ] + return tools def get_tool_by_name(name: str, tools: list[BaseTool]) -> BaseTool | None: diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 23cde2c7..ca47d9fb 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -18,17 +18,35 @@ from langchain_core.tools import BaseTool from pydantic import BaseModel, Field -from artemis.context import ArtemisContext +from artemis.context import ArtemisContext, DevicePlatform from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper from artemis.utils.cython_compat import CyFunctionDetector -from third_party.mobile_use.tools.mobile.launch_app import LAUNCH_APP_DOCSTRING, find_package +from third_party.mobile_use.tools.mobile.launch_app import ( + LAUNCH_APP_DOCSTRING, + find_package as find_android_package, +) from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries +async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = True) -> str | None: + """Resolve an installed Android package or iOS bundle identifier.""" + if getattr(getattr(ctx, "device", None), "mobile_platform", None) != DevicePlatform.IOS: + return await find_android_package(ctx, app_name, use_fallback=use_fallback) + from artemis.drivers.factory import get_driver + + apps = await get_driver(ctx).list_apps() + if app_name in apps: + return app_name + matches = [ + bundle_id for bundle_id, name in apps.items() if name.casefold() == app_name.casefold() + ] + return matches[0] if len(matches) == 1 else None + + class LaunchAppArgs(BaseModel): """Arguments schema for launching an application.""" @@ -82,11 +100,24 @@ async def execute( outcome = f"Failed to launch app '{app}': Package not found." error_msg = "Package not found." else: - success, error_msg = await launch_app_with_retries( - ctx=ctx, app_package=package_name - ) + if ( + getattr(getattr(ctx, "device", None), "mobile_platform", None) + == DevicePlatform.IOS + ): + from artemis.drivers.factory import get_driver + + success = await get_driver(ctx).launch_app(package_name) + error_msg = None if success else "Launch failed." + else: + success, error_msg = await launch_app_with_retries( + ctx=ctx, app_package=package_name + ) outcome = ( - f"Launched app '{app}' ({package_name}); foreground confirmed." + f"Launched app '{app}' ({package_name})." + if success + and getattr(getattr(ctx, "device", None), "mobile_platform", None) + == DevicePlatform.IOS + else f"Launched app '{app}' ({package_name}); foreground confirmed." if success else f"Failed to launch app '{app}': {error_msg}" ) diff --git a/artemis/tools/mobile/read_hierarchy.py b/artemis/tools/mobile/read_hierarchy.py index 02930e82..229bd7ce 100644 --- a/artemis/tools/mobile/read_hierarchy.py +++ b/artemis/tools/mobile/read_hierarchy.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect +import json from typing import Any from langchain_core.tools import BaseTool @@ -76,6 +77,8 @@ async def execute( xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None) if xml_hierarchy: return str(xml_hierarchy) + if getattr(screen_data, "platform", None) == "ios": + return json.dumps(screen_data.ui_elements, ensure_ascii=False) return ToolFailure("Error retrieving UI hierarchy: No UI hierarchy in screen data.") if driver is not None and hasattr(driver, "get_ui_hierarchy"): xml_hierarchy = await driver.get_ui_hierarchy() @@ -98,7 +101,10 @@ async def execute( def get_ui_hierarchy_tool(ctx: ArtemisContext) -> BaseTool: """Exports get_ui_hierarchy as a LangChain BaseTool.""" - return trace_langchain_tool(get_ui_hierarchy.to_langchain_tool(ctx), ctx) + tool = get_ui_hierarchy.to_langchain_tool(ctx) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + tool.description = "[DIAGNOSTIC] Retrieves the current iOS accessibility elements as JSON." + return trace_langchain_tool(tool, ctx) ui_hierarchy_wrapper = ToolWrapper( diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py index c6c2ddd2..68126481 100644 --- a/artemis/utils/visualization.py +++ b/artemis/utils/visualization.py @@ -13,6 +13,7 @@ # limitations under the License. import io +import math import os import re from typing import Any @@ -170,6 +171,38 @@ def get_center_coordinates(left: int, top: int, right: int, bottom: int) -> tupl return (left + right) // 2, (top + bottom) // 2 +def get_hit_point( + node: dict[str, Any], width: int | None = None, height: int | None = None +) -> tuple[int, int] | None: + """Return a valid native interaction point, when the driver supplied one.""" + point = node.get("hit_point") + if not isinstance(point, (list, tuple)) or len(point) != 2: + return None + try: + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + for value in point + ): + return None + except OverflowError: + return None + x, y = map(round, point) + if x < 0 or y < 0 or (width is not None and x >= width) or (height is not None and y >= height): + return None + return x, y + + +def _native_interaction_metadata(node: dict[str, Any], width: int, height: int) -> dict[str, Any]: + metadata: dict[str, Any] = {} + if (point := get_hit_point(node, width, height)) is not None: + metadata["hit_point"] = list(point) + if isinstance(bundle := node.get("activation_bundle_id"), str) and bundle: + metadata["activation_bundle_id"] = bundle + return metadata + + def _inject_mutual_occlusion_warnings( lines: list[str], items_bounds: list[tuple[int, tuple[int, int, int, int]]] ) -> list[str]: @@ -298,7 +331,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: ocr_bounds = parse_bounds(ocr.get("bounds")) if ocr_bounds: left, top, right, bottom = ocr_bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(ocr["text"], cx, cy): continue @@ -314,7 +349,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: if not registered and text.strip() and bounds: left, top, right, bottom = bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(text, cx, cy): continue @@ -374,7 +411,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: ocr_bounds = parse_bounds(ocr.get("bounds")) if ocr_bounds: left, top, right, bottom = ocr_bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(ocr["text"], cx, cy): continue @@ -389,8 +428,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: "text": ocr["text"].strip(), "bounds": [left, top, right, bottom], "class": node.get("class"), - "resource_id": node.get("resource-id"), + "resource_id": node.get("resource-id") or node.get("resource_id"), "is_ocr": True, + **_native_interaction_metadata(node, width, height), } ) labels.append(str(idx)) @@ -406,7 +446,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: shown = text.strip() or hint if not registered and shown and bounds: left, top, right, bottom = bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(shown, cx, cy): continue @@ -424,8 +466,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: "text": shown, "bounds": [left, top, right, bottom], "class": node.get("class"), - "resource_id": node.get("resource-id"), + "resource_id": node.get("resource-id") or node.get("resource_id"), "is_ocr": False, + **_native_interaction_metadata(node, width, height), } if not text.strip(): element["is_hint"] = True diff --git a/docs/ios.md b/docs/ios.md new file mode 100644 index 00000000..b3cff15b --- /dev/null +++ b/docs/ios.md @@ -0,0 +1,190 @@ +# iOS Simulator support + +Artemis can run Flash and Pro tasks on iOS simulators on macOS with Xcode 27 or +newer. Select iOS explicitly: the existing Android defaults still apply on a +Mac. The first implementation supports the standalone CLI and embedded Python +SDK. The web console, Artemis daemon, remote `artemis-client`, and Artemis MCP +server currently use the Android device path. + +The iOS driver uses tools included with Xcode and Artemis's existing Python MCP +dependency. It communicates with `xcrun mcpbridge` using an initialized MCP +session; it uses `xcrun simctl` for simulator and app lifecycle operations. +Appium, WebDriverAgent, and third-party simulator control utilities are not +required. Apple added native agent device interactions in Xcode 27; see the +[Xcode 27 release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). + +## Prepare Xcode + +1. Install Xcode 27 or newer, complete its first-launch setup, and install an iOS + simulator runtime. Confirm that `xcodebuild -version` reports the intended + version and that `xcode-select -p` points to that Xcode installation. You can + also set `DEVELOPER_DIR` for the Artemis process when multiple Xcode versions + are installed. +2. Enable external agent access using Apple's + [Xcode MCP access instructions](https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode) + and review any Xcode approval prompts. For the workspace-independent MCP + server introduced in Xcode 27, follow Apple's enablement instructions in the + [release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). + Artemis does not enable the server or grant permissions automatically. +3. Check the toolchain and available simulators from the repository root: + + ```bash + bash scripts/setup_ios_env.sh + ``` + + This check reads the Xcode version, tool locations, simulator inventory, and + `xcrun mcp-server status`. It does not install dependencies, boot a simulator, + or change access settings. A passing check confirms prerequisites; the first + driver connection checks native MCP tool availability and device access. +4. Install Artemis's Python dependencies and configure a model provider using + the normal project configuration: + + ```bash + uv sync --dev + cp .env.example .env + ``` + + Fill in the provider credentials in `.env` and select the desired models in + `config/artemis.jsonc`. The one-click Android startup and dependency installer + are not needed for this iOS workflow. + +## Run a task + +List the simulators to find the UDID for an available iOS device: + +```bash +xcrun simctl list devices available +``` + +Run a standalone task on that simulator: + +```bash +uv run artemis run "Open Settings and inspect the General screen" \ + --platform ios --standalone --device-serial --profile flash +``` + +An explicitly selected simulator is booted if necessary, and the driver waits +for it to finish booting. To use a simulator that is already running, omit +`--device-serial` or pass `--device-serial booted`. Exactly one available iOS +simulator must be booted; with zero or multiple booted devices, choose a UDID. +The driver resolves `booted` once and pins every later command to that UDID. + +Use `--profile pro` to run the planner, operator, and verification workflow on +the same driver. Prompts and app identifiers should refer to iOS apps and bundle +identifiers, such as `com.apple.Preferences`. + +To install a simulator build before the task, add +`--app-path /absolute/path/MyApp.app`. The bundle's `Info.plist` must provide +`CFBundleIdentifier`. + +## Embedded Python SDK + +Configure iOS through the embedded SDK's builder: + +```python +from artemis.sdk import Agent +from artemis.sdk.builders import AgentConfigBuilder + +config = ( + AgentConfigBuilder() + .for_ios_simulator("") + .with_default_profile("flash") + .build() +) +agent = Agent(config=config) +``` + +The generic builder also accepts +`for_device(DevicePlatform.IOS, "")`, with `DevicePlatform` +imported from `artemis.context`. Supplying only a `device_serial` without an iOS +configuration retains the existing Android selection behavior. + +A minimal run looks like: + +```python +agent = Agent(config=config) +try: + await agent.init() + result = await agent.run_task(goal="Open Settings", profile="flash") + screenshot = await agent.get_screenshot() +finally: + await agent.clean() +``` + +`init` resolves the simulator without opening a native session. Public +`get_screenshot()` and `install_app()` calls acquire the device execution +lease and own a short-lived native session that is closed before they return, +so they work both right after `init` and between tasks. An app installation +requested inside `run_task` reuses the task's existing lease and session +instead of acquiring a second one. + +## Supported operations and limits + +| Operation | iOS implementation | +| --- | --- | +| Device selection and boot | `simctl` inventory, explicit UDID, `boot`, and `bootstatus` | +| Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | +| Tap, long press, and swipe | Native synthesized touch events | +| Text entry | Native keyboard synthesis with `clear_exist=false` | +| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | +| App install, launch, and terminate | `simctl` using simulator `.app` bundles and bundle identifiers | + +The native hierarchy is normalized into the element tree used by Artemis's +perception and action tools. Screenshots and touch coordinates are kept in the +same coordinate space. Custom UI without accessible elements still relies on +visual targeting. + +This driver targets iOS simulators. Physical iPhones and iPads, watchOS, tvOS, +and visionOS are outside this implementation. Install a simulator build of an +app; a device `.ipa` or Android `.apk` is not interchangeable with a simulator +`.app` bundle. The driver does not build Xcode projects. + +iOS has no system Back button. Use the app's visible navigation controls. +The native driver supports `enter`, `home`, `power`, `volume_up`, `volume_down`, +and `app_switch`. The `erase_one_char` and `focus_and_clear_text` actions are +not exposed. The `back` and `delete` keys and automatic replacement of existing +text fail with an explicit unsupported-operation error. When calling the `input_text` action, +pass `clear_exist=false`; the default requests text replacement. In direct +driver or controller calls, the equivalent argument is `clear_existing=False`. +To replace text, clear the field using its visible UI first. +Android shell commands, Logcat, Android resource identifiers, Android package +discovery, and the Android Accessibility Helper are unavailable on iOS. +Platform-specific operations fail explicitly when unsupported. + +Screen recording, video analysis, and video-based replay are not supported by +this initial iOS driver. Use screenshots and the task trace to review a run. + +## Troubleshooting + +- **Xcode version or tools are incorrect:** inspect `xcodebuild -version`, + `xcode-select -p`, and any `DEVELOPER_DIR` override. Command Line Tools alone + do not provide the full Xcode simulator and device-interaction environment. +- **MCP tool missing or access denied:** follow Apple's MCP access instructions, + review approval prompts, and check `xcrun mcp-server status`. Xcode 27 release + notes mention that some settings may require relaunching Xcode or restarting + the Mac. If Xcode requests workspace approval before allowing a session, open + and approve the intended workspace through Xcode's normal access flow. +- **CoreSimulator inventory fails:** run `xcrun simctl list devices available` + in a local terminal. Sandboxed processes need access to the simulator service + and its device data. Install a runtime and create a simulator if the iOS + inventory is empty. +- **Multiple booted simulators:** supply `--device-serial` with the intended + simulator's UDID. +- **Android daemon or device checks appear:** include `--platform ios + --standalone`; iOS tasks currently run in the embedded process. + +## Contributing and validation + +Keep native MCP and subprocess interactions behind the iOS driver. Unit tests +should use fake MCP sessions and command runners so the normal deterministic +suite remains usable on Linux and without Xcode or model credentials. Run +`make test`, `make lint`, and `make typecheck` before submitting a change. + +For live acceptance on a configured Mac, run the prerequisite script and an +explicit standalone CLI task. Verify that the chosen simulator receives taps, +swipes, and text, and that the screenshot and hierarchy describe the same +screen. Test an ambiguous `booted` selection and denied MCP access as well as a +successful session, and confirm that the session is released when the task +ends or is cancelled. Record the Xcode version, simulator runtime, UDID, and +observed results in the pull request; unit tests alone do not establish live +device compatibility. diff --git a/pyright-core.json b/pyright-core.json index 3ae152e7..4252abfe 100644 --- a/pyright-core.json +++ b/pyright-core.json @@ -1,11 +1,13 @@ { "venvPath": ".", "venv": ".venv", + "pythonVersion": "3.12", "include": [ "artemis/graph/state.py", "artemis/graph/visibility.py", "artemis/runtime/adb_endpoint.py", "artemis/runtime/process_probe.py", + "artemis/drivers/ios", "artemis/mcp/action_types.py", "artemis/data_engine/models.py", "artemis/agents/explorer/native_runner.py", diff --git a/scripts/setup_ios_env.sh b/scripts/setup_ios_env.sh new file mode 100755 index 00000000..7f82fd14 --- /dev/null +++ b/scripts/setup_ios_env.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Read-only prerequisite checks. This script does not install tools, boot +# devices, select Xcode, or change Xcode's MCP permissions. +set -euo pipefail + +fail() { + printf 'Error: %s\n' "$1" >&2 + exit 1 +} + +[[ "$(uname -s)" == "Darwin" ]] || fail "iOS simulator support requires macOS." + +command -v xcodebuild >/dev/null 2>&1 || fail "Install Xcode 27 or newer." +command -v xcrun >/dev/null 2>&1 || fail "Xcode's xcrun is unavailable." + +if ! version_output=$(xcodebuild -version); then + fail "Cannot read the selected Xcode version. Check xcode-select -p." +fi +version=$(printf '%s\n' "$version_output" | awk '/^Xcode / {print $2; exit}') +[[ "$version" =~ ^([0-9]+)(\.[0-9]+)*$ ]] || fail "Cannot parse Xcode version: $version" +(( BASH_REMATCH[1] >= 27 )) || fail "Xcode 27 or newer is required (selected: $version)." + +if ! simctl_path=$(xcrun --find simctl); then + fail "The selected Xcode installation does not provide simctl." +fi +if ! bridge_path=$(xcrun --find mcpbridge); then + fail "The selected Xcode installation does not provide mcpbridge." +fi + +printf 'Xcode %s\nsimctl: %s\nmcpbridge: %s\n\n' "$version" "$simctl_path" "$bridge_path" + +if ! devices=$(xcrun simctl list devices available); then + fail "Cannot query CoreSimulator. Run this check in a local terminal with access to Xcode's simulator services." +fi +printf '%s\n\n' "$devices" +if ! printf '%s\n' "$devices" | awk ' + /^-- iOS / { ios = 1; next } + /^-- / { ios = 0 } + ios && /\([[:xdigit:]-]+\) \((Booted|Shutdown)\)/ { found = 1 } + END { exit !found } +'; then + fail "No available iOS simulator found. Install an iOS runtime and create a simulator in Xcode." +fi + +printf 'Xcode MCP server status:\n' +if ! xcrun mcp-server status; then + printf 'Could not read MCP server status. Check Xcode MCP access manually.\n' >&2 +fi + +printf '\nToolchain and simulator inventory checks passed.\n' +printf 'Before running Artemis, review and grant Xcode MCP access for your agent.\n' +printf 'This check does not verify that device-interaction permission is granted.\n' +printf 'See docs/ios.md and https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode\n' diff --git a/tests/unit/drivers/test_ios_bridge.py b/tests/unit/drivers/test_ios_bridge.py new file mode 100644 index 00000000..7afb6c09 --- /dev/null +++ b/tests/unit/drivers/test_ios_bridge.py @@ -0,0 +1,353 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MCP bridge ownership and failure behavior without launching Xcode.""" + +import asyncio +from contextlib import asynccontextmanager +import json +from types import SimpleNamespace + +from mcp.types import CallToolResult, TextContent +import pytest + +from artemis.drivers.ios import bridge +from artemis.drivers.ios.bridge import XcodeBridge + + +def response(data=None, *, text=None, error=False): + return CallToolResult( + content=[TextContent(type="text", text=text)] if text is not None else [], + structuredContent=data, + isError=error, + ) + + +class NativeSession: + def __init__(self): + self.events = [] + self.tools_calls = [] + self.tool_calls = [] + self.initialize_hook = None + self.call_hook = None + self.pages = { + None: (["DeviceInteractionStartSession"], "second"), + "second": (["DeviceInteractionSynthesize", "DeviceInteractionEndSession"], None), + } + + async def __aenter__(self): + self.events.append(("session-enter", asyncio.current_task())) + return self + + async def __aexit__(self, *args): + self.events.append(("session-exit", asyncio.current_task())) + + async def initialize(self): + self.events.append(("initialize", asyncio.current_task())) + if self.initialize_hook: + await self.initialize_hook() + + async def list_tools(self, cursor=None): + self.tools_calls.append(cursor) + names, next_cursor = self.pages[cursor] + return SimpleNamespace( + tools=[SimpleNamespace(name=name) for name in names], nextCursor=next_cursor + ) + + async def call_tool(self, name, arguments): + self.tool_calls.append((name, arguments)) + if self.call_hook: + return await self.call_hook(name, arguments) + return response({"userMessage": "OK"}) + + +@pytest.fixture +def native_transport(monkeypatch): + session = NativeSession() + events = [] + parameters = [] + + @asynccontextmanager + async def stdio(params): + parameters.append(params) + events.append(("stdio-enter", asyncio.current_task())) + try: + yield object(), object() + finally: + events.append(("stdio-exit", asyncio.current_task())) + + monkeypatch.setattr(bridge, "stdio_client", stdio) + monkeypatch.setattr(bridge, "ClientSession", lambda reader, writer: session) + return SimpleNamespace(session=session, events=events, parameters=parameters) + + +def assert_resources_closed_in_owner(native): + assert [event for event, task in native.events] == ["stdio-enter", "stdio-exit"] + assert [event for event, task in native.session.events if event.startswith("session-")] == [ + "session-enter", + "session-exit", + ] + owners = {task for event, task in native.events + native.session.events} + assert len(owners) == 1 + + +@pytest.mark.asyncio +async def test_initializes_discovers_paginated_tools_and_preserves_xcode_selection( + native_transport, monkeypatch +): + monkeypatch.setenv("DEVELOPER_DIR", "/Applications/Selected Xcode.app/Contents/Developer") + monkeypatch.setenv("MCP_XCODE_PID", "1234") + client = XcodeBridge() + await client.start() + assert client.connected + assert client.tools == { + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", + } + assert native_transport.session.tools_calls == [None, "second"] + params = native_transport.parameters[0] + assert params.command == "xcrun" and params.args == ["mcpbridge"] + assert params.env["DEVELOPER_DIR"] == "/Applications/Selected Xcode.app/Contents/Developer" + assert params.env["MCP_XCODE_PID"] == "1234" + assert await client.call("DeviceInteractionEndSession", {"interactionSessionKey": "key"}) == { + "userMessage": "OK" + } + await client.close() + assert not client.connected + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "native_response", + [ + response({"screenshotPath": "/tmp/native.png"}), + response(text=json.dumps({"screenshotPath": "/tmp/native.png"})), + ], +) +async def test_reads_structured_and_legacy_json_results(native_transport, native_response): + async def call(name, arguments): + return native_response + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + assert await client.call("DeviceInteractionSynthesize", {}) == { + "screenshotPath": "/tmp/native.png" + } + await client.close() + + +@pytest.mark.asyncio +async def test_native_tool_errors_propagate_without_retrying_or_destroying_connection( + native_transport, +): + results = iter( + [response(text="Agent is not approved", error=True), response({"userMessage": "Closed"})] + ) + + async def call(name, arguments): + return next(results) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises( + RuntimeError, match="DeviceInteractionStartSession failed: Agent is not approved" + ): + await client.call("DeviceInteractionStartSession", {}) + assert await client.call("DeviceInteractionEndSession", {}) == {"userMessage": "Closed"} + assert [name for name, arguments in native_transport.session.tool_calls] == [ + "DeviceInteractionStartSession", + "DeviceInteractionEndSession", + ] + await client.close() + + +@pytest.mark.asyncio +async def test_unstructured_success_is_rejected(native_transport): + async def call(name, arguments): + return response(text="not JSON") + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises(RuntimeError, match="no structured result"): + await client.call("DeviceInteractionSynthesize", {}) + await client.close() + + +@pytest.mark.asyncio +async def test_simulator_calls_are_serialized(native_transport): + first_started, release_first = asyncio.Event(), asyncio.Event() + + async def call(name, arguments): + if name == "first": + first_started.set() + await release_first.wait() + return response({"name": name}) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + first = asyncio.create_task(client.call("first", {})) + await first_started.wait() + second = asyncio.create_task(client.call("second", {})) + await asyncio.sleep(0) + assert [name for name, arguments in native_transport.session.tool_calls] == ["first"] + release_first.set() + assert await asyncio.gather(first, second) == [{"name": "first"}, {"name": "second"}] + await client.close() + + +@pytest.mark.asyncio +async def test_initialization_timeout_releases_all_resources(native_transport): + async def initialize(): + await asyncio.Event().wait() + + native_transport.session.initialize_hook = initialize + client = XcodeBridge(timeout_seconds=0.01) + with pytest.raises(TimeoutError): + await client.start() + assert_resources_closed_in_owner(native_transport) + with pytest.raises(RuntimeError, match="not connected"): + await client.call("DeviceInteractionSynthesize", {}) + + +@pytest.mark.asyncio +async def test_initialization_cancellation_releases_all_resources(native_transport): + started = asyncio.Event() + + async def initialize(): + started.set() + await asyncio.Event().wait() + + native_transport.session.initialize_hook = initialize + client = XcodeBridge() + task = asyncio.create_task(client.start()) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_initialization_failure_releases_all_resources(native_transport): + async def initialize(): + raise OSError("Cannot open native service") + + native_transport.session.initialize_hook = initialize + client = XcodeBridge() + with pytest.raises( + RuntimeError, match="Cannot connect to Xcode MCP: Cannot open native service" + ): + await client.start() + assert not client.connected + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_input_timeout_retires_bridge_without_repeating_possibly_executed_action( + native_transport, +): + async def call(name, arguments): + await asyncio.Event().wait() + + native_transport.session.call_hook = call + client = XcodeBridge(timeout_seconds=0.01) + await client.start() + with pytest.raises(TimeoutError): + await client.call("DeviceInteractionSynthesize", {"interactionCommand": "t 10 20"}) + assert len(native_transport.session.tool_calls) == 1 + assert_resources_closed_in_owner(native_transport) + with pytest.raises(RuntimeError, match="not connected"): + await client.call("DeviceInteractionSynthesize", {}) + + +@pytest.mark.asyncio +async def test_cancelled_input_retires_bridge_and_fails_queued_action(native_transport): + started = asyncio.Event() + + async def call(name, arguments): + started.set() + await asyncio.Event().wait() + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + first = asyncio.create_task(client.call("first", {})) + await started.wait() + second = asyncio.create_task(client.call("second", {})) + await asyncio.sleep(0) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + with pytest.raises(RuntimeError, match="bridge has closed"): + await second + assert [name for name, arguments in native_transport.session.tool_calls] == ["first"] + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_transport_failure_retires_bridge_and_fails_queued_action(native_transport): + started, fail = asyncio.Event(), asyncio.Event() + + async def call(name, arguments): + started.set() + await fail.wait() + raise OSError("pipe disconnected") + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + first = asyncio.create_task(client.call("first", {})) + await started.wait() + second = asyncio.create_task(client.call("second", {})) + await asyncio.sleep(0) + fail.set() + with pytest.raises(RuntimeError, match="connection failed: pipe disconnected"): + await first + with pytest.raises(RuntimeError, match="bridge has closed"): + await second + await client.close() + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_bridge_can_reconnect_after_timeout_for_session_cleanup(native_transport): + async def call(name, arguments): + await asyncio.Event().wait() + + native_transport.session.call_hook = call + client = XcodeBridge(timeout_seconds=0.01) + await client.start() + with pytest.raises(TimeoutError): + await client.call("DeviceInteractionSynthesize", {"interactionCommand": "t 10 20"}) + assert not client.connected + native_transport.session.call_hook = None + await client.start() + assert client.connected + assert await client.call( + "DeviceInteractionEndSession", {"interactionSessionKey": "known-key"} + ) == {"userMessage": "OK"} + assert [name for name, arguments in native_transport.session.tool_calls] == [ + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", + ] + await client.close() + assert [event for event, task in native_transport.events] == [ + "stdio-enter", + "stdio-exit", + "stdio-enter", + "stdio-exit", + ] diff --git a/tests/unit/drivers/test_ios_driver.py b/tests/unit/drivers/test_ios_driver.py new file mode 100644 index 00000000..37af12df --- /dev/null +++ b/tests/unit/drivers/test_ios_driver.py @@ -0,0 +1,614 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS behavior tested without Xcode, CoreSimulator, or a device.""" + +import asyncio +import base64 +import json +from pathlib import Path +import plistlib +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from PIL import Image +import pytest + +from artemis.drivers.base import KeyCode +from artemis.drivers.ios import xcode_driver +from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver, run_xcrun +from third_party.mobile_use.controllers.types import ElementQuery + + +IOS_A = "DE345DD3-5792-4DAD-B863-144682629565" +IOS_B = "F81B6533-B1EC-40CE-8A21-9CD4F2994E55" +WATCH = "362C24CC-751F-457F-B8BF-9FFAC1F855A0" +NATIVE_TOOLS = { + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", +} +# Apple's packaged device-interaction skill uses these frame/hitPoint formats. +APPLE_HIERARCHY = """Device orientation: Portrait +------------------------ +Application bundle identifier: com.example.app +Application UI orientation: Portrait +Application, pid: 123, label: 'Example' + Window, {{0.0, 0.0}, {200.0, 400.0}}, hitPoint: {100.0, 200.0} + Other, {{0.0, 0.0}, {200.0, 400.0}}, hitPoint: {100.0, 200.0} + Button, {{10.0, 20.0}, {60.0, 30.0}}, identifier: 'login', label: 'Login', hitPoint: {15.0, 25.0}, activationBundleId: com.example.app + StaticText, {{20.0, 70.0}, {100.0, 20.0}}, label: 'Welcome' +""" + + +def device(udid=IOS_A, state="Booted", available=True): + return {"udid": udid, "name": "iPhone", "state": state, "isAvailable": available} + + +class NativeBridge: + def __init__(self, observation): + self.tools = NATIVE_TOOLS.copy() + self.connected = False + self.start = AsyncMock(side_effect=self.open) + self.close = AsyncMock(side_effect=self.finish) + self.call = AsyncMock(side_effect=self.respond) + self.observation = observation + self.start_result = { + "interactionSessionKey": "test-session", + "deviceUUID": IOS_A, + "deviceIsSimulator": True, + } + + async def open(self): + self.connected = True + + async def finish(self): + self.connected = False + + async def respond(self, name, arguments): + if name == "DeviceInteractionStartSession": + return self.start_result + if name == "DeviceInteractionSynthesize": + return self.observation + if name == "DeviceInteractionEndSession": + return {"userMessage": "Closed"} + raise AssertionError(f"Unexpected native tool {name}") + + +@pytest.fixture +def simulator(tmp_path, monkeypatch): + screenshot = tmp_path / "native.png" + Image.new("RGB", (200, 400)).save(screenshot) + hierarchy = tmp_path / "native-hierarchy.txt" + hierarchy.write_text(APPLE_HIERARCHY, encoding="utf-8") + observation = { + "screenshotPath": str(screenshot), + "hierarchyPath": str(hierarchy), + "thumbnailScreenshotPath": str(screenshot), + "logsPath": str(tmp_path / "logs.txt"), + "applicationState": "NotRun", + } + native = NativeBridge(observation) + inventory = {"devices": {"com.apple.CoreSimulator.SimRuntime.iOS-27-0": [device()]}} + + async def command(*arguments, timeout=30): + if arguments == ("xcodebuild", "-version"): + return b"Xcode 27.0\nBuild version 27A266a\n" + if arguments == ("simctl", "list", "devices", "--json"): + return json.dumps(inventory).encode() + if arguments[:2] == ("simctl", "listapps"): + return plistlib.dumps({"com.example.app": {"CFBundleDisplayName": "Example"}}) + return b"" + + commands = AsyncMock(side_effect=command) + monkeypatch.setattr(xcode_driver.sys, "platform", "darwin") + monkeypatch.setattr(xcode_driver, "run_xcrun", commands) + monkeypatch.setattr(xcode_driver, "XcodeBridge", lambda: native) + return SimpleNamespace( + driver=XcodeSimulatorDriver(), + native=native, + commands=commands, + inventory=inventory, + screenshot=screenshot, + hierarchy=hierarchy, + ) + + +@pytest.mark.asyncio +async def test_booted_selection_pins_every_app_operation_to_one_ios_udid(simulator, tmp_path): + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"].append( + device(IOS_B, state="Shutdown") + ) + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.watchOS-27-0"] = [ + device(WATCH) + ] + driver = simulator.driver + await driver.connect() + assert driver.device_id == IOS_A + await driver.launch_app("com.example.app") + await driver.stop_app("com.example.app") + await driver.open_url("example://item?q=$literal") + assert await driver.list_apps() == {"com.example.app": "Example"} + app = tmp_path / "Example.app" + app.mkdir() + (app / "Info.plist").write_bytes(plistlib.dumps({"CFBundleIdentifier": "com.example.app"})) + assert await driver.install_app(app) == "com.example.app" + + native_start = simulator.native.call.await_args_list[0] + assert native_start.args[0] == "DeviceInteractionStartSession" + assert native_start.args[1]["deviceIdentifier"] == IOS_A + assert all( + call.args[2] == IOS_A + for call in simulator.commands.await_args_list + if len(call.args) >= 3 + and call.args[:2] + in { + ("simctl", "launch"), + ("simctl", "terminate"), + ("simctl", "openurl"), + ("simctl", "listapps"), + ("simctl", "install"), + } + ) + assert not any( + call.args[:2] == ("simctl", "boot") for call in simulator.commands.await_args_list + ) + await driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("booted_count", [0, 2]) +async def test_booted_selector_rejects_ambiguous_or_missing_devices(simulator, booted_count): + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"] = [ + device(udid) for udid in (IOS_A, IOS_B)[:booted_count] + ] + with pytest.raises(ValueError, match="exactly one booted iOS simulator"): + await simulator.driver.connect() + simulator.native.start.assert_not_awaited() + simulator.native.call.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("requested, available", [(WATCH, True), (IOS_A, False)]) +async def test_explicit_selection_rejects_non_ios_and_unavailable_devices( + simulator, requested, available +): + simulator.driver._device_id = requested + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"][0][ + "isAvailable" + ] = available + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.watchOS-27-0"] = [ + device(WATCH) + ] + with pytest.raises(ValueError, match="Unavailable iOS simulator"): + await simulator.driver.connect() + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_explicit_shutdown_device_is_booted_and_selected_case_insensitively(simulator): + simulator.driver._device_id = IOS_A.lower() + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"][0]["state"] = ( + "Shutdown" + ) + await simulator.driver.connect() + simulator.commands.assert_any_await("simctl", "boot", IOS_A) + simulator.commands.assert_any_await("simctl", "bootstatus", IOS_A, "-b", timeout=180.0) + assert simulator.driver.device_id == IOS_A + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_repeated_connect_reuses_session(simulator): + await simulator.driver.connect() + await simulator.driver.connect() + simulator.native.start.assert_awaited_once() + starts = [ + c for c in simulator.native.call.await_args_list if c.args[0].endswith("StartSession") + ] + assert len(starts) == 1 + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result", + [ + {"interactionSessionKey": "wrong-target", "deviceUUID": IOS_B, "deviceIsSimulator": True}, + { + "interactionSessionKey": "physical-target", + "deviceUUID": IOS_A, + "deviceIsSimulator": False, + }, + ], +) +async def test_native_wrong_target_is_closed_before_any_interaction(simulator, result): + simulator.native.start_result = result + with pytest.raises(RuntimeError, match="different device"): + await simulator.driver.connect() + simulator.native.call.assert_any_await( + "DeviceInteractionEndSession", {"interactionSessionKey": result["interactionSessionKey"]} + ) + assert not any( + c.args[0] == "DeviceInteractionSynthesize" for c in simulator.native.call.await_args_list + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_missing_native_tools_fail_before_session_start(simulator): + simulator.native.tools.remove("DeviceInteractionSynthesize") + with pytest.raises(RuntimeError, match="tools are unavailable"): + await simulator.driver.connect() + simulator.native.call.assert_not_awaited() + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_non_macos_host_is_rejected_before_launching_native_tools(simulator, monkeypatch): + monkeypatch.setattr(xcode_driver.sys, "platform", "linux") + with pytest.raises(RuntimeError, match="requires macOS"): + await simulator.driver.connect() + simulator.commands.assert_not_awaited() + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_old_xcode_is_rejected_before_device_boot(simulator): + simulator.commands.side_effect = None + simulator.commands.return_value = b"Xcode 26.3\nBuild version 17C529\n" + with pytest.raises(RuntimeError, match="requires Xcode 27"): + await simulator.driver.connect() + simulator.commands.assert_awaited_once_with("xcodebuild", "-version") + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_start_session_error_closes_bridge_without_fake_session_cleanup(simulator): + simulator.native.call.side_effect = RuntimeError("Agent approval required") + with pytest.raises(RuntimeError, match="Agent approval required"): + await simulator.driver.connect() + assert [c.args[0] for c in simulator.native.call.await_args_list] == [ + "DeviceInteractionStartSession" + ] + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_missing_session_key_is_rejected_and_bridge_closed(simulator): + simulator.native.start_result.pop("interactionSessionKey") + with pytest.raises(RuntimeError, match="session key"): + await simulator.driver.connect() + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connect_preserves_device_rejection_when_cleanup_also_fails(simulator): + simulator.native.start_result["deviceUUID"] = IOS_B + + async def respond(name, arguments): + if name == "DeviceInteractionEndSession": + raise RuntimeError("Cleanup unavailable") + return await simulator.native.respond(name, arguments) + + simulator.native.call.side_effect = respond + with pytest.raises(RuntimeError, match="different device"): + await simulator.driver.connect() + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_failed_initial_observation_releases_native_session(simulator): + async def fail_capture(name, arguments): + if name == "DeviceInteractionSynthesize": + raise RuntimeError("Accessibility service failed") + return await simulator.native.respond(name, arguments) + + simulator.native.call.side_effect = fail_capture + with pytest.raises(RuntimeError, match="Accessibility service failed"): + await simulator.driver.connect() + simulator.native.call.assert_any_await( + "DeviceInteractionEndSession", {"interactionSessionKey": "test-session"} + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_disconnect_closes_bridge_even_when_session_end_fails(simulator): + await simulator.driver.connect() + simulator.native.call.side_effect = RuntimeError("Native session expired") + with pytest.raises(RuntimeError, match="expired"): + await simulator.driver.disconnect() + simulator.native.close.assert_awaited_once() + with pytest.raises(RuntimeError, match="Connect"): + await simulator.driver.get_screen_data() + + +@pytest.mark.asyncio +async def test_cancelled_initial_capture_releases_session_and_preserves_cancellation(simulator): + started = asyncio.Event() + + async def respond(name, arguments): + if name == "DeviceInteractionSynthesize": + started.set() + await asyncio.Event().wait() + return await simulator.native.respond(name, arguments) + + simulator.native.call.side_effect = respond + task = asyncio.create_task(simulator.driver.connect()) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + simulator.native.call.assert_any_await( + "DeviceInteractionEndSession", {"interactionSessionKey": "test-session"} + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_disconnect_reopens_retired_transport_only_to_close_known_session(simulator): + await simulator.driver.connect() + simulator.native.connected = False + simulator.native.call.reset_mock() + await simulator.driver.disconnect() + assert simulator.native.start.await_count == 2 + simulator.native.call.assert_awaited_once_with( + "DeviceInteractionEndSession", {"interactionSessionKey": "test-session"} + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scale", [1, 3]) +async def test_observation_and_coordinate_actions_share_screenshot_geometry(simulator, scale): + Image.new("RGB", (200 * scale, 400 * scale)).save(simulator.screenshot) + await simulator.driver.connect() + data = await simulator.driver.get_screen_data() + assert data.platform == "ios" + assert (data.width, data.height) == (200 * scale, 400 * scale) + assert base64.b64decode(data.screenshot_base64) == data.screenshot_bytes + button = next(element for element in data.ui_elements if element["resource_id"] == "login") + assert button["hit_point"] == [15 * scale, 25 * scale] + assert button["parsed_bounds"] == { + "left": 10 * scale, + "top": 20 * scale, + "right": 70 * scale, + "bottom": 50 * scale, + } + + await simulator.driver.tap(15 * scale, 25 * scale, duration_ms=600) + arguments = simulator.native.call.await_args.args[1] + assert arguments["interactSessionKey"] == "test-session" + command = arguments["interactionCommand"].split() + assert command[0] == "t" + assert list(map(float, command[1:])) == [15, 25, 0.6] + + await simulator.driver.swipe(150 * scale, 300 * scale, 50 * scale, 100 * scale, 250) + command = simulator.native.call.await_args.args[1]["interactionCommand"].split() + assert command[0] == "t" and command[3] == "f" + assert list(map(float, command[1:3] + command[4:])) == [150, 300, 50, 100, 0.25] + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_element_action_uses_native_hit_point_and_app_activation(simulator): + await simulator.driver.connect() + assert await simulator.driver.tap_element(ElementQuery(resource_id="login")) + arguments = simulator.native.call.await_args.args[1] + assert arguments["activationBundleId"] == "com.example.app" + assert list(map(float, arguments["interactionCommand"].split()[1:])) == [15, 25, 0.1] + assert not await simulator.driver.tap_element(ElementQuery(text="Missing")) + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_text_preserves_unicode_whitespace_and_literal_native_escape_sequences(simulator): + await simulator.driver.connect() + text = "한글🙂 'quoted'\n\t" + r"\u{000A} b h $(literal)" + assert await simulator.driver.input_text(text, clear_existing=False) + command = simulator.native.call.await_args.args[1]["interactionCommand"] + assert command.startswith("sender keyboard kbd ") + encoded = command.removeprefix("sender keyboard kbd ") + # The native grammar decodes escapes once. Literal user escape syntax must + # survive that decode, while actual controls and supplementary Unicode work. + decoded = re.sub(r"\\u\{([0-9A-Fa-f]+)\}", lambda m: chr(int(m[1], 16)), encoded) + assert decoded == text + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_unverified_text_replacement_is_explicitly_rejected(simulator): + with pytest.raises(NotImplementedError, match="replace-text"): + await simulator.driver.input_text("replacement", clear_existing=True) + simulator.native.call.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key, command", + [ + (KeyCode.HOME, "b h"), + (KeyCode.APP_SWITCH, "b h b h"), + (KeyCode.ENTER, r"sender keyboard kbd \u{000A}"), + ], +) +async def test_supported_native_hardware_and_enter_keys(simulator, key, command): + await simulator.driver.connect() + assert await simulator.driver.press_key(key) + assert simulator.native.call.await_args.args[1]["interactionCommand"] == command + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_android_back_and_shell_are_not_reported_as_success(simulator): + with pytest.raises(NotImplementedError, match="not supported"): + await simulator.driver.press_key(KeyCode.BACK) + with pytest.raises(NotImplementedError, match="unavailable"): + await simulator.driver.execute_shell("input keyevent BACK") + simulator.native.call.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_missing_hierarchy_prevents_coordinate_input(simulator): + await simulator.driver.connect() + simulator.native.observation.pop("hierarchyPath") + before = len(simulator.native.call.await_args_list) + with pytest.raises(RuntimeError, match="no accessibility hierarchy"): + await simulator.driver.tap(15, 25) + new_calls = simulator.native.call.await_args_list[before:] + assert len(new_calls) == 1 + assert new_calls[0].args[1]["interactionCommand"] == "" + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "operation, arguments", + [ + ("launch_app", ("com.example.app",)), + ("stop_app", ("com.example.app",)), + ("open_url", ("example://",)), + ("list_apps", ()), + ("install_app", (Path("Untrusted.app"),)), + ], +) +async def test_app_operations_require_connected_pinned_device(simulator, operation, arguments): + with pytest.raises(RuntimeError, match="[Cc]onnect"): + await getattr(simulator.driver, operation)(*arguments) + simulator.commands.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_directional_swipe_uses_current_screen_after_rotation(simulator): + await simulator.driver.connect() + Image.new("RGB", (400, 200)).save(simulator.screenshot) + simulator.hierarchy.write_text( + APPLE_HIERARCHY.replace("{200.0, 400.0}", "{400.0, 200.0}"), encoding="utf-8" + ) + assert await simulator.driver.swipe_direction("up") + command = simulator.native.call.await_args.args[1]["interactionCommand"].split() + assert list(map(float, command[1:3] + command[4:6])) == [200, 150, 200, 50] + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action, arguments", [("tap", (15, 25)), ("swipe", (15, 25, 100, 150))]) +async def test_stale_absolute_coordinates_are_rejected_after_rotation(simulator, action, arguments): + await simulator.driver.connect() + Image.new("RGB", (400, 200)).save(simulator.screenshot) + simulator.hierarchy.write_text( + APPLE_HIERARCHY.replace("{200.0, 400.0}", "{400.0, 200.0}"), encoding="utf-8" + ) + simulator.native.call.reset_mock() + with pytest.raises(ValueError, match="changed orientation or size"): + await getattr(simulator.driver, action)(*arguments) + simulator.native.call.assert_awaited_once_with( + "DeviceInteractionSynthesize", + {"interactSessionKey": "test-session", "interactionCommand": ""}, + ) + await simulator.driver.disconnect() + + +def test_apple_frame_formats_and_hitpoints_are_parsed_without_retina_assumptions(): + hierarchy = """Application bundle identifier: com.example.app +UIWindow {{0, 0}, {200, 400}}, hitPoint: {100, 200} + UIButton "Login" {{10, 20}, {60, 30}}, hitPoint: {15, 25} + UIView {{-10, 50}, {20, 20}}, hitPoint: {5, 60} + UIButton "Offscreen" {{300, 500}, {60, 30}} + UIView {{10, 10}, {0, 5}} +""" + elements, scale = parse_hierarchy(hierarchy, 200, 400) + assert scale == (1, 1) + login = next(element for element in elements if element["text"] == "Login") + assert login["class"] == "UIButton" + assert login["bounds"] == "[10,20][70,50]" + assert login["hit_point"] == [15, 25] + assert not any(element["text"] == "Offscreen" for element in elements) + assert len(elements) == 3 + + +def test_known_rotated_window_geometry_is_not_guessed(): + with pytest.raises(RuntimeError, match="matching the screenshot"): + parse_hierarchy("Window {{0, 0}, {400, 200}}", 200, 400) + + +def test_inaccessible_canvas_can_use_native_logical_screenshot_space(): + elements, scale = parse_hierarchy("AX hierarchy is unavailable", 200, 400) + assert elements == [] + assert scale == (1, 1) + + +def test_foreground_package_is_unknown_when_multiple_apps_overlap(): + assert application_bundle(APPLE_HIERARCHY) == "com.example.app" + assert ( + application_bundle(APPLE_HIERARCHY + "Application bundle identifier: com.other.app") is None + ) + assert application_bundle("unavailable hierarchy") is None + + +class BlockingProcess: + def __init__(self): + self.returncode = None + self.started = asyncio.Event() + self.communications = 0 + self.kills = 0 + + async def communicate(self): + self.communications += 1 + if self.communications == 1: + self.started.set() + await asyncio.Event().wait() + return b"", b"" + + def kill(self): + self.kills += 1 + self.returncode = -9 + + +@pytest.mark.asyncio +async def test_simctl_timeout_kills_and_reaps_child(monkeypatch): + child = BlockingProcess() + spawn = AsyncMock(return_value=child) + monkeypatch.setattr(xcode_driver.asyncio, "create_subprocess_exec", spawn) + with pytest.raises(TimeoutError): + await run_xcrun("simctl", "bootstatus", IOS_A, "-b", timeout=0.01) + assert child.kills == 1 + assert child.communications == 2 + assert spawn.await_args.args == ("xcrun", "simctl", "bootstatus", IOS_A, "-b") + + +@pytest.mark.asyncio +async def test_simctl_cancellation_kills_and_reaps_child(monkeypatch): + child = BlockingProcess() + monkeypatch.setattr( + xcode_driver.asyncio, "create_subprocess_exec", AsyncMock(return_value=child) + ) + task = asyncio.create_task(run_xcrun("simctl", "launch", IOS_A, "com.example.app")) + await child.started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert child.kills == 1 + assert child.communications == 2 + + +@pytest.mark.asyncio +async def test_simctl_failures_report_stderr_and_preserve_literal_argv(monkeypatch): + child = SimpleNamespace(returncode=1, communicate=AsyncMock(return_value=(b"", b"No app\xff"))) + spawn = AsyncMock(return_value=child) + monkeypatch.setattr(xcode_driver.asyncio, "create_subprocess_exec", spawn) + url = "example://item?command=$(literal)&x='quote'" + with pytest.raises(RuntimeError, match="No app"): + await run_xcrun("simctl", "openurl", IOS_A, url) + assert spawn.await_args.args == ("xcrun", "simctl", "openurl", IOS_A, url) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 11e9077a..cc1d12cd 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -48,6 +48,7 @@ def test_cli_run_help(): assert "--traces-path" in result.output assert "--verification-level" in result.output assert "--explorer-pro-mode" in result.output + assert "--platform" in result.output def test_cli_batch_help(): diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py new file mode 100644 index 00000000..6d82d321 --- /dev/null +++ b/tests/unit/test_ios_integration.py @@ -0,0 +1,623 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic tests for the CLI/SDK/native-action boundaries of iOS support.""" + +import asyncio +import io +import sys +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from PIL import Image +from typer.testing import CliRunner + +from artemis.context import ArtemisContext, DeviceContext, DevicePlatform +from artemis.controllers.unified_controller import UnifiedMobileController +from artemis.drivers.base import ScreenData +from artemis.drivers.factory import create_driver +from artemis.graph.state import State +from artemis.interfaces.cli.commands import run as run_module +from artemis.interfaces.cli.main import app +from artemis.mcp.action_session import get_action_session +from artemis.mcp.action_types import ActionCode +from artemis.mcp.actuators.ios import IosActuator +from artemis.sdk.agent import Agent +from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder +from artemis.tools.mobile.launch_app import find_package, launch_app +from artemis.runtime import DeviceBusyError +from third_party.mobile_use.sdk.agent import AgentBase +from third_party.mobile_use.sdk.types.exceptions import AgentError, AgentNotInitializedError + + +@pytest.fixture +def native_driver(monkeypatch): + driver = MagicMock() + driver.device_id = "00000000-0000-0000-0000-000000000001" + driver.screen_size = (1170, 2532) + for method in ( + "connect", + "resolve_device", + "disconnect", + "input_text", + "press_key", + "open_url", + "tap", + "swipe", + "long_press", + "launch_app", + "stop_app", + "install_app", + "execute_shell", + ): + setattr(driver, method, AsyncMock(return_value=True)) + driver.list_apps = AsyncMock(return_value={"com.apple.Preferences": "Settings"}) + driver.get_screen_data = AsyncMock( + return_value=ScreenData( + screenshot_bytes=b"png", + screenshot_base64="cG5n", + platform="ios", + width=1170, + height=2532, + ui_hierarchy_xml="", + ) + ) + constructor = MagicMock(return_value=driver) + monkeypatch.setitem( + sys.modules, + "artemis.drivers.ios.xcode_driver", + SimpleNamespace(XcodeSimulatorDriver=constructor), + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.delenv("ARTEMIS_MOCK_DRIVER", raising=False) + driver.constructor = constructor + return driver + + +def ios_context(driver=None): + context = ArtemisContext( + device=DeviceContext( + mobile_platform=DevicePlatform.IOS, + device_id="booted", + device_width=1170, + device_height=2532, + ) + ) + context._active_driver = driver + return context + + +def ios_config(): + return AgentConfigBuilder().for_ios_simulator().build(validate_profiles=False) + + +def test_factory_selects_ios_without_creating_adb(native_driver, monkeypatch): + adb = MagicMock(side_effect=AssertionError("must not create ADB client")) + monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) + context = ios_context() + assert create_driver(context) is native_driver + native_driver.constructor.assert_called_once_with(device_id="booted") + assert context.adb_client is None + + +def test_macos_host_keeps_android_default(monkeypatch): + context = ArtemisContext(device=DeviceContext(host_platform="DARWIN", device_id="android-1")) + android = MagicMock() + adb = MagicMock() + monkeypatch.setattr("artemis.drivers.factory.AndroidAdbDriver", android) + monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) + assert create_driver(context) is android.return_value + adb.assert_called_once() + assert context.device.mobile_platform == DevicePlatform.ANDROID + + +def test_factory_rejects_cloud_ios_before_importing_gateway(native_driver, monkeypatch): + monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1") + with pytest.raises(ValueError, match="local only"): + create_driver(ios_context()) + native_driver.constructor.assert_not_called() + + +@pytest.mark.asyncio +async def test_sdk_ios_initialization_bypasses_android_and_cleans(native_driver, monkeypatch): + monkeypatch.setattr( + AgentBase, "_init_internal", AsyncMock(side_effect=AssertionError("ADB init")) + ) + agent = Agent(config=ios_config()) + agent._prewarm_llm_connections = AsyncMock() + assert await agent.init() is True + await asyncio.sleep(0) + native_driver.resolve_device.assert_awaited_once() + native_driver.connect.assert_not_awaited() + assert agent._device_context.device_id == native_driver.device_id + assert (agent._device_context.device_width, agent._device_context.device_height) == (1170, 2532) + assert agent._adb_client is None and agent._ui_adb_client is None + assert agent._config.video_recording_tools_enabled is False + assert agent._config.disable_device_probes is True + await agent.clean() + native_driver.disconnect.assert_awaited_once() + assert agent._initialized is False + + +@pytest.mark.asyncio +async def test_sdk_failed_ios_init_closes_partial_transport(native_driver): + native_driver.resolve_device.side_effect = RuntimeError("bridge failed") + agent = Agent(config=ios_config()) + with pytest.raises(RuntimeError, match="bridge failed"): + await agent.init() + native_driver.disconnect.assert_awaited_once() + assert agent._initialized is False + + +@pytest.mark.asyncio +async def test_sdk_device_overrides_preserve_ios(native_driver): + agent = Agent(config=ios_config(), device_serial="simulator-2") + assert agent._config.device_platform == DevicePlatform.IOS + agent._init_internal = AsyncMock(return_value=True) + await agent.init(device_serial="simulator-3") + assert agent._config.device_id == "simulator-3" + assert agent._config.device_platform == DevicePlatform.IOS + + +@pytest.mark.asyncio +async def test_sdk_sequential_run_reconnects_driver_without_android_unlock(native_driver): + agent = Agent(config=ios_config()) + agent._ios_driver = native_driver + agent._device_context = ios_context().device + await agent._ensure_device_unlocked() + await ios_context(native_driver).disconnect_driver() + await agent._ensure_device_unlocked() + assert native_driver.connect.await_count == 2 + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_context_closes_session_before_driver_and_only_once(native_driver): + order = [] + native_driver.disconnect.side_effect = lambda: order.append("driver") + context = ios_context(native_driver) + context.action_session = SimpleNamespace( + aclose=AsyncMock(side_effect=lambda: order.append("session")) + ) + with pytest.raises(RuntimeError, match="task failed"): + async with context: + raise RuntimeError("task failed") + await context.disconnect_driver() + assert order == ["session", "driver"] + assert context._active_driver is None + + +@pytest.mark.asyncio +async def test_action_session_selects_native_actuator_and_normalizes_coordinates(native_driver): + context = ios_context(native_driver) + async with context: + session = await get_action_session(context) + clicked = await session.call("click", {"target": [500, 500]}) + typed = await session.call("input_text", {"text": "Hello 한글", "clear_exist": False}) + assert clicked.ok and typed.ok + native_driver.tap.assert_awaited_once_with(585, 1266, duration_ms=100, times=1, delay_ms=100) + native_driver.input_text.assert_awaited_once_with("Hello 한글", clear_existing=False) + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_key_failure_is_not_reported_as_success(native_driver): + native_driver.press_key.return_value = False + result = await IosActuator(ios_context(native_driver)).press_key("back") + assert result.ok is False and result.code == ActionCode.UNSUPPORTED + + +@pytest.mark.asyncio +async def test_ios_clear_refusal_happens_before_focus_mutation(native_driver): + result = await IosActuator(ios_context(native_driver)).input_text( + "hello", target=(500, 500), clear_exist=True + ) + assert result.code == ActionCode.UNSUPPORTED + native_driver.tap.assert_not_awaited() + native_driver.input_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_coordinates_follow_latest_orientation(native_driver): + native_driver.screen_size = (2532, 1170) + assert (await IosActuator(ios_context(native_driver)).click(500, 500)).ok + native_driver.tap.assert_awaited_once_with(1266, 585, duration_ms=100, times=1, delay_ms=100) + + +@pytest.mark.asyncio +async def test_ios_urls_and_clear_use_native_driver(native_driver): + controller = UnifiedMobileController(ios_context(native_driver)) + assert await controller.open_url("https://example.com") is True + native_driver.open_url.assert_awaited_once_with("https://example.com") + native_driver.input_text.return_value = False + assert await controller.erase_text() is False + native_driver.input_text.assert_awaited_once_with("", clear_existing=True) + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_apps_resolve_native_names_and_launch_without_android_poll(native_driver): + context = ios_context(native_driver) + assert await find_package(context, "settings") == "com.apple.Preferences" + assert await find_package(context, "com.apple.Preferences") == "com.apple.Preferences" + result = await launch_app.execute(ctx=context, app_name="Settings") + assert result == "Launched app 'Settings' (com.apple.Preferences)." + native_driver.launch_app.assert_awaited_once_with("com.apple.Preferences") + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_sdk_rejects_unverifiable_ios_app_lock(native_driver): + agent = Agent(config=ios_config()) + task = SimpleNamespace(request=SimpleNamespace(locked_app_package="com.apple.Preferences")) + with pytest.raises(AgentError, match="foreground ownership"): + await agent._prepare_app_lock(task, ios_context(native_driver)) + + +@pytest.mark.asyncio +async def test_execute_task_ios_ignores_android_selection(monkeypatch): + builder = MagicMock() + automation = AsyncMock() + monkeypatch.setattr(run_module, "new_default_config_builder", lambda: builder) + monkeypatch.setattr(run_module, "run_automation", automation) + monkeypatch.setenv("ADB_DEVICE_SERIAL", "android-env-device") + monkeypatch.setattr( + "artemis.runtime.device_pool.select_device", + MagicMock(side_effect=AssertionError("ADB pool")), + ) + await run_module.execute_task("Open Settings", platform=DevicePlatform.IOS) + builder.for_device.assert_called_once_with(DevicePlatform.IOS, "booted") + builder.with_video_recording_tools.assert_called_once_with(enabled=False) + automation.assert_awaited_once() + + +def test_cli_ios_runs_locally_without_android_status_or_daemon(monkeypatch): + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setattr( + run_module, + "display_local_device_status", + MagicMock(side_effect=AssertionError("ADB status")), + ) + monkeypatch.setattr( + "artemis.runtime.ensure_daemon_running", MagicMock(side_effect=AssertionError("daemon")) + ) + monkeypatch.delenv("ARTEMIS_TASK_WORKER", raising=False) + monkeypatch.delenv("ARTEMIS_DEVICE_QUEUE_TICKET", raising=False) + result = CliRunner().invoke( + app, ["run", "--platform", "ios", "--device-serial", "booted", "Open Settings"] + ) + assert result.exit_code == 0, result.output + assert execute.call_args.kwargs["platform"] == DevicePlatform.IOS + assert execute.call_args.kwargs["device_serial"] == "booted" + + +@pytest.mark.parametrize("env_name", ["ARTEMIS_CLOUD_MODE", "ARTEMIS_TASK_WORKER"]) +def test_cli_rejects_ios_android_services_before_run(monkeypatch, env_name): + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setenv(env_name, "1") + result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) + assert result.exit_code != 0 + execute.assert_not_called() + + +def test_ios_probe_and_pro_tool_gates(native_driver, monkeypatch): + from artemis.agents.checker.checker import probes_enabled + from artemis.tools import index + + context = ios_context(native_driver) + assert probes_enabled(context) is False + tools = [ + SimpleNamespace(name=n) + for n in ("run_adb_command", "manage_task", "save_note", "ask_explorer") + ] + monkeypatch.setattr(index, "build_tools_from_wrappers", lambda *args, **kwargs: tools) + assert [t.name for t in index.get_tools_from_wrappers(context, [])] == [ + "save_note", + "ask_explorer", + ] + + +@pytest.mark.asyncio +async def test_ios_hierarchy_tool_returns_accessibility_json(native_driver): + import json + from artemis.tools.mobile.read_hierarchy import get_ui_hierarchy, get_ui_hierarchy_tool + + data = native_driver.get_screen_data.return_value + data.ui_hierarchy_xml = None + data.ui_elements = [{"text": "설정", "bounds": "[0,0][100,100]"}] + context = ios_context(native_driver) + result = await get_ui_hierarchy.execute(driver=native_driver, ctx=context) + assert json.loads(result) == data.ui_elements + assert "iOS accessibility" in get_ui_hierarchy_tool(context).description + + +def test_flash_and_pro_prompts_teach_ios_constraints(native_driver): + from artemis.agents.flash.runner import FlashRunner + from artemis.agents.operator.prompts import ( + PLAN_HISTORY_TEMPLATE_SECTION, + render_transcript_static_system, + resolve_operator_prompt_tools, + ) + + context = ios_context(native_driver) + context.actuator = IosActuator(context) + runner = object.__new__(FlashRunner) + runner.ctx = context + runner.goal = "Open Settings" + flash = runner._render_system_prompt([]) + prompts = {"main_template": PLAN_HISTORY_TEMPLATE_SECTION + "\n# CURRENT OBSERVATION"} + pro = render_transcript_static_system(prompts, context, State.initial("Open Settings")) + for prompt in (flash, pro): + assert "iOS Simulator" in prompt + assert "clear_exist=false" in prompt + assert ( + "press_key supports enter, home, " + "power, volume_up, volume_down, and app_switch" in prompt + ) + assert "Back/Enter/Delete" not in prompt + assert "run_adb_command" not in resolve_operator_prompt_tools(context) + assert "manage_task" not in resolve_operator_prompt_tools(context) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("profile", ["flash", "pro"]) +async def test_sdk_profiles_route_repeated_tasks_through_native_actions( + native_driver, monkeypatch, tmp_path, profile +): + import third_party.mobile_use.sdk.agent as sdk_base + + agent = Agent(config=ios_config()) + agent._tmp_traces_dir = tmp_path + agent._prewarm_llm_connections = AsyncMock() + agent._prepare_trace_paths = MagicMock(return_value="native-test") + agent._prepare_output_files = MagicMock() + agent._finalize_tracing_safely = AsyncMock() + agent._extract_output = AsyncMock(return_value="completed") + engine = MagicMock() + engine.shutdown = AsyncMock() + monkeypatch.setattr("artemis.sdk.agent.DataEngine", lambda **kwargs: engine) + lease = MagicMock() + lease.get_active_owner.return_value = None + monkeypatch.setattr(sdk_base, "DeviceExecutionLock", lease) + + class NativeFlash: + def __init__(self, context, **kwargs): + self.context = context + + async def run(self, state): + assert isinstance(self.context.actuator, IosActuator) + session = await get_action_session(self.context) + assert (await session.call("click", {"target": [500, 500]})).ok + return {"status": "completed"} + + async def native_graph(context): + assert isinstance(context.actuator, IosActuator) + + async def stream(**kwargs): + session = await get_action_session(context) + assert (await session.call("click", {"target": [500, 500]})).ok + yield "values", State.initial("Open Settings").model_dump() + + return SimpleNamespace(astream=stream) + + helper_lease = MagicMock() + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", helper_lease) + monkeypatch.setattr(sdk_base, "FlashRunner", NativeFlash) + monkeypatch.setattr(sdk_base, "get_graph", native_graph) + app_dir = tmp_path / "Demo.app" + app_dir.mkdir() + native_driver.install_app.return_value = "com.example.demo" + await agent.init() + for _ in range(2): + await agent.run_task(goal="Open Settings", profile=profile, app_path=app_dir) + await asyncio.sleep(0) + assert native_driver.tap.await_count == 2 + # Native app installation runs under the task's own lease and session, so + # no additional helper lease is constructed and no extra session is opened. + helper_lease.assert_not_called() + assert native_driver.install_app.await_count == 2 + native_driver.install_app.assert_awaited_with(app_dir) + assert native_driver.connect.await_count == 2 + assert native_driver.disconnect.await_count == 2 + native_driver.execute_shell.assert_not_awaited() + assert all(task.status == "completed" for task in agent._tasks) + + +def _png_bytes(width=1170, height=2532, color=(18, 52, 86)) -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (width, height), color).save(buffer, format="PNG") + return buffer.getvalue() + + +def _ordered_lease(order: list[str]): + class FakeLease: + def __init__(self, *args, **kwargs): + pass + + def acquire(self, *, cancel_event=None, **kwargs): + order.append("acquire") + + def release(self): + order.append("release") + + return FakeLease + + +async def _init_ios_agent(native_driver): + agent = Agent(config=ios_config()) + agent._prewarm_llm_connections = AsyncMock() + assert await agent.init() is True + await asyncio.sleep(0) + return agent + + +@pytest.mark.asyncio +async def test_ios_public_screenshot_owns_a_leased_session(native_driver, monkeypatch): + order: list[str] = [] + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", _ordered_lease(order)) + native_driver.connect.side_effect = lambda *a, **k: order.append("connect") + native_driver.disconnect.side_effect = lambda *a, **k: order.append("disconnect") + png = _png_bytes() + + async def capture(*args, **kwargs): + order.append("capture") + return ScreenData( + screenshot_bytes=png, + screenshot_base64="cG5n", + platform="ios", + width=1170, + height=2532, + ui_hierarchy_xml="", + ) + + native_driver.get_screen_data.side_effect = capture + agent = await _init_ios_agent(native_driver) + + image = await agent.get_screenshot() + assert isinstance(image, Image.Image) + assert image.size == (1170, 2532) + assert image.getpixel((0, 0)) == (18, 52, 86) + assert order == ["acquire", "connect", "capture", "disconnect", "release"] + + # A second public call opens a fresh lease and native session. + order.clear() + again = await agent.get_screenshot() + assert again.size == (1170, 2532) + assert order == ["acquire", "connect", "capture", "disconnect", "release"] + assert native_driver.connect.await_count == 2 + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_public_install_app_owns_a_leased_session(native_driver, monkeypatch, tmp_path): + order: list[str] = [] + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", _ordered_lease(order)) + native_driver.connect.side_effect = lambda *a, **k: order.append("connect") + native_driver.disconnect.side_effect = lambda *a, **k: order.append("disconnect") + + async def install(path): + order.append("install") + return "com.example.demo" + + native_driver.install_app.side_effect = install + app_dir = tmp_path / "Demo.app" + app_dir.mkdir() + (app_dir / "Info.plist").write_bytes(b"placeholder") + agent = await _init_ios_agent(native_driver) + + bundle = await agent.install_app(app_dir) + assert bundle == "com.example.demo" + native_driver.install_app.assert_awaited_once_with(app_dir) + assert order == ["acquire", "connect", "install", "disconnect", "release"] + + +@pytest.mark.asyncio +async def test_ios_public_helpers_require_initialization(native_driver, monkeypatch, tmp_path): + lease = MagicMock() + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", lease) + agent = Agent(config=ios_config()) + with pytest.raises(AgentNotInitializedError): + await agent.get_screenshot() + with pytest.raises(AgentNotInitializedError): + await agent.install_app(tmp_path / "Demo.app") + lease.assert_not_called() + native_driver.constructor.assert_not_called() + native_driver.connect.assert_not_awaited() + + # Android selection still delegates to the base implementation unchanged. + android = Agent(config=AgentConfigBuilder().build(validate_profiles=False)) + with pytest.raises(AgentNotInitializedError): + await android.get_screenshot() + with pytest.raises(FileNotFoundError): + await android.install_app(tmp_path / "missing.apk") + + +@pytest.mark.asyncio +async def test_ios_helper_failures_release_session_and_lease(native_driver, monkeypatch, tmp_path): + order: list[str] = [] + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", _ordered_lease(order)) + native_driver.disconnect.side_effect = lambda *a, **k: order.append("disconnect") + agent = await _init_ios_agent(native_driver) + + native_driver.connect.side_effect = RuntimeError("connect failed") + with pytest.raises(RuntimeError, match="connect failed"): + await agent.get_screenshot() + assert order == ["acquire", "disconnect", "release"] + + order.clear() + native_driver.connect.side_effect = lambda *a, **k: order.append("connect") + native_driver.get_screen_data.side_effect = RuntimeError("capture failed") + with pytest.raises(RuntimeError, match="capture failed"): + await agent.get_screenshot() + assert order == ["acquire", "connect", "disconnect", "release"] + + order.clear() + native_driver.install_app.side_effect = RuntimeError("install failed") + app_dir = tmp_path / "Demo.app" + app_dir.mkdir() + with pytest.raises(RuntimeError, match="install failed"): + await agent.install_app(app_dir) + assert order == ["acquire", "connect", "disconnect", "release"] + + +@pytest.mark.asyncio +async def test_ios_operation_cancellation_drains_queued_acquire(native_driver, monkeypatch): + started = threading.Event() + finished = threading.Event() + released = MagicMock() + + class FakeLease: + def __init__(self, *args, **kwargs): + pass + + def acquire(self, *, cancel_event=None, **kwargs): + started.set() + try: + cancel_event.wait(timeout=30) + finally: + finished.set() + raise DeviceBusyError("queue wait cancelled") + + def release(self): + released() + + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", FakeLease) + agent = await _init_ios_agent(native_driver) + + helper = asyncio.ensure_future(agent.get_screenshot()) + assert await asyncio.to_thread(started.wait, 10) is True + helper.cancel() + with pytest.raises(asyncio.CancelledError): + await helper + assert finished.is_set() + released.assert_called_once() + native_driver.connect.assert_not_awaited() + native_driver.get_screen_data.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_context_disconnect_driver_swallows_expected_cleanup_errors(native_driver): + native_driver.disconnect.side_effect = OSError("transport already closed") + context = ios_context(native_driver) + await context.disconnect_driver() + assert context._active_driver is None + + native_driver.disconnect.side_effect = KeyError("unexpected") + context = ios_context(native_driver) + with pytest.raises(KeyError): + await context.disconnect_driver() diff --git a/tests/unit/test_ios_observation.py b/tests/unit/test_ios_observation.py new file mode 100644 index 00000000..4abd95c9 --- /dev/null +++ b/tests/unit/test_ios_observation.py @@ -0,0 +1,103 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native hit points must survive the shared perception-to-action pipeline.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from PIL import Image +import pytest + +from artemis.agents.explorer.screen_index import ScreenIndex +from artemis.context import ArtemisContext, DeviceContext, DevicePlatform +from artemis.drivers.ios.hierarchy import parse_hierarchy +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from artemis.mcp.action_executor import McpActionExecutor +from artemis.mcp.actuators.ios import IosActuator +from artemis.mcp.observation import observe +from artemis.utils.visualization import ( + format_minimal_list_with_elements, + format_minimal_list_with_points, +) + + +HIERARCHY = """Application bundle identifier: com.example.Test +Application, pid: 123, label: ' ' + Window, {{0.0, 0.0}, {400.0, 800.0}}, hitPoint: {200.0, 400.0} + UIButton \"Continue\" {{100.0, 200.0}, {60.0, 30.0}}, hitPoint: {150.0, 220.0}, identifier: 'continueButton', activationBundleId: com.example.Test +""" + + +@pytest.mark.asyncio +async def test_ios_observation_preserves_hit_point_for_indexed_native_tap(tmp_path, monkeypatch): + screenshot = tmp_path / "native.png" + Image.new("RGB", (800, 1600)).save(screenshot) + hierarchy = tmp_path / "native.txt" + hierarchy.write_text(HIERARCHY) + driver = XcodeSimulatorDriver("simulator-id") + driver._session_key = "session" + driver._bridge = SimpleNamespace( + call=AsyncMock( + return_value={"screenshotPath": str(screenshot), "hierarchyPath": str(hierarchy)} + ) + ) + context = ArtemisContext( + device=DeviceContext(mobile_platform=DevicePlatform.IOS, device_id="simulator-id") + ) + context._active_driver = driver + actuator = IosActuator(context) + monkeypatch.setattr("artemis.mcp.observation.get_temp_dir", lambda name: tmp_path) + + observation, image = await observe(context, actuator.controller, settle_ms=0) + assert observation.ok and observation.hierarchy_ok and image == screenshot.read_bytes() + assert len(observation.elements) == 1 + element = observation.elements[0] + assert element["center"] == element["hit_point"] == [300, 440] + assert element["resource_id"] == "continueButton" + assert element["activation_bundle_id"] == "com.example.Test" + assert "Continue" in observation.elements_text + + state = SimpleNamespace(indexed_elements=observation.elements) + executor = McpActionExecutor(context, actuator=actuator) + name, arguments, _, _ = executor._translate("click", {"target": 1}, state) + assert name == "click" and arguments["target"] == [375, 275] + assert (await actuator.click(*arguments["target"])).ok + command = driver._bridge.call.await_args.args[1]["interactionCommand"] + assert command == "t 150.0000 220.0000 0.100" + assert driver._bridge.call.await_args.args[1]["activationBundleId"] == "com.example.Test" + + +def test_ios_explorer_and_point_formatter_keep_native_hit_point(): + elements, _ = parse_hierarchy(HIERARCHY, 800, 1600) + _, points, _ = format_minimal_list_with_points(elements, 800, 1600) + assert points == [[300, 440]] + index = ScreenIndex.from_hierarchy(elements, 800, 1600) + match = index.search_text("Continue")[0].element + assert match.center == (300, 440) + assert match.resource_id == "continueButton" + + +@pytest.mark.parametrize("point", [None, [True, 20], [float("nan"), 20], [800, 200], [-1, 10], [1]]) +def test_shared_formatter_keeps_android_centroid_for_missing_or_invalid_native_point(point): + node = { + "text": "Continue", + "bounds": "[200,400][320,460]", + "resource-id": "android:id/button1", + } + if point is not None: + node["hit_point"] = point + _, elements, _ = format_minimal_list_with_elements([node], 800, 1600) + _, points, _ = format_minimal_list_with_points([node], 800, 1600) + assert elements[0]["center"] == points[0] == [260, 430] + assert elements[0]["resource_id"] == "android:id/button1" + assert "hit_point" not in elements[0] + assert ScreenIndex.from_hierarchy([node], 800, 1600).elements[0].center == (260, 430) diff --git a/third_party/mobile_use/context.py b/third_party/mobile_use/context.py index 89533dc7..a9405138 100644 --- a/third_party/mobile_use/context.py +++ b/third_party/mobile_use/context.py @@ -50,6 +50,7 @@ class DevicePlatform(StrEnum): """Mobile device platform enumeration.""" ANDROID = "android" + IOS = "ios" class DeviceContext(BaseModel): diff --git a/third_party/mobile_use/sdk/agent.py b/third_party/mobile_use/sdk/agent.py index c6a81203..5efc1822 100644 --- a/third_party/mobile_use/sdk/agent.py +++ b/third_party/mobile_use/sdk/agent.py @@ -123,7 +123,10 @@ async def init( target_dev = device_serial or device_id if target_dev: self._config = self._config.model_copy( - update={"device_id": target_dev, "device_platform": DevicePlatform.ANDROID} + update={ + "device_id": target_dev, + "device_platform": self._config.device_platform or DevicePlatform.ANDROID, + } ) return await self._init_internal( @@ -440,6 +443,8 @@ async def _run_task( llm_config=agent_profile.llm_config, agent_config=self._config, ) + if context.device.mobile_platform == DevicePlatform.IOS: + context._active_driver = getattr(self, "_ios_driver", None) output_config = None if request.output_description or request.output_format: @@ -516,12 +521,15 @@ async def _execute_task_logic(): # DataEngine uses a shared database, so a queued task must not # publish a new active session while the current task is still # finishing. + if context.device.mobile_platform == DevicePlatform.IOS: + await self._ensure_device_unlocked() self._prepare_tracing(task=task, context=context) self._prepare_output_files(task=task) if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": if self._ui_adb_client is not None: await self._connect_screen_client(context, str(sess_id)) - await self._ensure_device_unlocked() + if context.device.mobile_platform != DevicePlatform.IOS: + await self._ensure_device_unlocked() publish_startup_progress( "environment", "Preparing the device environment", session_id=str(sess_id) ) @@ -749,6 +757,10 @@ async def _safe_stop_recording(): raise finally: + # The native simulator transport is task-owned even when setup + # fails before entering the context manager. + if context.device.mobile_platform == DevicePlatform.IOS: + await context.disconnect_driver() try: # Background ADB processes (logcat, screenrecord, ...) started # by the Operator must not outlive the automation task. From 6e9f64fc1cb75964ae79317ec0579dabdb999f3d Mon Sep 17 00:00:00 2001 From: James Lee Date: Sat, 3 Oct 2026 23:48:31 +0900 Subject: [PATCH 02/32] feat: guide first-run iOS approvals --- artemis/drivers/factory.py | 8 +- artemis/drivers/ios/bridge.py | 56 +++++ artemis/drivers/ios/xcode_driver.py | 71 +++++- artemis/interfaces/cli/commands/run.py | 37 +++ artemis/sdk/agent.py | 5 +- artemis/sdk/builders/agent_config_builder.py | 16 +- artemis/sdk/types/agent.py | 5 + docs/ios.md | 45 ++++ tests/unit/drivers/test_ios_bridge.py | 79 ++++++- tests/unit/drivers/test_ios_driver.py | 215 ++++++++++++++++++ .../sdk/test_agent_config_builder_tuning.py | 34 +++ tests/unit/test_cli.py | 1 + tests/unit/test_ios_integration.py | 110 ++++++++- 13 files changed, 666 insertions(+), 16 deletions(-) diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index 8b024595..f4055eb4 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -39,7 +39,13 @@ def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: raise ValueError("iOS Simulator support is local only; cloud mode targets Android.") from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver - return XcodeSimulatorDriver(device_id=ctx.device.device_id) + config = getattr(ctx, "agent_config", None) + return XcodeSimulatorDriver( + device_id=ctx.device.device_id, + workspace_path=( + getattr(config, "ios_workspace_path", None) if config is not None else None + ), + ) if mobile_platform not in {"android", "mock"}: raise ValueError(f"Unsupported mobile platform: {mobile_platform}") diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py index 2d06366a..91aedf26 100644 --- a/artemis/drivers/ios/bridge.py +++ b/artemis/drivers/ios/bridge.py @@ -19,6 +19,8 @@ import asyncio import json import os +from pathlib import Path +import sys from typing import Any import anyio @@ -27,6 +29,58 @@ from mcp.shared.exceptions import McpError +_APPROVAL_MESSAGES = ( + "agent isn't approved", + "agent is not approved", + "waiting for the user to approve", +) + + +class XcodeApprovalRequiredError(RuntimeError): + """Xcode refused a tool because this interpreter lacks agent/folder approval.""" + + def __init__( + self, + tool_name: str, + native_message: str, + *, + workspace_path: str | Path | None = None, + ): + super().__init__(f"Xcode tool {tool_name} failed: {native_message}") + self.tool_name = tool_name + self.native_message = native_message + self.workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) + + @property + def guidance(self) -> str: + """Human steps to approve this interpreter; never executes them.""" + return xcode_approval_guidance(self.workspace_path) + + +def xcode_approval_guidance(workspace_path: str | Path | None = None) -> str: + """Human-facing Xcode approval steps; printing is the only action taken.""" + workspace = str(workspace_path) if workspace_path else "not supplied" + return ( + "Xcode approval is required for the Python interpreter running Artemis.\n" + f"Interpreter: {Path(sys.executable).resolve()}\n" + f"Workspace: {workspace}\n\n" + "Open the Xcode MCP menu bar icon and approve this interpreter and the selected " + "project folder. Choose Always Allow or persistent approval there if offered. " + "Alternatively, inspect the relevant pending request IDs and approve only those " + "entries from your own terminal:\n" + " xcrun mcp-server status\n" + " sudo xcrun mcp-server approve --always\n\n" + "Artemis never runs sudo or changes global access settings. After approval, " + "rerun the task. Interpreter upgrades, a different project folder, or expiring " + "grants can require approval again.\n" + "If Xcode has not recorded a request yet, rerun with " + "--platform ios --ios-workspace /absolute/path/YourApp.xcodeproj " + "(or pass workspace_path to for_ios_simulator())." + ) + + _ERRORS = ( OSError, ValueError, @@ -127,6 +181,8 @@ async def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: raise if result.isError: message = " ".join(block.text for block in result.content if block.type == "text") + if any(marker in message.casefold() for marker in _APPROVAL_MESSAGES): + raise XcodeApprovalRequiredError(name, message) raise RuntimeError(f"Xcode tool {name} failed: {message}") if isinstance(result.structuredContent, dict): return result.structuredContent diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index d9caf37a..7576bc93 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -25,7 +25,7 @@ from PIL import Image from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection -from artemis.drivers.ios.bridge import XcodeBridge +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, XcodeBridge from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy from third_party.mobile_use.controllers.types import ElementQuery from third_party.mobile_use.utils.logger import get_logger @@ -64,9 +64,19 @@ class XcodeSimulatorDriver(BaseDeviceDriver): report an error instead of returning a synthetic success. """ - def __init__(self, device_id: str = "booted", width: int = 0, height: int = 0): + def __init__( + self, + device_id: str = "booted", + width: int = 0, + height: int = 0, + *, + workspace_path: str | Path | None = None, + ): self._device_id = device_id self._width, self._height = width, height + self._workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) self._scale: tuple[float, float] | None = None self._session_key: str | None = None self._bridge = XcodeBridge() @@ -88,6 +98,18 @@ async def resolve_device(self) -> str: return self._device_id async def _resolve_device(self) -> dict[str, Any]: + if self._workspace_path is not None: + if ( + self._workspace_path.suffix.lower() + not in { + ".xcodeproj", + ".xcworkspace", + } + or not self._workspace_path.is_dir() + ): + raise ValueError( + "iOS workspace must be an existing .xcodeproj or .xcworkspace directory." + ) if sys.platform != "darwin": raise RuntimeError("iOS Simulator support requires macOS and Xcode 27 or later.") version = (await run_xcrun("xcodebuild", "-version")).decode() @@ -141,13 +163,34 @@ async def connect(self) -> None: "Xcode's native device interaction tools are unavailable. Select Xcode 27 and approve Artemis's access in Xcode." ) try: - session = await self._bridge.call( - "DeviceInteractionStartSession", - { - "deviceIdentifier": self._device_id, - "sessionIdentifier": f"Artemis Simulator {uuid4().hex[:8]}", - }, - ) + start_arguments = { + "deviceIdentifier": self._device_id, + "sessionIdentifier": f"Artemis Simulator {uuid4().hex[:8]}", + } + try: + session = await self._bridge.call( + "DeviceInteractionStartSession", start_arguments + ) + except XcodeApprovalRequiredError: + if self._workspace_path is None: + raise + if "XcodeOpenWorkspace" not in self._bridge.tools: + raise XcodeApprovalRequiredError( + "DeviceInteractionStartSession", + "Xcode's XcodeOpenWorkspace tool is unavailable; " + "approve Artemis's access in Xcode.", + ) + opened = await self._bridge.call( + "XcodeOpenWorkspace", {"path": str(self._workspace_path)} + ) + identifier = opened.get("workspaceIdentifier") + if not isinstance(identifier, str) or not identifier: + raise RuntimeError( + "XcodeOpenWorkspace returned no usable workspace identifier." + ) + session = await self._bridge.call( + "DeviceInteractionStartSession", start_arguments + ) self._session_key = session.get("interactionSessionKey") if not self._session_key: raise RuntimeError("Xcode did not return a device interaction session key.") @@ -157,7 +200,15 @@ async def connect(self) -> None: ): raise RuntimeError("Xcode selected a different device; refusing to interact.") await self.get_screen_data(skip_settling=True) - except (OSError, ValueError, RuntimeError, TimeoutError, asyncio.CancelledError): + except ( + OSError, + ValueError, + RuntimeError, + TimeoutError, + asyncio.CancelledError, + ) as error: + if isinstance(error, XcodeApprovalRequiredError): + error.workspace_path = self._workspace_path try: await self.disconnect() except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 65e4196c..c11a2fef 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -16,11 +16,13 @@ import asyncio import os +from pathlib import Path from typing import Annotated from langchain_core.callbacks.base import Callbacks from artemis.config import checker_overrides_for_level, settings from artemis.context import DevicePlatform +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, xcode_approval_guidance from artemis.utils.startup_progress import publish_startup_progress from third_party.mobile_use.main import ( GoalArgument, @@ -37,6 +39,7 @@ import signal from rich.console import Console from rich.panel import Panel +from rich.text import Text import typer logger = get_logger(__name__) @@ -65,6 +68,7 @@ async def execute_task( explorer_pro_mode: str | None = None, verification_level: str | None = None, platform: DevicePlatform = DevicePlatform.ANDROID, + ios_workspace_path: Path | None = None, ) -> None: """Executes a single mobile automation task end-to-end. @@ -87,6 +91,8 @@ async def execute_task( 'strict'); applied before the explicit ``enable_checker`` switch. """ platform = DevicePlatform(platform) + if ios_workspace_path is not None and platform != DevicePlatform.IOS: + raise ValueError("--ios-workspace requires --platform ios.") if platform == DevicePlatform.IOS and os.environ.get("ARTEMIS_CLOUD_MODE") == "1": raise ValueError("iOS Simulator support is local only; cloud mode targets Android.") effective_sid = ( @@ -158,6 +164,9 @@ async def execute_task( if target_serial: config.for_device(platform, target_serial) + if ios_workspace_path is not None: + config.with_ios_workspace(ios_workspace_path) + if graph_config_callbacks: config.with_graph_config_callbacks(graph_config_callbacks) @@ -193,6 +202,19 @@ def run_command( help="Target mobile platform: Android (default) or a local iOS simulator.", ), ] = DevicePlatform.ANDROID, + ios_workspace: Annotated[ + Path | None, + typer.Option( + "--ios-workspace", + metavar="PATH", + rich_help_panel="Platform", + exists=True, + dir_okay=True, + file_okay=False, + resolve_path=True, + help="Existing Xcode project/workspace to request iOS first-run agent approval.", + ), + ] = None, locked_app_package: Annotated[ str | None, typer.Option( @@ -323,6 +345,9 @@ def run_command( ) is_standalone = standalone or os.environ.get("ARTEMIS_STANDALONE") == "1" + if ios_workspace is not None and platform != DevicePlatform.IOS: + raise typer.BadParameter("--ios-workspace requires --platform ios.") + if platform == DevicePlatform.IOS: if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": raise typer.BadParameter( @@ -465,6 +490,7 @@ def on_status(sess_info): explorer_pro_mode=explorer_pro_mode, verification_level=verification_level, platform=platform, + ios_workspace_path=ios_workspace, ) ) except (KeyboardInterrupt, asyncio.CancelledError): @@ -472,6 +498,17 @@ def on_status(sess_info): # this worker to stop through a cancel marker and the Agent cancelled # its own task so the recording and trace could be finalized. cancelled = True + except XcodeApprovalRequiredError as e: + console.print() + console.print( + Panel( + Text(f"{e}\n\n{xcode_approval_guidance(workspace_path=ios_workspace)}"), + title="Xcode Approval Required", + expand=False, + ) + ) + console.print() + raise SystemExit(2) except Exception as e: err_msg = str(e) if "API_KEY" in err_msg or "requires" in err_msg: diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index e28395d9..a99f6d81 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -151,7 +151,10 @@ async def _init_internal( publish_startup_progress( "device_check", "Checking the iOS simulator", session_id=self._session_id ) - driver = XcodeSimulatorDriver(device_id=self._config.device_id or "booted") + driver = XcodeSimulatorDriver( + device_id=self._config.device_id or "booted", + workspace_path=getattr(self._config, "ios_workspace_path", None), + ) self._ios_driver = driver try: # Resolve the simulator without booting it or opening a native UI diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index c982a3d8..772f8ef2 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -15,6 +15,7 @@ """Builder for AgentConfig objects using a fluent interface.""" import os +from pathlib import Path from typing import Any, cast from artemis.config import ( @@ -42,6 +43,7 @@ def __init__(self): self._disable_checker: bool = False self._concurrency_mode: str = "per_device" self._max_concurrency: int | None = None + self._ios_workspace_path: Path | None = None agent_cfg = load_agent_config() self._explorer = agent_cfg.explorer @@ -76,10 +78,21 @@ def for_device_serial(self, device_serial: str) -> "AgentConfigBuilder": """Target a specific Android device by its ADB serial number.""" return self.for_device(DevicePlatform.ANDROID, device_serial) - def for_ios_simulator(self, device_id: str = "booted") -> "AgentConfigBuilder": + def for_ios_simulator( + self, device_id: str = "booted", *, workspace_path: str | Path | None = None + ) -> "AgentConfigBuilder": """Target an iOS simulator UDID, or the single booted simulator.""" + if workspace_path is not None: + self.with_ios_workspace(workspace_path) return self.for_device(DevicePlatform.IOS, device_id) + def with_ios_workspace(self, workspace_path: str | Path | None) -> "AgentConfigBuilder": + """Set an existing Xcode project/workspace for iOS first-run approval.""" + self._ios_workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) + return self + def with_concurrency_mode(self, mode: str) -> "AgentConfigBuilder": """Configure concurrency mode: 'global' (1 task globally) or 'per_device' (1 task per device).""" self._concurrency_mode = str(mode).strip().lower() @@ -458,6 +471,7 @@ def _extra_config_fields(self) -> dict[str, Any]: ), "concurrency_mode": self._concurrency_mode, "max_concurrency": self._max_concurrency, + "ios_workspace_path": self._ios_workspace_path, } def build(self, validate_profiles: bool = True) -> AgentConfig: diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 10f4e1d9..72e41db5 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path from typing import Literal from artemis.config import ( @@ -71,6 +72,10 @@ class AgentConfig(AgentConfigBase): video_analyzer: VideoAnalyzerConfig = Field(default_factory=VideoAnalyzerConfig) concurrency_mode: Literal["global", "per_device"] = "per_device" max_concurrency: int | None = None + ios_workspace_path: Path | None = Field( + default=None, + description="Optional existing Xcode project/workspace for local iOS first-run approval; no permission grants.", + ) model_config = {"arbitrary_types_allowed": True} diff --git a/docs/ios.md b/docs/ios.md index b3cff15b..19dc9d9b 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -77,6 +77,33 @@ To install a simulator build before the task, add `--app-path /absolute/path/MyApp.app`. The bundle's `Info.plist` must provide `CFBundleIdentifier`. +### First-run Xcode approval + +The very first time a new Python interpreter asks Xcode for device access, +Xcode requires user approval for that interpreter and, if one is supplied, the +selected project folder. Point Artemis at an existing Xcode project or +workspace so the request can be recorded: + +```bash +uv run artemis run "Open Settings" --platform ios --standalone \ + --ios-workspace /absolute/path/MyApp.xcodeproj +``` + +If approval is still pending, the run stops with an "Xcode Approval Required" +panel instead of retrying. Approve the interpreter and the selected folder from +the Xcode MCP menu bar icon, choosing **Always Allow** there if offered and you +want later runs to skip the prompt; then rerun the task. Advanced users can instead +inspect pending request IDs with `xcrun mcp-server status` and approve only +those entries via `sudo xcrun mcp-server approve --always` from +their own terminal; the CLI path requires admin rights and is performed by the +user, never by Artemis. + +Granted approvals persist across fresh bridge processes; there is no need to +keep a bridge alive. A different interpreter path or build, a different project +folder, or an expiring grant can require approval again. The Always/persistent +choice belongs to you and Xcode; Artemis requests only scoped approval for its +interpreter and the folder you select and never enables global access. + ## Embedded Python SDK Configure iOS through the embedded SDK's builder: @@ -94,6 +121,24 @@ config = ( agent = Agent(config=config) ``` +To run the first-run approval flow against a specific project, pass the +existing project or workspace through the builder: + +```python +config = ( + AgentConfigBuilder() + .for_ios_simulator( + "", + workspace_path="/absolute/path/MyApp.xcodeproj", + ) + .build() +) +``` + +`with_ios_workspace(path)` applies the same setting, and `None` clears it. +The path is used only if Xcode refuses the initial session with an approval +error; already-approved runs never open a workspace. + The generic builder also accepts `for_device(DevicePlatform.IOS, "")`, with `DevicePlatform` imported from `artemis.context`. Supplying only a `device_serial` without an iOS diff --git a/tests/unit/drivers/test_ios_bridge.py b/tests/unit/drivers/test_ios_bridge.py index 7afb6c09..ceb7d06e 100644 --- a/tests/unit/drivers/test_ios_bridge.py +++ b/tests/unit/drivers/test_ios_bridge.py @@ -14,13 +14,20 @@ import asyncio from contextlib import asynccontextmanager import json +from pathlib import Path +import subprocess +import sys from types import SimpleNamespace from mcp.types import CallToolResult, TextContent import pytest from artemis.drivers.ios import bridge -from artemis.drivers.ios.bridge import XcodeBridge +from artemis.drivers.ios.bridge import ( + XcodeApprovalRequiredError, + XcodeBridge, + xcode_approval_guidance, +) def response(data=None, *, text=None, error=False): @@ -162,7 +169,8 @@ async def call(name, arguments): client = XcodeBridge() await client.start() with pytest.raises( - RuntimeError, match="DeviceInteractionStartSession failed: Agent is not approved" + XcodeApprovalRequiredError, + match="DeviceInteractionStartSession failed: Agent is not approved", ): await client.call("DeviceInteractionStartSession", {}) assert await client.call("DeviceInteractionEndSession", {}) == {"userMessage": "Closed"} @@ -173,6 +181,73 @@ async def call(name, arguments): await client.close() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "native_message", + [ + "This agent isn't approved to use Xcode's tools yet. Call XcodeOpenWorkspace first.", + "Agent is not approved", + "Xcode is waiting for the user to approve this request; it has been recorded.", + ], +) +async def test_native_approval_refusals_become_typed_errors(native_transport, native_message): + async def call(name, arguments): + return response(text=native_message, error=True) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises(XcodeApprovalRequiredError) as caught: + await client.call("DeviceInteractionStartSession", {}) + error = caught.value + assert error.tool_name == "DeviceInteractionStartSession" + assert error.native_message == native_message + assert str(error) == f"Xcode tool DeviceInteractionStartSession failed: {native_message}" + await client.close() + + +@pytest.mark.asyncio +async def test_unrelated_tool_errors_stay_generic_and_keep_bridge_alive(native_transport): + results = iter( + [response(text="Simulator is busy", error=True), response({"userMessage": "OK"})] + ) + + async def call(name, arguments): + return next(results) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises(RuntimeError, match="Simulator is busy") as caught: + await client.call("DeviceInteractionSynthesize", {}) + assert not isinstance(caught.value, XcodeApprovalRequiredError) + assert await client.call("DeviceInteractionEndSession", {}) == {"userMessage": "OK"} + await client.close() + + +def test_approval_guidance_names_interpreter_and_workspace_without_subprocess( + tmp_path, monkeypatch +): + def forbidden(*args, **kwargs): + raise AssertionError("guidance must never launch a subprocess") + + monkeypatch.setattr(subprocess, "run", forbidden) + monkeypatch.setattr(subprocess, "Popen", forbidden) + monkeypatch.setattr(bridge.asyncio, "create_subprocess_exec", forbidden) + guidance = xcode_approval_guidance(workspace_path=tmp_path) + assert str(Path(sys.executable).resolve()) in guidance + assert str(tmp_path) in guidance + assert "Always Allow" in guidance + assert "--always" in guidance + assert "not supplied" not in guidance + bare = xcode_approval_guidance() + assert "Workspace: not supplied" in bare + assert isinstance(XcodeApprovalRequiredError("tool", "msg").guidance, str) + scoped = XcodeApprovalRequiredError("tool", "msg", workspace_path=tmp_path) + assert scoped.workspace_path == tmp_path + assert str(tmp_path) in scoped.guidance + + @pytest.mark.asyncio async def test_unstructured_success_is_rejected(native_transport): async def call(name, arguments): diff --git a/tests/unit/drivers/test_ios_driver.py b/tests/unit/drivers/test_ios_driver.py index 37af12df..f45c9d9a 100644 --- a/tests/unit/drivers/test_ios_driver.py +++ b/tests/unit/drivers/test_ios_driver.py @@ -25,6 +25,7 @@ from artemis.drivers.base import KeyCode from artemis.drivers.ios import xcode_driver +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver, run_xcrun from third_party.mobile_use.controllers.types import ElementQuery @@ -63,6 +64,7 @@ def __init__(self, observation): self.close = AsyncMock(side_effect=self.finish) self.call = AsyncMock(side_effect=self.respond) self.observation = observation + self.hooks = {} self.start_result = { "interactionSessionKey": "test-session", "deviceUUID": IOS_A, @@ -76,12 +78,16 @@ async def finish(self): self.connected = False async def respond(self, name, arguments): + if name in self.hooks: + return await self.hooks[name](arguments) if name == "DeviceInteractionStartSession": return self.start_result if name == "DeviceInteractionSynthesize": return self.observation if name == "DeviceInteractionEndSession": return {"userMessage": "Closed"} + if name == "XcodeOpenWorkspace": + return {"workspaceIdentifier": "workspace-1"} raise AssertionError(f"Unexpected native tool {name}") @@ -612,3 +618,212 @@ async def test_simctl_failures_report_stderr_and_preserve_literal_argv(monkeypat with pytest.raises(RuntimeError, match="No app"): await run_xcrun("simctl", "openurl", IOS_A, url) assert spawn.await_args.args == ("xcrun", "simctl", "openurl", IOS_A, url) + + +DENIAL = "This agent isn't approved to use Xcode's tools yet. Call XcodeOpenWorkspace first." +PENDING = "Xcode is waiting for the user to approve this request; it has been recorded." + + +def _tool_names(native): + return [call.args[0] for call in native.call.call_args_list] + + +@pytest.mark.asyncio +async def test_initial_approval_refusal_opens_workspace_once_and_retries_start(simulator, tmp_path): + project = tmp_path / "Example App [2].xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + starts = [] + + async def start_hook(arguments): + starts.append(dict(arguments)) + if len(starts) == 1: + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + return native.start_result + + native.hooks["DeviceInteractionStartSession"] = start_hook + driver = XcodeSimulatorDriver(workspace_path=project) + await driver.connect() + assert _tool_names(native) == [ + "DeviceInteractionStartSession", + "XcodeOpenWorkspace", + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + ] + assert native.call.call_args_list[1].args == ( + "XcodeOpenWorkspace", + {"path": str(project)}, + ) + assert starts[0] == starts[1] + assert starts[0]["deviceIdentifier"] == IOS_A + invoked = [call.args for call in simulator.commands.call_args_list] + assert ("simctl", "boot") not in [args[:2] for args in invoked] + assert not any("mcp-server" in args for args in invoked) + await driver.disconnect() + assert "DeviceInteractionEndSession" in _tool_names(native) + + +@pytest.mark.asyncio +async def test_pending_workspace_approval_propagates_without_retry(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + async def pending_open(arguments): + raise XcodeApprovalRequiredError("XcodeOpenWorkspace", PENDING) + + native.hooks["DeviceInteractionStartSession"] = fail_start + native.hooks["XcodeOpenWorkspace"] = pending_open + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(XcodeApprovalRequiredError, match="waiting for the user") as caught: + await driver.connect() + assert caught.value.workspace_path == project + assert str(project) in caught.value.guidance + assert "not supplied" not in caught.value.guidance + assert _tool_names(native) == ["DeviceInteractionStartSession", "XcodeOpenWorkspace"] + native.start.assert_awaited_once() + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("identifier", [None, "", 123]) +async def test_malformed_workspace_acceptance_fails_closed(simulator, tmp_path, identifier): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + async def open_workspace(arguments): + return {"workspaceIdentifier": identifier} + + native.hooks["DeviceInteractionStartSession"] = fail_start + native.hooks["XcodeOpenWorkspace"] = open_workspace + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(RuntimeError, match="no usable workspace identifier"): + await driver.connect() + assert _tool_names(native) == ["DeviceInteractionStartSession", "XcodeOpenWorkspace"] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_still_pending_after_workspace_open_stops_after_one_retry(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", PENDING) + + native.hooks["DeviceInteractionStartSession"] = fail_start + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(XcodeApprovalRequiredError, match="waiting for the user") as caught: + await driver.connect() + assert caught.value.workspace_path == project + assert _tool_names(native) == [ + "DeviceInteractionStartSession", + "XcodeOpenWorkspace", + "DeviceInteractionStartSession", + ] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_approval_refusal_without_workspace_propagates_guidance(simulator): + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + native.hooks["DeviceInteractionStartSession"] = fail_start + with pytest.raises(XcodeApprovalRequiredError) as caught: + await simulator.driver.connect() + assert _tool_names(native) == ["DeviceInteractionStartSession"] + assert caught.value.workspace_path is None + assert "Workspace: not supplied" in caught.value.guidance + assert "--ios-workspace" in caught.value.guidance + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_approved_sessions_never_open_workspace(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + simulator.native.tools.add("XcodeOpenWorkspace") + driver = XcodeSimulatorDriver(workspace_path=project) + await driver.connect() + await driver.connect() + names = _tool_names(simulator.native) + assert "XcodeOpenWorkspace" not in names + assert names.count("DeviceInteractionStartSession") == 1 + await driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("native boom"), TimeoutError("slow tool")]) +async def test_non_approval_failures_skip_workspace_and_retry(simulator, tmp_path, failure): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail(arguments): + raise failure + + native.hooks["DeviceInteractionStartSession"] = fail + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(type(failure)): + await driver.connect() + assert _tool_names(native) == ["DeviceInteractionStartSession"] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_missing_workspace_tool_reports_actionable_approval_error(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + native.hooks["DeviceInteractionStartSession"] = fail_start + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(XcodeApprovalRequiredError, match="XcodeOpenWorkspace") as caught: + await driver.connect() + assert caught.value.workspace_path == project + assert str(project) in caught.value.guidance + assert _tool_names(native) == ["DeviceInteractionStartSession"] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_workspace_must_be_existing_project_directory(simulator, tmp_path): + (tmp_path / "plain-folder").mkdir() + (tmp_path / "file.xcodeproj").write_text("x") + bad_paths = [ + tmp_path / "missing.xcodeproj", + tmp_path / "plain-folder", + tmp_path / "file.xcodeproj", + ] + for path in bad_paths: + driver = XcodeSimulatorDriver(workspace_path=path) + with pytest.raises(ValueError, match="xcodeproj or .xcworkspace"): + await driver.connect() + simulator.commands.assert_not_called() + simulator.native.start.assert_not_called() + + workspace = tmp_path / "Nested Dir [x].xcworkspace" + workspace.mkdir() + driver = XcodeSimulatorDriver(workspace_path=workspace) + await driver.connect() + await driver.disconnect() diff --git a/tests/unit/sdk/test_agent_config_builder_tuning.py b/tests/unit/sdk/test_agent_config_builder_tuning.py index ad200411..6b808c9d 100644 --- a/tests/unit/sdk/test_agent_config_builder_tuning.py +++ b/tests/unit/sdk/test_agent_config_builder_tuning.py @@ -121,3 +121,37 @@ def test_with_explorer_versions_is_an_advanced_per_agent_override(monkeypatch): ) assert cfg.get_explorer_version(agent_name="validator") == "ultra" assert cfg.get_explorer_version(agent_name="operator") == "flash" + + +def test_ios_workspace_path_defaults_to_none_and_survives_build(): + cfg = AgentConfigBuilder().build() + assert cfg.ios_workspace_path is None + + +def test_with_ios_workspace_accepts_str_and_path_and_clears(tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + builder = AgentConfigBuilder() + cfg = builder.with_ios_workspace(str(project)).build() + assert cfg.ios_workspace_path == project + cfg = builder.with_ios_workspace(None).build() + assert cfg.ios_workspace_path is None + + +def test_for_ios_simulator_workspace_kwarg_and_omission(tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + cfg = ( + AgentConfigBuilder() + .for_ios_simulator(workspace_path=project) + .build(validate_profiles=False) + ) + assert cfg.ios_workspace_path == project + # Omitting the kwarg does not clear a workspace set earlier on the builder. + cfg = ( + AgentConfigBuilder() + .with_ios_workspace(project) + .for_ios_simulator() + .build(validate_profiles=False) + ) + assert cfg.ios_workspace_path == project diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index cc1d12cd..4392ed7a 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -49,6 +49,7 @@ def test_cli_run_help(): assert "--verification-level" in result.output assert "--explorer-pro-mode" in result.output assert "--platform" in result.output + assert "--ios-workspace" in result.output def test_cli_batch_help(): diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py index 6d82d321..82424709 100644 --- a/tests/unit/test_ios_integration.py +++ b/tests/unit/test_ios_integration.py @@ -38,6 +38,7 @@ from artemis.sdk.agent import Agent from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder from artemis.tools.mobile.launch_app import find_package, launch_app +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError from artemis.runtime import DeviceBusyError from third_party.mobile_use.sdk.agent import AgentBase from third_party.mobile_use.sdk.types.exceptions import AgentError, AgentNotInitializedError @@ -109,10 +110,30 @@ def test_factory_selects_ios_without_creating_adb(native_driver, monkeypatch): monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) context = ios_context() assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted") + native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=None) assert context.adb_client is None +def test_factory_forwards_configured_ios_workspace(native_driver, tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + context = ios_context() + context.agent_config = ( + AgentConfigBuilder() + .for_ios_simulator() + .with_ios_workspace(project) + .build(validate_profiles=False) + ) + assert create_driver(context) is native_driver + native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=project) + + +def test_factory_tolerates_missing_agent_config(native_driver): + context = ios_context() + assert create_driver(context) is native_driver + native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=None) + + def test_macos_host_keeps_android_default(monkeypatch): context = ArtemisContext(device=DeviceContext(host_platform="DARWIN", device_id="android-1")) android = MagicMock() @@ -152,6 +173,22 @@ async def test_sdk_ios_initialization_bypasses_android_and_cleans(native_driver, assert agent._initialized is False +@pytest.mark.asyncio +async def test_sdk_init_forwards_ios_workspace(native_driver, tmp_path): + project = tmp_path / "Example.xcodeproj" + project.mkdir() + config = ( + AgentConfigBuilder() + .for_ios_simulator(workspace_path=project) + .build(validate_profiles=False) + ) + agent = Agent(config=config) + agent._prewarm_llm_connections = AsyncMock() + assert await agent.init() is True + native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=project) + await agent.clean() + + @pytest.mark.asyncio async def test_sdk_failed_ios_init_closes_partial_transport(native_driver): native_driver.resolve_device.side_effect = RuntimeError("bridge failed") @@ -284,6 +321,31 @@ async def test_execute_task_ios_ignores_android_selection(monkeypatch): automation.assert_awaited_once() +@pytest.mark.asyncio +async def test_execute_task_ios_forwards_workspace(native_driver, monkeypatch, tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + builder = MagicMock() + automation = AsyncMock() + monkeypatch.setattr(run_module, "new_default_config_builder", lambda: builder) + monkeypatch.setattr(run_module, "run_automation", automation) + await run_module.execute_task( + "Open Settings", platform=DevicePlatform.IOS, ios_workspace_path=project + ) + builder.with_ios_workspace.assert_called_once_with(project) + automation.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_execute_task_rejects_workspace_without_ios(native_driver, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + with pytest.raises(ValueError, match="--platform ios"): + await run_module.execute_task( + "Open Settings", platform=DevicePlatform.ANDROID, ios_workspace_path=project + ) + + def test_cli_ios_runs_locally_without_android_status_or_daemon(monkeypatch): execute = AsyncMock() monkeypatch.setattr(run_module, "execute_task", execute) @@ -305,6 +367,52 @@ def test_cli_ios_runs_locally_without_android_status_or_daemon(monkeypatch): assert execute.call_args.kwargs["device_serial"] == "booted" +def test_cli_forwards_ios_workspace(monkeypatch, tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + result = CliRunner().invoke( + app, + ["run", "--platform", "ios", "--ios-workspace", str(project), "Open Settings"], + ) + assert result.exit_code == 0, result.output + assert execute.call_args.kwargs["ios_workspace_path"] == project + + +def test_cli_rejects_ios_workspace_for_android(monkeypatch, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setattr( + "artemis.runtime.ensure_daemon_running", MagicMock(side_effect=AssertionError("daemon")) + ) + result = CliRunner().invoke(app, ["run", "--ios-workspace", str(project), "Open Settings"]) + assert result.exit_code != 0 + execute.assert_not_called() + + +def test_cli_approval_error_exits_2_with_guidance_panel(monkeypatch): + monkeypatch.setattr( + run_module, + "execute_task", + AsyncMock( + side_effect=XcodeApprovalRequiredError( + "DeviceInteractionStartSession", + "This agent isn't approved to use Xcode's tools yet.", + ) + ), + ) + result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) + assert result.exit_code == 2, result.output + assert "Xcode Approval Required" in result.output + assert "Always Allow" in result.output + assert "isn't approved" in result.output + assert "Missing API Key" not in result.output + assert "GEMINI" not in result.output + + @pytest.mark.parametrize("env_name", ["ARTEMIS_CLOUD_MODE", "ARTEMIS_TASK_WORKER"]) def test_cli_rejects_ios_android_services_before_run(monkeypatch, env_name): execute = AsyncMock() From a6fcf812cdd937eb81abc631294f54f20c1a60d9 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 00:59:52 +0900 Subject: [PATCH 03/32] fix: preserve native iOS text field metadata --- artemis/drivers/ios/hierarchy.py | 28 ++++++++- tests/unit/drivers/test_ios_driver.py | 89 +++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/artemis/drivers/ios/hierarchy.py b/artemis/drivers/ios/hierarchy.py index b0cf4dc2..bddf69e7 100644 --- a/artemis/drivers/ios/hierarchy.py +++ b/artemis/drivers/ios/hierarchy.py @@ -77,10 +77,32 @@ def parse_hierarchy( label = re.search(r"label:\s*'((?:\\.|[^'])*)'", line) quoted = re.search(r'"([^"\n]*)"', line[: match.start()]) identifier = re.search(r"identifier:\s*'((?:\\.|[^'])*)'", line) + placeholder = re.search(r"placeholderValue:\s*'((?:\\.|[^'])*)'", line) + value = re.search(r"value:\s*'((?:\\.|[^'])*)'", line) + if value: + value_text = value.group(1) + else: + # Xcode elides long values and drops quotes, e.g. `value: Text...`, + # so read until the next native metadata delimiter verbatim. + unquoted = re.search( + r"\bvalue:\s*(.*?)(?=,\s*(?:Keyboard Focused\b|Selected\b|Disabled\b" + r"|hitPoint:|activationBundleId:|identifier:|label:|placeholderValue:)|$)", + line, + ) + value_text = unquoted.group(1).strip() if unquoted else None hit = _HIT.search(line) activation = re.search(r"activationBundleId:\s*(\S+)", line) + text = "" + if label: + text = label.group(1) + elif quoted: + text = quoted.group(1) + elif value_text: + text = value_text + elif placeholder and placeholder.group(1): + text = placeholder.group(1) element: dict[str, Any] = { - "text": label.group(1) if label else quoted.group(1) if quoted else "", + "text": text, "resource_id": identifier.group(1) if identifier else "", "class": line.strip().split(",", 1)[0].split(" ", 1)[0], "bounds": f"[{left},{top}][{right},{bottom}]", @@ -91,5 +113,9 @@ def parse_hierarchy( element["hit_point"] = [round(hx * scale[0]), round(hy * scale[1])] if activation: element["activation_bundle_id"] = activation.group(1) + if placeholder: + element["placeholder"] = placeholder.group(1) + if value_text is not None: + element["value"] = value_text elements.append(element) return elements, scale diff --git a/tests/unit/drivers/test_ios_driver.py b/tests/unit/drivers/test_ios_driver.py index f45c9d9a..117aedc2 100644 --- a/tests/unit/drivers/test_ios_driver.py +++ b/tests/unit/drivers/test_ios_driver.py @@ -555,6 +555,95 @@ def test_inaccessible_canvas_can_use_native_logical_screenshot_space(): assert scale == (1, 1) +EMPTY_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "placeholderValue: 'Habit to avoid', Keyboard Focused, hitPoint: {201.0, 210.0}" +) +FILLED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "placeholderValue: 'Habit to avoid', value: caf\u00e9 \U0001f642, " + "Keyboard Focused, hitPoint: {201.0, 210.0}" +) +TRUNCATED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "placeholderValue: 'Habit to avoid', value: Artemis iOS direct..., " + "Keyboard Focused, hitPoint: {201.0, 210.0}" +) +COMMA_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "value: hello, world, Keyboard Focused, hitPoint: {201.0, 210.0}" +) +QUOTED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "value: 'quoted text', Keyboard Focused, hitPoint: {201.0, 210.0}" +) +LABELED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, label: 'Title', " + "value: 'typed', placeholderValue: 'Habit to avoid', hitPoint: {201.0, 210.0}" +) + + +def test_empty_textfield_exposes_placeholder_as_text_and_metadata(): + elements, _ = parse_hierarchy(EMPTY_FIELD, 402, 874) + assert len(elements) == 1 + field = elements[0] + assert field["class"] == "TextField" + assert field["placeholder"] == "Habit to avoid" + assert field["text"] == "Habit to avoid" + assert "value" not in field + assert field["hit_point"] == [201, 210] + + +def test_populated_unlabeled_textfield_exposes_value_as_text_and_metadata(): + elements, _ = parse_hierarchy(FILLED_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "caf\u00e9 \U0001f642" + assert field["text"] == "caf\u00e9 \U0001f642" + assert field["placeholder"] == "Habit to avoid" + assert field["hit_point"] == [201, 210] + + +def test_elided_unquoted_value_is_preserved_verbatim(): + elements, _ = parse_hierarchy(TRUNCATED_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "Artemis iOS direct..." + assert field["text"] == "Artemis iOS direct..." + + +def test_unquoted_value_keeps_ordinary_commas_before_metadata(): + elements, _ = parse_hierarchy(COMMA_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "hello, world" + assert field["text"] == "hello, world" + + +def test_quoted_value_format_still_parsed(): + elements, _ = parse_hierarchy(QUOTED_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "quoted text" + assert field["text"] == "quoted text" + + +def test_labeled_textfield_keeps_label_text_and_retains_field_metadata(): + elements, _ = parse_hierarchy(LABELED_FIELD, 402, 874) + field = elements[0] + assert field["text"] == "Title" + assert field["value"] == "typed" + assert field["placeholder"] == "Habit to avoid" + assert field["hit_point"] == [201, 210] + + +def test_unquoted_native_value_metadata_keeps_label_text(): + line = ( + "Other, {{369.0, 132.0}, {30.0, 414.0}}, " + "label: 'Vertical scroll bar, 1 page', value: 0%, hitPoint: {384.0, 339.0}" + ) + elements, _ = parse_hierarchy(line, 402, 874) + field = elements[0] + assert field["text"] == "Vertical scroll bar, 1 page" + assert field["value"] == "0%" + + def test_foreground_package_is_unknown_when_multiple_apps_overlap(): assert application_bundle(APPLE_HIERARCHY) == "com.example.app" assert ( From 381281f3719c692728795fa16f63f97cecc47875 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 04:10:13 +0900 Subject: [PATCH 04/32] feat: add native iOS Simulator video recording Capture via xcrun simctl io recordVideo anchored to the first-frame marker, seal segments on demand for live analyzer clips, roll on rotation/duration/crash, and finalize VFR .mov segments to CFR MP4 with a version-2 manifest so iOS recordings flow through the same DataEngine, extraction, and replay pipeline as Android scrcpy. Video tools now auto-detect per platform (simctl+ffmpeg on iOS) and the CLI/SDK no longer disable recording for iOS tasks. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/controllers/unified_controller.py | 271 +++++++- artemis/drivers/ios/recording.py | 515 ++++++++++++++++ artemis/drivers/ios/xcode_driver.py | 77 ++- artemis/interfaces/cli/commands/run.py | 8 +- artemis/sdk/agent.py | 7 +- artemis/sdk/builders/agent_config_builder.py | 15 +- artemis/sdk/types/agent.py | 13 +- artemis/utils/video.py | 71 ++- docs/ios.md | 13 +- tests/unit/drivers/test_ios_recording.py | 611 +++++++++++++++++++ tests/unit/test_ios_integration.py | 104 +++- tests/unit/test_unified_controller_video.py | 255 ++++++++ 12 files changed, 1898 insertions(+), 62 deletions(-) create mode 100644 artemis/drivers/ios/recording.py create mode 100644 tests/unit/drivers/test_ios_recording.py diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 818d4caa..0a6311e9 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -19,12 +19,13 @@ import subprocess import tempfile import time -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 from artemis.config.paths import get_temp_dir from artemis.context import ArtemisContext from artemis.drivers.factory import get_driver +from artemis.drivers.ios.recording import IosRecordingSession from artemis.utils.video import ( ANDROID_RECORDING_SEGMENT_SECONDS, await_scrcpy_first_frame, @@ -48,6 +49,9 @@ set_active_session, ) +if TYPE_CHECKING: + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + logger = get_logger(__name__) @@ -146,6 +150,21 @@ async def extract_segment_metadata( ) return cached_res + if isinstance(session, IosRecordingSession): + result = await self._extract_ios_segment(session, start_time, end_time, output_path) + if ( + cache_key is not None + and result.success + and result.video_path + and result.video_path.exists() + ): + # Sealing bumps the generation; key the entry under the + # post-seal generation a repeated lookup will observe. + self._segment_cache[ + (cache_key[0], session.generation, cache_key[2], cache_key[3]) + ] = result + return result + try: mkv_path = session.local_video_path if not mkv_path: @@ -412,13 +431,10 @@ async def start_video_recording( output_dir: Path | None = None, max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, ) -> VideoRecordingResult: - """Start screen recording on Android device using scrcpy.""" + """Start screen recording using the platform's native capture.""" self._segment_cache.clear() if self.ctx.device.mobile_platform == "ios": - return VideoRecordingResult( - success=False, - message="iOS video analysis is not supported by the recording controller.", - ) + return await self._start_ios_recording(output_dir, max_duration_seconds) device_id = self._get_device_id() # Check mock driver first @@ -537,12 +553,10 @@ async def start_video_recording( return abort_recording(device_id, "start", e) async def stop_video_recording(self) -> VideoRecordingResult: - """Stop scrcpy recording and return the converted MP4 video file.""" + """Stop recording and return the finalized MP4 video file.""" self._segment_cache.clear() if self.ctx.device.mobile_platform == "ios": - return VideoRecordingResult( - success=False, message="No iOS controller recording is active." - ) + return await self._stop_ios_recording() device_id = self._get_device_id() # Check mock driver first @@ -669,5 +683,242 @@ async def stop_video_recording(self) -> VideoRecordingResult: self._record_recording_failure(session, str(e)) return abort_recording(device_id, "stop", e) + def _ios_recording_driver(self) -> "XcodeSimulatorDriver | None": + """The active driver when it supports native iOS capture, else ``None``.""" + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + return self._driver if isinstance(self._driver, XcodeSimulatorDriver) else None + + async def _start_ios_recording( + self, output_dir: Path | None, max_duration_seconds: int + ) -> VideoRecordingResult: + """Start native simctl capture through the iOS driver.""" + device_id = self._get_device_id() + if already_active := recording_already_active(device_id): + return already_active + driver = self._ios_recording_driver() + if driver is None: + return VideoRecordingResult( + success=False, + message="iOS recording requires the Xcode simulator driver.", + ) + try: + await driver.start_video_recording( + output_dir, max_duration_seconds=max_duration_seconds + ) + except Exception as exc: + session = driver.recording_session + if session is None: + session = IosRecordingSession( + video_id=uuid4(), + device_id=device_id, + start_time=time.time(), + local_video_path=(Path(output_dir) / "recording.mov" if output_dir else None), + is_active=False, + ) + if session.data_engine_start_time is None and self.ctx and self.ctx.data_engine: + session.data_engine_start_time = ( + self.ctx.data_engine.session_start_time or session.start_time + ) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=session.video_id, + device_id=device_id, + local_video_path=session.local_video_path or "", + start_time=session.start_time, + ) + self._record_recording_failure(session, f"iOS recording failed to start: {exc}") + return VideoRecordingResult( + success=False, message=f"iOS recording failed to start: {exc}" + ) + session = driver.recording_session + if session is None: + return VideoRecordingResult(success=False, message="iOS recorder returned no session") + session.data_engine_start_time = ( + self.ctx.data_engine.session_start_time + if (self.ctx and self.ctx.data_engine) + else session.start_time + ) + set_active_session(device_id, session) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=session.video_id, + device_id=device_id, + local_video_path=session.local_video_path, + start_time=session.start_time, + ) + warning = "; ".join(session.errors) or None + return VideoRecordingResult( + success=True, + message=f"iOS recording started on {device_id}", + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:active", + warning=warning, + ) + + async def _stop_ios_recording(self) -> VideoRecordingResult: + """Finalize the driver's recorder and report the sealed manifest.""" + device_id = self._get_device_id() + driver = self._ios_recording_driver() + session = get_active_session(device_id) + if session is None: + session = driver.recording_session if driver is not None else None + if session is None or session.video_id is None: + return no_active_recording(device_id) + if driver is None: + message = "iOS recording stop requires the Xcode simulator driver." + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + async def _finalize() -> str | None: + session.is_active = False + return await driver.stop_video_recording() + + try: + final_path = await asyncio.shield(_finalize()) + except asyncio.CancelledError: + raise + except Exception as exc: + self._record_recording_failure(session, f"iOS recording stop failed: {exc}") + return VideoRecordingResult(success=False, message=f"iOS recording stop failed: {exc}") + finally: + remove_active_session(device_id) + + missing = [ + record + for record in session.segments + if not ( + record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ) + ] + if final_path is None or not session.segments or missing: + message = "; ".join(session.errors) or "iOS recording finalized with no valid segments" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + final_video_path = Path(final_path) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_stop( + video_id=session.video_id, + device_id=device_id, + local_video_path=final_video_path, + start_time=session.start_time, + end_time=session.start_time + session.sealed_until, + ) + warning = "; ".join(session.errors) or None + return VideoRecordingResult( + success=True, + message=f"iOS recording stopped, saved {len(session.segments)} segments", + video_path=final_video_path, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:ready", + warning=warning, + ) + + async def _extract_ios_segment( + self, + session: IosRecordingSession, + start_time: float, + end_time: float | None, + output_path: Path | None, + ) -> VideoRecordingResult: + """Render a clip from sealed iOS MP4 segments only. + + The open .mov is never handed to FFmpeg: the driver seals the segment + at the requested recording-relative end first, then finalized segments + render through the shared timeline renderer. Recorder restarts leave + their gaps black rather than stretching adjacent frames. + """ + device_id = self._get_device_id() + anchor = session.data_engine_start_time + shift = (session.start_time - anchor) if anchor is not None else 0.0 + recording_start = float(start_time) - shift + recording_end = float(end_time) - shift if end_time is not None else None + + driver = self._ios_recording_driver() + if session.is_active and recording_end is not None and driver is not None: + await driver.seal_recording_segment(through_time=recording_end) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + + warning = None + available_end = session.sealed_until + if recording_end is None or recording_end > available_end: + if recording_end is not None: + warning = ( + "Requested end clipped to latest sealed iOS capture " + f"({available_end + shift:.1f}s of session time)" + ) + recording_end = available_end + + records = [dict(record) for record in session.segments] + first_capture = min((float(record["start"]) for record in records), default=None) + if first_capture is not None and recording_start < first_capture: + leading = "Requested start preceded the first captured frame; clipped to capture start" + warning = f"{warning}; {leading}" if warning else leading + recording_start = first_capture + + if not records or recording_end <= recording_start: + message = "Requested range does not overlap sealed iOS recording" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + timeline = [ + { + "path": str(record["output_path"]), + "start": float(record["start"]), + "end": float(record["end"]), + } + for record in records + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and float(record["end"]) > recording_start + and float(record["start"]) < recording_end + ] + timeline.sort(key=lambda segment: segment["start"]) + if not timeline: + message = "No finalized iOS MP4 segments overlap the requested range" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + trim_output_path = ( + Path(output_path) + if output_path is not None + else Path(tempfile.mkdtemp(prefix="video_trimmed_", dir=get_temp_dir("trimmed_videos"))) + / "segment.mp4" + ) + success = await render_timeline_clip( + timeline, recording_start, recording_end, trim_output_path + ) + if not success or not trim_output_path.exists(): + message = "Failed to render clip from sealed iOS segments" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + actual_start = recording_start + shift + actual_end = recording_end + shift + file_size_mb = trim_output_path.stat().st_size / (1024 * 1024) + return VideoRecordingResult( + success=True, + message=( + f"iOS clip rendered for session range {actual_start:.1f}s to {actual_end:.1f}s" + ), + video_path=trim_output_path, + file_size_mb=round(file_size_mb, 2), + duration_seconds=round(recording_end - recording_start, 2), + actual_start_relative_time=actual_start, + warning=warning, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=(f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"), + ) + async def cleanup(self) -> None: await self._driver.disconnect() diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py new file mode 100644 index 00000000..ba89c45f --- /dev/null +++ b/artemis/drivers/ios/recording.py @@ -0,0 +1,515 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS Simulator recording via ``xcrun simctl io recordVideo``. + +simctl writes a variable frame-rate H.264 ``.mov`` and announces the first +processed frame on stderr as ``Recording started``; that marker anchors the +recording timeline, not process spawn. Each owned ``.mov`` is finalized to a +browser-safe CFR MP4 whose last frame is cloned across the segment's static +tail, so recorded event timestamps are never stretched. iOS capture has no +audio. Segments roll on display-dimension change, max duration, or recorder +exit; restart gaps stay gaps in rendered clips. +""" + +import asyncio +from io import BytesIO +from pathlib import Path +import signal +import tempfile +import time +from typing import Any +from uuid import uuid4 + +from PIL import Image + +from artemis.config.paths import get_temp_dir +from artemis.utils.video import ( + get_ffmpeg_path, + probe_video_segment, + write_recording_manifest, +) +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import RecordingSession + +logger = get_logger(__name__) + +RECORDING_STARTED_MARKER = "Recording started" +STARTUP_TIMEOUT_SECONDS = 30.0 +SIGINT_FLUSH_TIMEOUT_SECONDS = 10.0 +TERMINATE_TIMEOUT_SECONDS = 3.0 +PROBES_TIMEOUT_SECONDS = 3.0 +WATCHDOG_INTERVAL_SECONDS = 0.5 +MAX_CONSECUTIVE_FAILURES = 3 +STDERR_BUFFER_LINES = 200 + + +class IosRecordingSession(RecordingSession): + """Recording session for native iOS ``simctl`` capture.""" + + segments: list[dict[str, Any]] = [] + segment_index: int = 0 + segment_started_at: float | None = None + segment_started_monotonic: float | None = None + anchor_monotonic: float | None = None + stderr_task: asyncio.Task | None = None + stderr_lines: list[str] = [] + conversion_tasks: list[asyncio.Task] = [] + + +async def probe_display_dimensions(device_id: str) -> tuple[int, int] | None: + """Read the raw capture dimensions from a simctl PNG screenshot.""" + try: + process = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + device_id, + "screenshot", + "--type=png", + "-", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + stdout, _stderr = await asyncio.wait_for(process.communicate(), PROBES_TIMEOUT_SECONDS) + with Image.open(BytesIO(stdout)) as image: + return image.size + except (OSError, TimeoutError, ValueError): + return None + + +async def finalize_mov_to_mp4( + source_path: Path, + output_path: Path, + width: int, + height: int, + wall_span_seconds: float, +) -> bool: + """Atomically finalize an owned .mov to a fixed-canvas CFR MP4. + + ``tpad`` clones the final frame across the segment's remaining wall span + so sparse VFR tails pad without stretching any recorded event time. + """ + if not source_path.exists() or source_path.stat().st_size == 0: + return False + width = max(2, int(width)) // 2 * 2 + height = max(2, int(height)) // 2 * 2 + span = max(0.001, float(wall_span_seconds)) + # fps resamples the raw VFR timeline before scale: scale buffers a frame + # and feeding it to fps first drops the final frame on sparse sources. + video_filter = ( + "setpts=PTS-STARTPTS,fps=30," + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:" + f"color=black,setsar=1," + f"tpad=stop_mode=clone:stop_duration={span:.3f}" + ) + temporary_path = output_path.with_name(f"{output_path.stem}.part.mp4") + if temporary_path.exists(): + temporary_path.unlink() + try: + process = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + "-y", + "-i", + str(source_path), + "-map", + "0:v:0", + "-vf", + video_filter, + "-t", + f"{span:.3f}", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _stdout, stderr = await process.communicate() + metadata = await probe_video_segment(temporary_path) + valid = ( + process.returncode == 0 + and temporary_path.exists() + and metadata.get("duration", 0) > 0 + and metadata.get("width", 0) > 0 + and metadata.get("height", 0) > 0 + ) + if valid: + temporary_path.replace(output_path) + return True + logger.error( + f"iOS recording finalization failed (code {process.returncode}): " + f"{stderr.decode(errors='replace')[-2000:]}" + ) + except OSError as exc: + logger.error(f"iOS recording finalization failed: {exc}") + if temporary_path.exists(): + temporary_path.unlink() + return False + + +class IosScreenRecorder: + """Owns one simctl recording lifecycle for one pinned simulator UDID.""" + + def __init__(self, device_id: str): + self._device_id = device_id + self._session: IosRecordingSession | None = None + self._output_dir: Path | None = None + self._lock = asyncio.Lock() + self._max_duration_seconds = 900 + self._consecutive_failures = 0 + + @property + def session(self) -> IosRecordingSession | None: + """The latest session, kept after stop/failure for error reporting.""" + return self._session + + def _output_root(self) -> Path: + if self._output_dir is None: + raise RuntimeError("iOS recording has no output directory") + return self._output_dir + + def _segment_source_path(self, session: IosRecordingSession, index: int) -> Path: + output_dir = self._output_root() + while True: + name = "recording.mov" if index == 0 else f"recording_{index:03d}.mov" + candidate = output_dir / name + if not candidate.exists(): + return candidate + index += 1 + session.segment_index = index + + def _segment_output_path(self, session: IosRecordingSession, index: int) -> Path: + output_dir = self._output_root() + if index == 0: + return output_dir / "recording.mp4" + return output_dir / f"recording_{index:03d}.mp4" + + async def _drain_stderr( + self, + process: asyncio.subprocess.Process, + session: IosRecordingSession, + first_frame: asyncio.Future, + ) -> None: + try: + assert process.stderr is not None + while True: + line = await process.stderr.readline() + if not line: + if not first_frame.done(): + first_frame.set_exception( + RuntimeError( + "simctl recordVideo closed stderr before 'Recording started'" + ) + ) + return + text = line.decode(errors="replace").rstrip() + session.stderr_lines.append(text) + if len(session.stderr_lines) > STDERR_BUFFER_LINES: + del session.stderr_lines[: len(session.stderr_lines) - STDERR_BUFFER_LINES] + if RECORDING_STARTED_MARKER in text and not first_frame.done(): + first_frame.set_result((time.time(), time.monotonic())) + except asyncio.CancelledError: + raise + except Exception as exc: + if not first_frame.done(): + first_frame.set_exception(exc) + + async def _reap(self, process: asyncio.subprocess.Process) -> float: + """SIGINT to flush, escalate to terminate/kill; returns end monotonic.""" + end_monotonic = time.monotonic() + if process.returncode is not None: + return end_monotonic + try: + process.send_signal(signal.SIGINT) + await asyncio.wait_for(process.wait(), SIGINT_FLUSH_TIMEOUT_SECONDS) + except (ProcessLookupError, TimeoutError): + if process.returncode is None: + process.terminate() + try: + await asyncio.wait_for(process.wait(), TERMINATE_TIMEOUT_SECONDS) + except (ProcessLookupError, TimeoutError): + if process.returncode is None: + process.kill() + await process.wait() + return end_monotonic + + async def _spawn_recorder(self, session: IosRecordingSession) -> None: + """Spawn simctl recordVideo and anchor at its first-frame marker.""" + index = session.segment_index + source_path = self._segment_source_path(session, index) + process = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + self._device_id, + "recordVideo", + "--codec=h264", + str(source_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + first_frame: asyncio.Future = asyncio.get_running_loop().create_future() + session.stderr_task = asyncio.create_task(self._drain_stderr(process, session, first_frame)) + try: + marker_wall, marker_monotonic = await asyncio.wait_for( + asyncio.shield(first_frame), STARTUP_TIMEOUT_SECONDS + ) + except BaseException: + if not first_frame.done(): + first_frame.cancel() + session.stderr_task.cancel() + reap = asyncio.ensure_future(self._reap(process)) + try: + await asyncio.shield(reap) + except asyncio.CancelledError: + await reap # finish reaping even when the caller was cancelled + raise + session.process = process + session.local_video_path = source_path + session.segment_started_at = marker_wall + session.segment_started_monotonic = marker_monotonic + if session.anchor_monotonic is None: + session.start_time = marker_wall + session.anchor_monotonic = marker_monotonic + dimensions = await probe_display_dimensions(self._device_id) + if dimensions: + session.capture_width, session.capture_height = dimensions + logger.info( + f"iOS recording segment {index} first frame " + f"{marker_wall - session.start_time:.2f}s after anchor on {self._device_id}" + ) + + def _seal_current_segment( + self, session: IosRecordingSession, end_monotonic: float + ) -> dict[str, Any]: + if session.segment_started_monotonic is None or session.anchor_monotonic is None: + raise RuntimeError("No started iOS recording segment to seal") + record = { + "path": session.local_video_path, + "output_path": self._segment_output_path(session, session.segment_index), + "start": max(0.0, session.segment_started_monotonic - session.anchor_monotonic), + "end": max(0.0, end_monotonic - session.anchor_monotonic), + "width": session.capture_width, + "height": session.capture_height, + "generation": session.generation, + "conversion_done": False, + "conversion_error": None, + } + session.segments.append(record) + session.sealed_until = max(session.sealed_until, record["end"]) + session.generation += 1 + session.segment_index += 1 + session.segment_started_monotonic = None + session.segment_started_at = None + return record + + async def _convert_record(self, session: IosRecordingSession, record: dict[str, Any]) -> None: + source = Path(record["path"]) + output = Path(record["output_path"]) + span = max(0.001, float(record["end"]) - float(record["start"])) + ok = await finalize_mov_to_mp4( + source, output, int(record["width"] or 1080), int(record["height"] or 1920), span + ) + if ok: + record["conversion_done"] = True + else: + record["conversion_error"] = "finalization produced no valid MP4" + session.errors.append( + f"Segment {record['output_path']} failed finalization; raw capture kept" + ) + + async def _roll(self, session: IosRecordingSession, end_monotonic: float, reason: str) -> None: + """Seal the current segment and start the next recorder first.""" + record = None + if session.segment_started_monotonic is not None: + process = session.process + if process is not None and process.returncode is None: + end_monotonic = await self._reap(process) + if session.stderr_task and not session.stderr_task.done(): + session.stderr_task.cancel() + record = self._seal_current_segment(session, end_monotonic) + logger.info(f"Rolling iOS recording segment after {reason}") + restarted = False + if session.is_active: + try: + await self._spawn_recorder(session) + self._consecutive_failures = 0 + restarted = True + except asyncio.CancelledError: + raise + except (OSError, RuntimeError, TimeoutError) as exc: + self._consecutive_failures += 1 + session.errors.append(f"Recorder restart failed: {exc}") + if record is not None: + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if not restarted and self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + session.errors.append("Recording recovery limit reached; session stopped") + session.is_active = False + + async def _watchdog(self, session: IosRecordingSession) -> None: + try: + while True: + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS) + async with self._lock: + if not session.is_active: + return + # Re-verify the trigger under the lock: a concurrent roll + # may have already replaced the dead process. + process = session.process + crashed = process is None or process.returncode is not None + dimensions = await probe_display_dimensions(self._device_id) + rotated = bool( + dimensions + and session.capture_width + and session.capture_height + and dimensions != (session.capture_width, session.capture_height) + ) + age = time.monotonic() - (session.segment_started_monotonic or time.monotonic()) + overdue = age >= self._max_duration_seconds + if not crashed and not rotated and not overdue: + self._consecutive_failures = 0 + continue + reason = ( + "recorder exit" if crashed else "rotation" if rotated else "duration limit" + ) + await self._roll(session, time.monotonic(), reason) + except asyncio.CancelledError: + return + except Exception as exc: + logger.error(f"iOS recording supervisor failed: {exc}") + session.errors.append(f"recording supervisor failed: {exc}") + + async def start( + self, + output_dir: Path | None = None, + max_duration_seconds: int = 900, + ) -> IosRecordingSession: + async with self._lock: + if self._session is not None and self._session.is_active: + raise RuntimeError(f"iOS recording is already active on {self._device_id}") + output = ( + Path(output_dir) + if output_dir is not None + else Path(tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings"))) + ) + output.mkdir(parents=True, exist_ok=True) + self._output_dir = output + self._max_duration_seconds = max_duration_seconds + self._consecutive_failures = 0 + session = IosRecordingSession( + video_id=uuid4(), + device_id=self._device_id, + start_time=time.time(), + local_video_path=output / "recording.mov", + is_active=True, + ) + self._session = session + try: + await self._spawn_recorder(session) + except BaseException: + session.is_active = False + raise + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + return session + + async def seal(self, through_time: float | None = None) -> None: + """Seal the current segment through ``through_time`` (recording-relative).""" + async with self._lock: + session = self._session + if session is None or not session.is_active: + return + if session.anchor_monotonic is None: + return + if through_time is not None and session.sealed_until >= through_time: + return + end_monotonic = ( + session.anchor_monotonic + through_time + if through_time is not None + else time.monotonic() + ) + end_monotonic = min(end_monotonic, time.monotonic()) + await self._roll(session, end_monotonic, "seal") + + async def _finalize(self, session: IosRecordingSession) -> Path | None: + if session.is_active: + session.is_active = False + if session.watchdog_task and not session.watchdog_task.done(): + session.watchdog_task.cancel() + try: + await session.watchdog_task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.debug(f"iOS recording watchdog ended with an error: {exc}") + end_monotonic = time.monotonic() + if session.process is not None and session.process.returncode is None: + end_monotonic = await self._reap(session.process) + if session.stderr_task and not session.stderr_task.done(): + session.stderr_task.cancel() + if session.segment_started_monotonic is not None: + record = self._seal_current_segment(session, end_monotonic) + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + mp4_paths = [ + Path(record["output_path"]) + for record in session.segments + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ] + if not mp4_paths: + session.errors.append("No finalized iOS recording segments") + return None + shift = 0.0 + if session.data_engine_start_time is not None: + shift = session.start_time - session.data_engine_start_time + offsets = { + Path(record["output_path"]): max(0.0, float(record["start"]) + shift) + for record in session.segments + if record.get("conversion_done") and Path(record["output_path"]).exists() + } + output_dir = mp4_paths[0].parent + manifest = await write_recording_manifest(output_dir, mp4_paths, offsets) + if manifest is None: + session.errors.append("Recording manifest has no valid segments") + return None + return mp4_paths[0] + + async def stop(self) -> Path | None: + """Finalize all owned segments and write the version-2 manifest.""" + async with self._lock: + session = self._session + if session is None: + return None + finalize = asyncio.ensure_future(self._finalize(session)) + try: + return await asyncio.shield(finalize) + except asyncio.CancelledError: + await finalize # let cleanup finish before propagating + raise diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index 7576bc93..10f213b8 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -27,8 +27,10 @@ from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, XcodeBridge from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy +from artemis.drivers.ios.recording import IosRecordingSession, IosScreenRecorder from third_party.mobile_use.controllers.types import ElementQuery from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session, remove_active_session logger = get_logger(__name__) @@ -82,6 +84,7 @@ def __init__( self._bridge = XcodeBridge() self._connect_lock = asyncio.Lock() self._operation_lock = asyncio.Lock() + self._recorder: IosScreenRecorder | None = None @property def device_id(self) -> str: @@ -218,20 +221,36 @@ async def connect(self) -> None: raise async def disconnect(self) -> None: - async with self._operation_lock: - key, self._session_key = self._session_key, None - try: - if key: - # A timeout retires the old connection; cleanup may create - # a fresh bridge solely to close the known native session. - if not self._bridge.connected: - await self._bridge.start() - await self._bridge.call( - "DeviceInteractionEndSession", {"interactionSessionKey": key} - ) - finally: - self._scale = None - await self._bridge.close() + try: + if self._recorder is not None: + session = self._recorder.session + if session is not None: + if session.is_active: + try: + await self._recorder.stop() + except Exception as exc: + logger.error( + f"iOS recording finalization failed during disconnect: {exc}" + ) + if get_active_session(self._device_id) is session: + remove_active_session(self._device_id) + finally: + async with self._operation_lock: + key, self._session_key = self._session_key, None + try: + if key: + # A timeout retires the old connection; cleanup may + # create a fresh bridge solely to close the known + # native session. + if not self._bridge.connected: + await self._bridge.start() + await self._bridge.call( + "DeviceInteractionEndSession", + {"interactionSessionKey": key}, + ) + finally: + self._scale = None + await self._bridge.close() async def _synthesize(self, command: str = "", activation: str | None = None) -> dict[str, Any]: self._require_connected() @@ -471,8 +490,32 @@ async def tap_element( async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: raise NotImplementedError("Android shell commands are unavailable on iOS Simulator.") - async def start_video_recording(self, output_dir: Path | None = None) -> None: - raise NotImplementedError("Video recording is not yet integrated for iOS Simulator.") + # --- Recording (native simctl segmented capture) --- + + @property + def recording_session(self) -> IosRecordingSession | None: + """Latest recording session, retained for error reporting after stop.""" + if self._recorder is None: + return None + return self._recorder.session + + async def start_video_recording( + self, output_dir: Path | None = None, max_duration_seconds: int = 900 + ) -> None: + self._require_connected() + if not self._device_id: + raise RuntimeError("iOS recording requires a pinned simulator UDID") + if self._recorder is None: + self._recorder = IosScreenRecorder(self._device_id) + await self._recorder.start(output_dir, max_duration_seconds) + + async def seal_recording_segment(self, through_time: float | None = None) -> None: + """Seal the current segment so its final MP4 can be read safely.""" + if self._recorder is not None: + await self._recorder.seal(through_time) async def stop_video_recording(self) -> str | None: - return None + if self._recorder is None: + return None + path = await self._recorder.stop() + return str(path) if path is not None else None diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index c11a2fef..11204934 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -148,7 +148,6 @@ async def execute_task( target_serial = device_serial if platform == DevicePlatform.IOS: target_serial = target_serial or "booted" - config.with_video_recording_tools(enabled=False) else: target_serial = ( target_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") @@ -359,14 +358,9 @@ def run_command( ) if locked_app_package: raise typer.BadParameter("--locked-app is unavailable for iOS simulator tasks.") - if with_video_recording_tools: - raise typer.BadParameter( - "iOS video analysis is unavailable; omit --with-video-recording-tools." - ) is_standalone = True console.print("[dim]Running the iOS simulator task in standalone mode.[/dim]") - else: - ensure_video_recording_available(with_video_recording_tools) + ensure_video_recording_available(with_video_recording_tools) # All platforms route through unified Artemis Daemon unless specifically configured as standalone if not is_worker and not is_standalone: diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index a99f6d81..47ea8354 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -172,10 +172,9 @@ async def _init_internal( device_width=width, device_height=height, ) - # Android recording and read-only ADB probes have no iOS equivalent yet. - self._config = self._config.model_copy( - update={"video_recording_tools_enabled": False, "disable_device_probes": True} - ) + # Android read-only ADB probes have no iOS equivalent yet; native + # simctl recording is supported and honors the configured flag. + self._config = self._config.model_copy(update={"disable_device_probes": True}) publish_startup_progress( "device_ready", "iOS simulator selected", session_id=self._session_id ) diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 772f8ef2..a717e171 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -38,6 +38,7 @@ class AgentConfigBuilder(AgentConfigBuilderBase): def __init__(self): """Initialize an empty AgentConfigBuilder with Artemis defaults.""" super().__init__(servers=get_default_servers()) + self._video_recording_tools_explicit: bool = False self._video_recording_tools_enabled: bool = detect_video_tools_enabled() self._force_web_accessibility: bool = False self._disable_checker: bool = False @@ -52,9 +53,8 @@ def __init__(self): self._video_analyzer = agent_cfg.video_analyzer self._enable_video_ledger = agent_cfg.video_analyzer.enable_ledger if agent_cfg.video_analyzer.enabled is not None: + self._video_recording_tools_explicit = True self._video_recording_tools_enabled = agent_cfg.video_analyzer.enabled - else: - self._video_recording_tools_enabled = detect_video_tools_enabled() self._disable_planner_validation = not agent_cfg.planner_validation.enabled self._enable_committee = agent_cfg.committee.enabled self._committee_debate_rounds = agent_cfg.committee.debate_rounds @@ -109,6 +109,7 @@ def with_video_recording_tools(self, enabled: bool = True) -> "AgentConfigBuilde Args: enabled: Whether to enable video recording tools """ + self._video_recording_tools_explicit = True self._video_recording_tools_enabled = enabled return self @@ -442,7 +443,15 @@ def _extra_config_fields(self) -> dict[str, Any]: or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL") ), - "video_recording_tools_enabled": self._video_recording_tools_enabled, + "video_recording_tools_enabled": ( + self._video_recording_tools_enabled + if self._video_recording_tools_explicit + else detect_video_tools_enabled( + self._device_platform.value + if self._device_platform is not None + else DevicePlatform.ANDROID.value + ) + ), "force_web_accessibility": self._force_web_accessibility, "disable_checker": self._disable_checker, "disable_midway_checks": self._disable_midway_checks, diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 72e41db5..ccff0a98 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -13,7 +13,7 @@ # limitations under the License. from pathlib import Path -from typing import Literal +from typing import Any, Literal from artemis.config import ( ExplorerConfig, @@ -35,6 +35,13 @@ __all__ = ["AgentConfig", "AgentProfile", "ApiBaseUrl", "DevicePlatform", "ServerConfig"] +def _default_video_recording_tools(data: dict[str, Any]) -> bool: + """Detect the recording toolchain for the configured device platform.""" + platform = data.get("device_platform") + platform_name = getattr(platform, "value", platform) or DevicePlatform.ANDROID.value + return detect_video_tools_enabled(str(platform_name)) + + class AgentConfig(AgentConfigBase): """ARTEMIS agent configuration. @@ -42,7 +49,9 @@ class AgentConfig(AgentConfigBase): settings to :class:`AgentConfigBase`. """ - video_recording_tools_enabled: bool = Field(default_factory=detect_video_tools_enabled) + video_recording_tools_enabled: bool = Field( + default_factory=lambda data: _default_video_recording_tools(data) + ) force_web_accessibility: bool = False disable_checker: bool = False disable_midway_checks: bool = True diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 76afb5df..01c5a4b9 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -21,11 +21,13 @@ import asyncio import importlib.util import json +import math from pathlib import Path import re import shutil -import time import subprocess +import sys +import time from typing import Any import cv2 @@ -289,20 +291,51 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool: return False +def _probe_video_segment_cv2(video_path: Path) -> dict[str, float | int]: + """Read metadata from a finalized CFR video when ffprobe is unavailable.""" + capture = cv2.VideoCapture(str(video_path)) + try: + width = capture.get(cv2.CAP_PROP_FRAME_WIDTH) + height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + fps = capture.get(cv2.CAP_PROP_FPS) + frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) + finally: + capture.release() + if not ( + math.isfinite(width) + and math.isfinite(height) + and math.isfinite(fps) + and math.isfinite(frame_count) + and width > 0 + and height > 0 + and fps > 0 + and frame_count > 0 + ): + return {} + return { + "duration": frame_count / fps, + "width": int(width), + "height": int(height), + } + + async def probe_video_segment(video_path: Path) -> dict[str, float | int]: """Read duration and coded dimensions for a finalized segment.""" - process = await asyncio.create_subprocess_exec( - get_ffprobe_path(), - "-v", - "error", - "-show_entries", - "stream=width,height,duration,codec_type:format=duration", - "-of", - "json", - str(video_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) + try: + process = await asyncio.create_subprocess_exec( + get_ffprobe_path(), + "-v", + "error", + "-show_entries", + "stream=width,height,duration,codec_type:format=duration", + "-of", + "json", + str(video_path), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError: + return await asyncio.to_thread(_probe_video_segment_cv2, video_path) stdout, _stderr = await process.communicate() if process.returncode != 0: return {} @@ -564,8 +597,16 @@ def is_scrcpy_installed() -> bool: return shutil.which("scrcpy") is not None -def detect_video_tools_enabled() -> bool: - """Check if both scrcpy and ffmpeg are available to enable automated video features.""" +def detect_video_tools_enabled(platform: str = "android") -> bool: + """Check whether the platform's recording toolchain is available locally. + + iOS uses native ``xcrun simctl io recordVideo`` capture plus the bundled + FFmpeg for post-processing; Android requires scrcpy plus FFmpeg. + """ + if platform == "ios": + return ( + sys.platform == "darwin" and shutil.which("xcrun") is not None and is_ffmpeg_installed() + ) return is_ffmpeg_installed() and is_scrcpy_installed() diff --git a/docs/ios.md b/docs/ios.md index 19dc9d9b..9ced6a7a 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -196,8 +196,17 @@ Android shell commands, Logcat, Android resource identifiers, Android package discovery, and the Android Accessibility Helper are unavailable on iOS. Platform-specific operations fail explicitly when unsupported. -Screen recording, video analysis, and video-based replay are not supported by -this initial iOS driver. Use screenshots and the task trace to review a run. +Screen recording uses the native `xcrun simctl io recordVideo` capture with +no third-party device automation. Each segment is a variable-frame-rate H.264 +`.mov` anchored to its first captured frame; after recording stops (or when a +rotation or the duration limit rolls a segment), the bundled FFmpeg post- +processing finalizes a browser-safe 30 fps MP4 and a `recording.json` +manifest mapping every segment to its session-time offset. The video analyzer +can clip the sealed portion of an in-progress recording; request ranges that +reach past the sealed boundary are clipped with a warning. Limitations: iOS +capture is silent (no audio), pre-first-frame startup time is not captured, +and recorder restarts leave a brief gap in the timeline rather than stretching +recorded frames. ## Troubleshooting diff --git a/tests/unit/drivers/test_ios_recording.py b/tests/unit/drivers/test_ios_recording.py new file mode 100644 index 00000000..7d00cce4 --- /dev/null +++ b/tests/unit/drivers/test_ios_recording.py @@ -0,0 +1,611 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic tests for the native iOS simctl recorder.""" + +import asyncio +from collections import deque +import json +from pathlib import Path +import signal +import subprocess +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import cv2 +import pytest + +from artemis.drivers.ios import recording as rec +from artemis.drivers.ios.recording import ( + IosRecordingSession, + IosScreenRecorder, + finalize_mov_to_mp4, +) +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from third_party.mobile_use.utils.video import ( + get_active_session, + remove_active_session, + set_active_session, +) + +UDID = "E1D9F1D1-04E5-4E95-801F-830B854FD3E2" + + +class FakeStderr: + """Queued stderr lines; hangs like a live pipe until ``feed_eof``.""" + + def __init__(self, lines=None, delay: float = 0.0, eof: bool = False): + self._lines = deque(lines if lines is not None else [b"Recording started\n"]) + self._delay = delay + self._eof = asyncio.Event() + if eof: + self._eof.set() + + def feed(self, line: bytes) -> None: + self._lines.append(line) + + def feed_eof(self) -> None: + self._eof.set() + + async def readline(self): + while True: + if self._lines: + if self._delay: + await asyncio.sleep(self._delay) + return self._lines.popleft() + if self._eof.is_set(): + return b"" + await asyncio.sleep(0.005) + + +class FakeRecorderProcess: + """Minimal asyncio.subprocess.Process stand-in for simctl recordVideo.""" + + def __init__(self, stderr=None, *, ignore_signals=()): + self.stderr = stderr if stderr is not None else FakeStderr() + self.returncode = None + self.signals: list[int] = [] + self.terminated = False + self.killed = False + self._exit = asyncio.Event() + self._ignore = set(ignore_signals) + + def send_signal(self, sig): + self.signals.append(sig) + if sig not in self._ignore and self.returncode is None: + self.returncode = 0 + self._exit.set() + + def terminate(self): + self.terminated = True + if self.returncode is None: + self.returncode = -int(signal.SIGTERM) + self._exit.set() + + def kill(self): + self.killed = True + if self.returncode is None: + self.returncode = -int(signal.SIGKILL) + self._exit.set() + + async def wait(self): + await self._exit.wait() + return self.returncode + + +def spawn_factory(procs): + """Return (spawn, created) where spawn serves FakeRecorderProcess items.""" + created: list[tuple] = [] + + async def spawn(*argv, **kwargs): + created.append(argv) + index = min(len(created) - 1, len(procs) - 1) + return procs[index] + + return spawn, created + + +@pytest.fixture +def recorder_env(monkeypatch): + """Patch the recorder's child-process and probe seams.""" + env = SimpleNamespace(finalized=[], spawns=None) + monkeypatch.setattr(rec, "probe_display_dimensions", AsyncMock(return_value=(1206, 2622))) + + async def fake_finalize(source_path, output_path, width, height, span): + env.finalized.append((Path(source_path), Path(output_path), width, height, span)) + output_path.write_bytes(b"fake mp4") + return True + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", fake_finalize) + monkeypatch.setattr( + "artemis.utils.video.probe_video_segment", + AsyncMock(return_value={"duration": 1.0, "width": 100, "height": 200}), + ) + monkeypatch.setattr(rec, "WATCHDOG_INTERVAL_SECONDS", 0.01) + return env + + +def patch_spawn(monkeypatch, procs): + spawn, created = spawn_factory(procs) + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + return created + + +@pytest.mark.asyncio +async def test_recordvideo_argv_pins_udid_without_force_or_android_tools( + recorder_env, monkeypatch, tmp_path +): + procs = [FakeRecorderProcess()] + created = patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + + argv = created[0] + assert argv[:4] == ("xcrun", "simctl", "io", UDID) + assert "recordVideo" in argv and "--codec=h264" in argv + assert str(tmp_path / "recording.mov") in argv + assert "--force" not in argv + assert not any("adb" in part or "scrcpy" in part for part in argv) + assert session.is_active and session.capture_width == 1206 + await recorder.stop() + + +@pytest.mark.asyncio +async def test_marker_anchors_timeline_not_process_spawn(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(FakeStderr(delay=0.05))] + created = patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + + before = time.time() + session = await recorder.start(output_dir=tmp_path) + + assert session.start_time >= before + 0.03 # anchored at marker, not spawn + assert session.anchor_monotonic is not None + assert session.segment_started_monotonic == session.anchor_monotonic + await recorder.stop() + + +@pytest.mark.asyncio +async def test_missing_marker_times_out_and_reaps(recorder_env, monkeypatch, tmp_path): + monkeypatch.setattr(rec, "STARTUP_TIMEOUT_SECONDS", 0.05) + proc = FakeRecorderProcess(FakeStderr(lines=[])) # hangs, never marks + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + + with pytest.raises(TimeoutError): + await recorder.start(output_dir=tmp_path) + + assert signal.SIGINT in proc.signals + assert proc.returncode == 0 + assert recorder.session is not None and not recorder.session.is_active + + +@pytest.mark.asyncio +async def test_stderr_eof_before_marker_fails(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess(FakeStderr(lines=[], eof=True)) + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + + with pytest.raises(RuntimeError, match="Recording started"): + await recorder.start(output_dir=tmp_path) + assert proc.returncode == 0 # reaped via SIGINT + + +@pytest.mark.asyncio +async def test_startup_cancellation_reaps_child(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess(FakeStderr(lines=[])) # marker never arrives + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + + task = asyncio.create_task(recorder.start(output_dir=tmp_path)) + await asyncio.sleep(0.02) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert proc.returncode is not None + assert signal.SIGINT in proc.signals + + +@pytest.mark.asyncio +async def test_sigint_terminate_kill_escalation(recorder_env, monkeypatch, tmp_path): + monkeypatch.setattr(rec, "SIGINT_FLUSH_TIMEOUT_SECONDS", 0.05) + monkeypatch.setattr(rec, "TERMINATE_TIMEOUT_SECONDS", 0.05) + term_proc = FakeRecorderProcess(ignore_signals={signal.SIGINT}) + kill_proc = FakeRecorderProcess(ignore_signals={signal.SIGINT}) + kill_proc.terminate = lambda: setattr(kill_proc, "terminated", True) + + patch_spawn(monkeypatch, [term_proc]) + recorder = IosScreenRecorder(UDID) + await recorder.start(output_dir=tmp_path) + await recorder.stop() + assert term_proc.signals == [signal.SIGINT] + assert term_proc.terminated + assert not term_proc.killed + + patch_spawn(monkeypatch, [kill_proc]) + recorder2 = IosScreenRecorder(UDID) + await recorder2.start(output_dir=tmp_path / "b") + await recorder2.stop() + assert kill_proc.terminated and kill_proc.killed + + +@pytest.mark.asyncio +async def test_stderr_buffer_is_bounded(recorder_env, monkeypatch, tmp_path): + stderr = FakeStderr() + proc = FakeRecorderProcess(stderr) + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + for index in range(rec.STDERR_BUFFER_LINES * 2): + stderr.feed(f"line {index}\n".encode()) + await asyncio.sleep(0.05) + + assert len(session.stderr_lines) <= rec.STDERR_BUFFER_LINES + assert session.stderr_lines[-1].startswith("line") + await recorder.stop() + + +@pytest.mark.asyncio +async def test_double_start_rejected_and_stop_without_recording( + recorder_env, monkeypatch, tmp_path +): + patch_spawn(monkeypatch, [FakeRecorderProcess()]) + recorder = IosScreenRecorder(UDID) + + assert await recorder.stop() is None # no session yet + + await recorder.start(output_dir=tmp_path) + with pytest.raises(RuntimeError, match="already active"): + await recorder.start(output_dir=tmp_path) + await recorder.stop() + assert await recorder.stop() is not None # stopped session metadata retained + + +@pytest.mark.asyncio +async def test_output_collision_never_overwrites(recorder_env, monkeypatch, tmp_path): + original = tmp_path / "recording.mov" + original.write_bytes(b"pre-existing capture") + created = patch_spawn(monkeypatch, [FakeRecorderProcess()]) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + + assert "recording_001.mov" in str(created[0][-1]) + assert session.local_video_path.name == "recording_001.mov" + assert original.read_bytes() == b"pre-existing capture" + await recorder.stop() + + +@pytest.mark.asyncio +async def test_rotation_rolls_segment_with_raw_dimensions(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(), FakeRecorderProcess()] + patch_spawn(monkeypatch, procs) + dims_state = {"value": (1206, 2622)} + monkeypatch.setattr( + rec, + "probe_display_dimensions", + AsyncMock(side_effect=lambda *_: dims_state["value"]), + ) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + dims_state["value"] = (2622, 1206) # rotate the display + for _ in range(200): + if session.segments and session.capture_width == 2622: + break + await asyncio.sleep(0.01) + + assert session.segments, "watchdog did not roll on rotation" + first = session.segments[0] + assert (first["width"], first["height"]) == (1206, 2622) + assert session.capture_width == 2622 and session.capture_height == 1206 + assert session.generation >= 1 + assert signal.SIGINT in procs[0].signals + await recorder.stop() + + +@pytest.mark.asyncio +async def test_duration_limit_rolls_segment(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(), FakeRecorderProcess()] + patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path, max_duration_seconds=1) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and not session.segments: + await asyncio.sleep(0.02) + + assert session.segments, "watchdog did not roll on duration limit" + await recorder.stop() + + +@pytest.mark.asyncio +async def test_crash_recovery_is_bounded(recorder_env, monkeypatch, tmp_path): + healthy = FakeRecorderProcess() + dead = [FakeRecorderProcess(FakeStderr(lines=[], eof=True)) for _ in range(5)] + created = patch_spawn(monkeypatch, [healthy] + dead) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + healthy.returncode = 1 # simulate recorder crash + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and session.is_active: + await asyncio.sleep(0.02) + + assert not session.is_active + assert len(created) <= rec.MAX_CONSECUTIVE_FAILURES + 1 + assert any("recovery limit" in error for error in session.errors) + + +@pytest.mark.asyncio +async def test_seal_finalizes_segment_and_continues(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(), FakeRecorderProcess()] + patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + await asyncio.sleep(0.05) + await recorder.seal(through_time=0.02) + + # The sealed boundary covers through_time; content runs to the SIGINT point. + assert 0.02 <= session.segments[0]["end"] <= 0.5 + assert session.sealed_until >= 0.02 + assert session.generation == 1 + assert session.is_active and session.process is procs[1] + await recorder.seal(through_time=0.01) # already sealed through this value + assert len(session.segments) == 1 + await recorder.stop() + + +@pytest.mark.asyncio +async def test_stop_finalizes_manifest_and_returns_mp4(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + result = await recorder.stop() + + assert result == tmp_path / "recording.mp4" + assert signal.SIGINT in proc.signals + manifest = json.loads((tmp_path / "recording.json").read_text()) + assert manifest["version"] == 2 + assert manifest["segments"] and manifest["segments"][0]["file"] == "recording.mp4" + assert not session.is_active + + +@pytest.mark.asyncio +async def test_stop_cancelled_caller_still_reaps_and_finalizes(recorder_env, monkeypatch, tmp_path): + gate = asyncio.Event() + + async def slow_finalize(source_path, output_path, width, height, span): + await gate.wait() + output_path.write_bytes(b"fake mp4") + return True + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", slow_finalize) + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + await recorder.start(output_dir=tmp_path) + + task = asyncio.create_task(recorder.stop()) + await asyncio.sleep(0.05) + task.cancel() + gate.set() + with pytest.raises(asyncio.CancelledError): + await task + + assert proc.returncode == 0 + assert (tmp_path / "recording.mp4").exists() + + +@pytest.mark.asyncio +async def test_conversion_failure_reports_partial_and_keeps_raw( + recorder_env, monkeypatch, tmp_path +): + async def failing(*args, **kwargs): + return False + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", failing) + patch_spawn(monkeypatch, [FakeRecorderProcess()]) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + assert await recorder.stop() is None + assert any("finalization" in error for error in session.errors) + + +@pytest.mark.asyncio +async def test_disconnect_finalizes_recorder_and_clears_registry( + recorder_env, monkeypatch, tmp_path +): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + driver = XcodeSimulatorDriver(device_id=UDID) + driver._session_key = "session-key" + bridge = SimpleNamespace( + connected=True, + call=AsyncMock(), + close=AsyncMock(), + start=AsyncMock(), + ) + driver._bridge = bridge + + await driver.start_video_recording(output_dir=tmp_path) + session = driver.recording_session + set_active_session(UDID, session) + + await driver.disconnect() + + assert proc.returncode == 0 + assert get_active_session(UDID) is None + bridge.call.assert_awaited_once() + assert bridge.call.await_args.args[0] == "DeviceInteractionEndSession" + bridge.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_disconnect_closes_bridge_even_when_finalize_fails( + recorder_env, monkeypatch, tmp_path +): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + monkeypatch.setattr(rec, "finalize_mov_to_mp4", AsyncMock(return_value=False)) + driver = XcodeSimulatorDriver(device_id=UDID) + driver._session_key = "session-key" + bridge = SimpleNamespace(connected=True, call=AsyncMock(), close=AsyncMock(), start=AsyncMock()) + driver._bridge = bridge + + await driver.start_video_recording(output_dir=tmp_path) + set_active_session(UDID, driver.recording_session) + await driver.disconnect() + + assert proc.returncode == 0 + assert get_active_session(UDID) is None + bridge.close.assert_awaited_once() + assert driver.recording_session.errors + + +@pytest.mark.asyncio +async def test_disconnect_leaves_other_sessions_registry(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + other = IosRecordingSession(video_id=uuid4(), device_id=UDID, start_time=0.0) + set_active_session(UDID, other) + driver = XcodeSimulatorDriver(device_id=UDID) + driver._session_key = "session-key" + driver._bridge = SimpleNamespace( + connected=True, call=AsyncMock(), close=AsyncMock(), start=AsyncMock() + ) + + await driver.start_video_recording(output_dir=tmp_path) + await driver.disconnect() + + assert get_active_session(UDID) is other + remove_active_session(UDID) + + +# --- Real FFmpeg regression coverage (bundled binary, no Xcode/simulator) --- + + +def _make_vfr_mov(path: Path, width: int = 320, height: int = 240) -> Path: + """Two VFR frames: red at t=0, blue at t=0.5 (from 30fps concat).""" + ffmpeg = rec.get_ffmpeg_path() + subprocess.run( + [ + ffmpeg, + "-y", + "-f", + "lavfi", + "-i", + f"color=c=red:s={width}x{height}:r=30:d=0.5", + "-f", + "lavfi", + "-i", + f"color=c=blue:s={width}x{height}:r=30:d=0.5", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1,select='eq(n,0)+eq(n,15)'[v]", + "-map", + "[v]", + "-fps_mode", + "vfr", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + str(path), + ], + check=True, + capture_output=True, + ) + return path + + +def _frame_at(video: Path, seconds: float): + capture = cv2.VideoCapture(str(video)) + try: + capture.set(cv2.CAP_PROP_POS_MSEC, seconds * 1000) + ok, frame = capture.read() + assert ok and frame is not None + return frame + finally: + capture.release() + + +def _mean_bgr(frame): + return frame.mean(axis=(0, 1)) + + +@pytest.mark.asyncio +async def test_finalize_preserves_vfr_event_times_without_stretching(tmp_path): + source = _make_vfr_mov(tmp_path / "segment.mov") + output = tmp_path / "recording.mp4" + + assert await finalize_mov_to_mp4(source, output, 320, 240, 2.0) + + capture = cv2.VideoCapture(str(output)) + fps = capture.get(cv2.CAP_PROP_FPS) + frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) + capture.release() + assert fps == pytest.approx(30, abs=0.5) + assert frame_count / fps == pytest.approx(2.0, abs=0.05) + + red = _mean_bgr(_frame_at(output, 0.1)) + blue_early = _mean_bgr(_frame_at(output, 0.6)) + blue_tail = _mean_bgr(_frame_at(output, 1.6)) + assert red[2] > 150 and red[0] < 60 # red frame, not stretched into tail + assert blue_early[0] > 150 and blue_early[2] < 60 + assert blue_tail[0] > 150 and blue_tail[2] < 60 # cloned tail stays blue + + +@pytest.mark.asyncio +async def test_finalize_normalizes_mixed_resolution_sources(tmp_path): + portrait = _make_vfr_mov(tmp_path / "portrait.mov", width=240, height=320) + landscape = _make_vfr_mov(tmp_path / "landscape.mov", width=320, height=240) + + out_p = tmp_path / "portrait.mp4" + out_l = tmp_path / "landscape.mp4" + assert await finalize_mov_to_mp4(portrait, out_p, 240, 320, 1.0) + assert await finalize_mov_to_mp4(landscape, out_l, 240, 320, 1.0) + + for produced in (out_p, out_l): + capture = cv2.VideoCapture(str(produced)) + width = capture.get(cv2.CAP_PROP_FRAME_WIDTH) + height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + ok, _frame = capture.read() + capture.release() + assert ok and (width, height) == (240, 320) + + +@pytest.mark.asyncio +async def test_probe_falls_back_to_cv2_only_when_ffprobe_missing(monkeypatch, tmp_path): + from artemis.utils import video as video_utils + + mp4 = tmp_path / "segment.mp4" + assert await finalize_mov_to_mp4(_make_vfr_mov(tmp_path / "src.mov"), mp4, 320, 240, 1.0) + monkeypatch.setattr(video_utils, "get_ffprobe_path", lambda: str(tmp_path / "missing-ffprobe")) + + metadata = await video_utils.probe_video_segment(mp4) + assert metadata["width"] == 320 and metadata["height"] == 240 + assert metadata["duration"] == pytest.approx(1.0, abs=0.05) + + corrupted = tmp_path / "corrupt.mp4" + corrupted.write_bytes(b"not a movie") + assert await video_utils.probe_video_segment(corrupted) == {} diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py index 82424709..5398db3c 100644 --- a/tests/unit/test_ios_integration.py +++ b/tests/unit/test_ios_integration.py @@ -158,6 +158,7 @@ async def test_sdk_ios_initialization_bypasses_android_and_cleans(native_driver, AgentBase, "_init_internal", AsyncMock(side_effect=AssertionError("ADB init")) ) agent = Agent(config=ios_config()) + configured_video = agent._config.video_recording_tools_enabled agent._prewarm_llm_connections = AsyncMock() assert await agent.init() is True await asyncio.sleep(0) @@ -166,7 +167,7 @@ async def test_sdk_ios_initialization_bypasses_android_and_cleans(native_driver, assert agent._device_context.device_id == native_driver.device_id assert (agent._device_context.device_width, agent._device_context.device_height) == (1170, 2532) assert agent._adb_client is None and agent._ui_adb_client is None - assert agent._config.video_recording_tools_enabled is False + assert agent._config.video_recording_tools_enabled == configured_video assert agent._config.disable_device_probes is True await agent.clean() native_driver.disconnect.assert_awaited_once() @@ -317,7 +318,7 @@ async def test_execute_task_ios_ignores_android_selection(monkeypatch): ) await run_module.execute_task("Open Settings", platform=DevicePlatform.IOS) builder.for_device.assert_called_once_with(DevicePlatform.IOS, "booted") - builder.with_video_recording_tools.assert_called_once_with(enabled=False) + builder.with_video_recording_tools.assert_not_called() automation.assert_awaited_once() @@ -729,3 +730,102 @@ async def test_context_disconnect_driver_swallows_expected_cleanup_errors(native context = ios_context(native_driver) with pytest.raises(KeyError): await context.disconnect_driver() + + +@pytest.mark.parametrize( + "flag", ["--with-video-recording-tools", "--without-video-recording-tools"] +) +def test_cli_ios_accepts_video_flag_in_standalone(monkeypatch, flag): + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + result = CliRunner().invoke(app, ["run", "--platform", "ios", flag, "Open Settings"]) + assert result.exit_code == 0, result.output + execute.assert_awaited_once() + + +def _unconfigured_agent_cfg(monkeypatch, builder_module): + """Neutralize any configured video_analyzer.enabled so detection runs.""" + real_load = builder_module.load_agent_config + + def load(): + cfg = real_load() + cfg.video_analyzer.enabled = None + return cfg + + monkeypatch.setattr(builder_module, "load_agent_config", load) + + +def test_builder_auto_detects_video_tools_per_platform(monkeypatch): + from artemis.sdk.builders import agent_config_builder as builder_module + + _unconfigured_agent_cfg(monkeypatch, builder_module) + observed: list[str] = [] + monkeypatch.setattr( + builder_module, + "detect_video_tools_enabled", + lambda platform="android": observed.append(platform) or platform == "ios", + ) + + ios_config = AgentConfigBuilder().for_ios_simulator().build(validate_profiles=False) + assert ios_config.video_recording_tools_enabled is True + assert observed[-1] == "ios" + + android_config = AgentConfigBuilder().build(validate_profiles=False) + assert android_config.video_recording_tools_enabled is False + assert observed[-1] == "android" + + +def test_builder_explicit_video_flag_wins_regardless_of_order(monkeypatch): + from artemis.sdk.builders import agent_config_builder as builder_module + + monkeypatch.setattr( + builder_module, + "detect_video_tools_enabled", + lambda platform="android": True, + ) + + before = ( + AgentConfigBuilder() + .with_video_recording_tools(enabled=False) + .for_ios_simulator() + .build(validate_profiles=False) + ) + after = ( + AgentConfigBuilder() + .for_ios_simulator() + .with_video_recording_tools(enabled=False) + .build(validate_profiles=False) + ) + enabled = ( + AgentConfigBuilder() + .for_ios_simulator() + .with_video_recording_tools(enabled=True) + .build(validate_profiles=False) + ) + assert before.video_recording_tools_enabled is False + assert after.video_recording_tools_enabled is False + assert enabled.video_recording_tools_enabled is True + + +def test_agent_config_default_factory_detects_platform_from_data(monkeypatch): + from artemis.sdk.types import agent as agent_types + + monkeypatch.setattr( + agent_types, + "detect_video_tools_enabled", + lambda platform="android": platform == "ios", + ) + from artemis.sdk.types.agent import AgentConfig + + data = AgentConfigBuilder().build(validate_profiles=False).model_dump() + data.pop("video_recording_tools_enabled") + data["device_platform"] = DevicePlatform.IOS + ios = AgentConfig.model_validate(data) + data["device_platform"] = DevicePlatform.ANDROID + android = AgentConfig.model_validate(data) + explicit = AgentConfig.model_validate( + {**data, "device_platform": DevicePlatform.IOS, "video_recording_tools_enabled": False} + ) + assert ios.video_recording_tools_enabled is True + assert android.video_recording_tools_enabled is False + assert explicit.video_recording_tools_enabled is False diff --git a/tests/unit/test_unified_controller_video.py b/tests/unit/test_unified_controller_video.py index e87330dd..ad1eeb38 100644 --- a/tests/unit/test_unified_controller_video.py +++ b/tests/unit/test_unified_controller_video.py @@ -15,9 +15,11 @@ """Unit tests for UnifiedMobileController video recording and playback features.""" import asyncio +import collections import subprocess import time from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 @@ -794,3 +796,256 @@ def test_segment_session_offsets_without_data_engine_anchor(tmp_path): assert offsets[tmp_path / "recording_001.mp4"] == 7.5 # An emergency fallback remux has no record; it starts at the recording anchor. assert offsets[fallback] == 0.0 + + +# --- iOS native recording through the shared controller contract --- + + +class _FakeIosStderr: + def __init__(self, lines=None, eof: bool = False): + self._lines = collections.deque(lines if lines is not None else [b"Recording started\n"]) + self._eof = asyncio.Event() + if eof: + self._eof.set() + + async def readline(self): + if self._lines: + return self._lines.popleft() + await self._eof.wait() + return b"" + + +class _FakeIosRecorderProcess: + def __init__(self, stderr=None): + self.stderr = stderr or _FakeIosStderr() + self.returncode = None + self.signals = [] + self._exit = asyncio.Event() + + def send_signal(self, sig): + self.signals.append(sig) + self.returncode = 0 + self._exit.set() + + def terminate(self): + self.returncode = -15 + self._exit.set() + + def kill(self): + self.returncode = -9 + self._exit.set() + + async def wait(self): + await self._exit.wait() + return self.returncode + + +@pytest.fixture +def ios_ctx(): + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + ctx = MagicMock(spec=ArtemisContext) + ctx.device = MagicMock() + ctx.device.device_id = "sim-udid-1" + ctx.device.mobile_platform = "ios" + ctx.data_engine = MagicMock() + ctx.data_engine.current_session_id = uuid4() + ctx.data_engine.session_start_time = time.time() - 1.0 + ctx.data_engine.storage = MagicMock() + driver = XcodeSimulatorDriver(device_id="sim-udid-1") + driver._session_key = "test-session" + driver._bridge = SimpleNamespace( + connected=True, call=AsyncMock(), close=AsyncMock(), start=AsyncMock() + ) + ctx._active_driver = driver + return ctx + + +@pytest.fixture +def ios_recording_seams(monkeypatch): + """Fake the simctl child process, display probe, and MP4 finalization.""" + from artemis.drivers.ios import recording as rec + from artemis.utils import video as video_utils + + procs: list[_FakeIosRecorderProcess] = [] + + async def spawn(*argv, **kwargs): + proc = _FakeIosRecorderProcess() + procs.append(proc) + return proc + + async def fake_finalize(source_path, output_path, width, height, span): + Path(output_path).write_bytes(b"fake mp4") + return True + + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + monkeypatch.setattr(rec, "probe_display_dimensions", AsyncMock(return_value=(1206, 2622))) + monkeypatch.setattr(rec, "finalize_mov_to_mp4", fake_finalize) + monkeypatch.setattr( + video_utils, + "probe_video_segment", + AsyncMock(return_value={"duration": 1.0, "width": 100, "height": 200}), + ) + monkeypatch.setattr(rec, "WATCHDOG_INTERVAL_SECONDS", 0.01) + return procs + + +@pytest.mark.asyncio +async def test_ios_start_registers_driver_session_and_publishes_de_anchor( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + + res = await controller.start_video_recording(output_dir=tmp_path) + + try: + assert res.success is True + session = get_active_session("sim-udid-1") + assert session is not None + # Controller and driver share one recording session/process. + assert session is ios_ctx._active_driver.recording_session + assert session.data_engine_start_time == ios_ctx.data_engine.session_start_time + ios_ctx.data_engine.record_video_start.assert_called_once() + assert res.video_id == session.video_id + + duplicate = await controller.start_video_recording(output_dir=tmp_path) + assert duplicate.success is False + finally: + remove_active_session("sim-udid-1") + await ios_ctx._active_driver.stop_video_recording() + + +@pytest.mark.asyncio +async def test_ios_start_failure_publishes_start_and_failure( + ios_ctx, ios_recording_seams, tmp_path, monkeypatch +): + from artemis.drivers.ios import recording as rec + + async def dead_spawn(*argv, **kwargs): + return _FakeIosRecorderProcess(_FakeIosStderr(lines=[], eof=True)) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", dead_spawn) + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + + res = await controller.start_video_recording(output_dir=tmp_path) + + assert res.success is False + assert "failed to start" in res.message + ios_ctx.data_engine.record_video_start.assert_called_once() + ios_ctx.data_engine.record_video_failure.assert_called_once() + assert get_active_session("sim-udid-1") is None + # The driver's failed session is still inspectable. + assert ios_ctx._active_driver.recording_session is not None + + +@pytest.mark.asyncio +async def test_ios_stop_records_stop_metadata_and_clears_registry( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + + res = await controller.stop_video_recording() + + assert res.success is True + assert res.video_path == tmp_path / "recording.mp4" + assert res.source_revision and res.source_revision.endswith(":ready") + ios_ctx.data_engine.record_video_stop.assert_called_once() + assert get_active_session("sim-udid-1") is None + + +@pytest.mark.asyncio +async def test_ios_stop_failure_reports_unsuccessful_result( + ios_ctx, ios_recording_seams, tmp_path, monkeypatch +): + from artemis.drivers.ios import recording as rec + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", AsyncMock(return_value=False)) + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + + res = await controller.stop_video_recording() + + assert res.success is False + ios_ctx.data_engine.record_video_failure.assert_called() + assert get_active_session("sim-udid-1") is None + # The failed session is retained with actionable errors for reporting. + retained = ios_ctx._active_driver.recording_session + assert retained is not None and retained.errors + + +@pytest.mark.asyncio +async def test_ios_extract_seals_then_renders_only_final_mp4( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + await asyncio.sleep(0.05) + session = get_active_session("sim-udid-1") + seal_spy = AsyncMock(wraps=ios_ctx._active_driver.seal_recording_segment) + ios_ctx._active_driver.seal_recording_segment = seal_spy + + rendered = {} + + async def fake_render(segments, start, end, output_path, **kwargs): + rendered["segments"] = segments + rendered["range"] = (start, end) + Path(output_path).write_bytes(b"clip") + return True + + output_path = tmp_path / "clip.mp4" + shift = session.start_time - session.data_engine_start_time + with patch( + "artemis.controllers.unified_controller.render_timeline_clip", + AsyncMock(side_effect=fake_render), + ) as render: + res = await controller.extract_segment_metadata( + start_time=shift + 0.0, end_time=shift + 0.02, output_path=output_path + ) + assert res.success is True + assert res.video_path == output_path + seal_spy.assert_awaited_once() + # Only finalized MP4 segment paths are rendered — never the open .mov. + assert rendered["segments"] + assert all(s["path"].endswith(".mp4") for s in rendered["segments"]) + assert all(".mov" not in s["path"] for s in rendered["segments"]) + + # Generation-scoped cache: same range reuses the clip without render. + res2 = await controller.extract_segment_metadata( + start_time=shift + 0.0, end_time=shift + 0.02, output_path=output_path + ) + assert res2.success is True + assert render.await_count == 1 + + await controller._driver.stop_video_recording() + remove_active_session("sim-udid-1") + + +@pytest.mark.asyncio +async def test_ios_extract_rejects_range_outside_sealed_capture( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + session = get_active_session("sim-udid-1") + shift = session.start_time - session.data_engine_start_time + + with patch( + "artemis.controllers.unified_controller.render_timeline_clip", + AsyncMock(return_value=True), + ) as render: + res = await controller.extract_segment_metadata( + start_time=shift + 50.0, end_time=shift + 55.0 + ) + + assert res.success is False + assert "overlap" in res.message + render.assert_not_awaited() + await controller._driver.stop_video_recording() + remove_active_session("sim-udid-1") From 6f077079032a11e488d217ee3e0840067f30f285 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 04:16:40 +0900 Subject: [PATCH 05/32] fix: read iOS capture size from simctl io enumerate simctl io screenshot does not stream to stdout on Xcode 27, so the dimension probe always failed and segments finalized on the 1080x1920 fallback canvas. Parse the framebuffer IOSurface port size instead - it is the exact surface recordVideo encodes, excludes external scene displays, and swaps on rotation. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/drivers/ios/recording.py | 36 ++++++++++----- tests/unit/drivers/test_ios_recording.py | 57 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py index ba89c45f..bbdb717c 100644 --- a/artemis/drivers/ios/recording.py +++ b/artemis/drivers/ios/recording.py @@ -21,16 +21,14 @@ """ import asyncio -from io import BytesIO from pathlib import Path +import re import signal import tempfile import time from typing import Any from uuid import uuid4 -from PIL import Image - from artemis.config.paths import get_temp_dir from artemis.utils.video import ( get_ffmpeg_path, @@ -65,26 +63,44 @@ class IosRecordingSession(RecordingSession): conversion_tasks: list[asyncio.Task] = [] +def _parse_display_dimensions(text: str) -> tuple[int, int] | None: + """Largest ``IOSurface port`` (width, height) from ``simctl io enumerate``. + + ``recordVideo`` captures the device LCD without ``--display``; only that + framebuffer reports an ``IOSurface port`` (external scene displays carry + only ``Default width``/``height``), and its dimensions swap on rotation, + so the largest reported surface is the segment's coded size. + """ + best: tuple[int, int] | None = None + best_area = 0 + for match in re.finditer( + r"IOSurface port:\s*\n\s*width\s*=\s*(\d+)\s*\n\s*height\s*=\s*(\d+)", text + ): + width, height = int(match.group(1)), int(match.group(2)) + if width * height > best_area: + best, best_area = (width, height), width * height + return best + + async def probe_display_dimensions(device_id: str) -> tuple[int, int] | None: - """Read the raw capture dimensions from a simctl PNG screenshot.""" + """Read the recordVideo target display's pixel size via simctl.""" try: process = await asyncio.create_subprocess_exec( "xcrun", "simctl", "io", device_id, - "screenshot", - "--type=png", - "-", + "enumerate", stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, ) stdout, _stderr = await asyncio.wait_for(process.communicate(), PROBES_TIMEOUT_SECONDS) - with Image.open(BytesIO(stdout)) as image: - return image.size - except (OSError, TimeoutError, ValueError): + except (OSError, TimeoutError): + return None + if process.returncode != 0: return None + return _parse_display_dimensions(stdout.decode(errors="replace")) async def finalize_mov_to_mp4( diff --git a/tests/unit/drivers/test_ios_recording.py b/tests/unit/drivers/test_ios_recording.py index 7d00cce4..552c8408 100644 --- a/tests/unit/drivers/test_ios_recording.py +++ b/tests/unit/drivers/test_ios_recording.py @@ -609,3 +609,60 @@ async def test_probe_falls_back_to_cv2_only_when_ffprobe_missing(monkeypatch, tm corrupted = tmp_path / "corrupt.mp4" corrupted.write_bytes(b"not a movie") assert await video_utils.probe_video_segment(corrupted) == {} + + +# Mirrors real ``simctl io enumerate`` output on an iPhone simulator: +# the LCD is the only display with an IOSurface port; external scene displays +# (CarPlay wireless, resizable) report Default dims only and must be ignored. +ENUMERATE_SAMPLE = """\ +Port: + UUID: 0E2F25F7-2FF1-44B7-93A2-36AE37042D2D + Class: Unknown + Port Identifier: com.apple.display.captureservice + Power state: On + +Port: + UUID: 99261FD3-48E9-4CB4-B58D-5D2AF9CB5538 + Class: Display + Port Identifier: com.apple.framebuffer.display + Power state: On + Display class: 1 + Default width: 720 + Default height: 480 + Default pixel format: 'BGRA' + +Port: + UUID: 9A996636-389D-48C9-86D4-D2A252086D44 + Class: Display + Port Identifier: com.apple.framebuffer.display + Power state: On + Display class: 0 + Default width: 1206 + Default height: 2622 + Default pixel format: 'BGRA' + IOSurface port: + width = 1206 + height = 2622 + bytes per row = 4864 + size = 12763136 + +Port: + UUID: E7A14973-04D1-456A-858D-BD0EEF2FF5D5 + Class: Display + Port Identifier: com.apple.framebuffer.display + Power state: On + Display class: 1 + Default width: 7680 + Default height: 4320 + Default pixel format: 'BGRA' +""" + + +def test_parse_display_dimensions_uses_iosurface_not_default_dims(): + assert rec._parse_display_dimensions(ENUMERATE_SAMPLE) == (1206, 2622) + rotated = ENUMERATE_SAMPLE.replace( + "width = 1206", "width = 2622" + ).replace("height = 2622", "height = 1206") + assert rec._parse_display_dimensions(rotated) == (2622, 1206) + assert rec._parse_display_dimensions("Port:\n Class: Unknown\n") is None + assert rec._parse_display_dimensions("") is None From b7a7df698431a2e824593b73a77aa39f17c2c5b5 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 05:05:02 +0900 Subject: [PATCH 06/32] feat: route iOS Simulator through web, daemon, MCP, replay, and stream Extend the native Xcode 27 iOS support to every Artemis surface so iOS reaches feature parity with Android where the platform permits: - Shared simctl discovery (artemis/drivers/ios/discovery.py) and an IosDevicePool producing platform-tagged DeviceStatus entries, with explicit-UDID validation, Booted/Shutdown admission, and ambiguous "booted" rejection. - iOS locks scoped under "ios" (ios__) so a simulator UDID can never collide with an Android serial; queue items carry platform, ios_workspace, and the pinned UDID, and workers spawn with --platform ios --ios-workspace without leaking ADB_DEVICE_SERIAL. - Web API accepts platform="ios", validates UDIDs, skips the Android screen-lock probe, rejects locked_app_package, and lists simulators alongside Android devices in /api/devices. - Live stream follows the active iOS lock owner and captures frames via simctl io screenshot; /api/stream/device-state reports platform. - Replay preserves mobile_platform from session device_info, rebuilds iOS contexts with DevicePlatform.IOS, honors UDID overrides, and lists simulators as replay targets. - MCP mobile_run_task accepts a platform argument (daemon + standalone paths), mobile_get_device_state observes iOS through UnifiedMobileController + XcodeSimulatorDriver, and mobile_diagnose adjusts verdicts/probes for iOS targets. - An ios_simulators readiness probe reports Xcode/simulator/MCP availability without becoming a blocker for Android-only hosts. - artemis run and artemis batch forward platform, device_serial, and ios_workspace through the daemon. Android remains the default platform and its behavior is unchanged. Android-only operations (shell, logcat, UIAutomator, AVD launch, accessibility helper) fail explicitly for iOS. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/admin_console/replay_manager.py | 74 +- apps/admin_console/routers/stream.py | 11 +- apps/admin_console/routers/tasks.py | 78 +- apps/admin_console/schemas/task_schema.py | 4 + .../services/device_stream_service.py | 108 ++- .../services/task_queue_service.py | 69 +- artemis/core/diagnostics/device_smoke.py | 72 +- artemis/core/diagnostics/engine.py | 3 + artemis/core/diagnostics/probes/ios_probe.py | 142 ++++ artemis/core/diagnostics/readiness.py | 5 +- artemis/drivers/ios/discovery.py | 117 +++ artemis/drivers/ios/xcode_driver.py | 32 +- artemis/interfaces/cli/commands/batch.py | 52 ++ artemis/interfaces/cli/commands/run.py | 4 + artemis/mcp/adb_server.py | 33 +- artemis/runtime/__init__.py | 7 + artemis/runtime/adb_endpoint.py | 66 +- artemis/runtime/daemon_client.py | 14 +- artemis/runtime/device_pool.py | 4 +- artemis/runtime/ios_device_pool.py | 151 ++++ artemis/sdk/agent.py | 2 + docs/ios.md | 44 +- mcp_server/background/task_runner.py | 33 +- mcp_server/tools/device_state.py | 37 +- mcp_server/tools/diagnose.py | 104 ++- mcp_server/tools/task_runner.py | 56 +- tests/unit/core/test_readiness.py | 1 + tests/unit/test_ios_parity.py | 727 ++++++++++++++++++ third_party/mobile_use/sdk/agent.py | 11 +- 29 files changed, 1879 insertions(+), 182 deletions(-) create mode 100644 artemis/core/diagnostics/probes/ios_probe.py create mode 100644 artemis/drivers/ios/discovery.py create mode 100644 artemis/runtime/ios_device_pool.py create mode 100644 tests/unit/test_ios_parity.py diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index c708629d..8686a104 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -397,16 +397,33 @@ def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path return output_dir def list_devices(self) -> list[dict]: - """Dynamically queries the ADB server for connected Android devices.""" + """Connected Android devices plus available iOS simulators.""" + devices: list[dict] = [] try: from adbutils import AdbClient adb = AdbClient(host="localhost", port=5037) - devices = adb.device_list() - return [{"serial": d.serial, "status": "online"} for d in devices] + devices.extend( + {"serial": d.serial, "status": "online", "platform": "android"} + for d in adb.device_list() + ) except Exception as e: print(f"Warning: Failed to query device list from ADB: {e}") - return [] + try: + from artemis.drivers.ios.discovery import list_ios_simulators_sync + + for sim in list_ios_simulators_sync() or []: + devices.append( + { + "serial": sim["udid"], + "status": "online" if sim.get("state") == "Booted" else "shutdown", + "platform": "ios", + "model": sim.get("name"), + } + ) + except Exception as e: + print(f"Warning: Failed to query iOS simulators: {e}") + return devices def load_session_goal( self, session_id: str, step_dir: Path, original_db_path: str = None @@ -1604,12 +1621,44 @@ def _preemptive_clip_video( except Exception as e: print(f"Warning: Failed to update sandbox SQLite database: {e}") + @staticmethod + def _replay_device_context(device_info: dict, override_device_id: str | None): + """Build the simulated ``DeviceContext`` for a replayed session. + + The recorded session decides the platform: an iOS session replays + through the Xcode driver against a simulator, never through ADB. The + frontend's device pick only retargets iOS replays (the picker was + always decorative for Android and stays that way); the driver still + validates the UDID against simctl at connect time. + """ + from artemis.context import DeviceContext, DevicePlatform + + sim_platform = str( + device_info.get("mobile_platform") or DevicePlatform.ANDROID.value + ).lower() + mobile_platform = ( + DevicePlatform.IOS + if sim_platform == DevicePlatform.IOS.value + else DevicePlatform.ANDROID + ) + sim_device_id = device_info.get("device_id", "replay-device") + if mobile_platform == DevicePlatform.IOS and override_device_id: + sim_device_id = override_device_id + return DeviceContext( + host_platform=("DARWIN" if mobile_platform == DevicePlatform.IOS else "LINUX"), + mobile_platform=mobile_platform, + device_id=sim_device_id, + device_width=device_info.get("device_width", 1080), + device_height=device_info.get("device_height", 2400), + ) + def create_ctx( self, session_id: str, step_number: int, agent_name: str = "explorer", replay_id: str = None, + override_device_id: str = None, ) -> ArtemisContext: """Creates a sandboxed ArtemisContext and DataEngine for the given session and step.""" self._ensure_session_chunked(session_id) @@ -1701,18 +1750,12 @@ def create_ctx( # Malformed device_info: simulate with the defaults below. pass - sim_device_id = device_info.get("device_id", "replay-device") - sim_w = device_info.get("device_width", 1080) - sim_h = device_info.get("device_height", 2400) - - device_context = DeviceContext( - host_platform="LINUX", - mobile_platform=DevicePlatform.ANDROID, - device_id=sim_device_id, - device_width=sim_w, - device_height=sim_h, + device_context = self._replay_device_context(device_info, override_device_id) + sim_device_id = device_context.device_id + print( + f"Replay simulated device: {sim_device_id} " + f"({device_context.device_width}x{device_context.device_height})" ) - print(f"Replay simulated device: {sim_device_id} ({sim_w}x{sim_h})") from artemis.config import get_default_llm_config @@ -1872,6 +1915,7 @@ async def replay_step_tool( step_number=step_number, agent_name=agent_name, replay_id=replay_id, + override_device_id=override_device_id, ) state = self.instantiate_state( diff --git a/apps/admin_console/routers/stream.py b/apps/admin_console/routers/stream.py index 5bc3c688..6cdd2f58 100644 --- a/apps/admin_console/routers/stream.py +++ b/apps/admin_console/routers/stream.py @@ -45,12 +45,13 @@ async def stream_device_live(): @router.get("/api/stream/device-state") async def get_device_stream_state(): - """Returns whether an ADB device is connected and live streaming is available.""" - serial = await device_stream_service.get_device_serial() + """Returns whether a device is connected and live streaming is available.""" + target = await device_stream_service.get_stream_target() return JSONResponse( { - "connected": serial is not None, - "serial": serial, - "live_stream_url": "/api/stream/device-live" if serial else None, + "connected": target is not None, + "serial": target["serial"] if target else None, + "platform": target["platform"] if target else None, + "live_stream_url": "/api/stream/device-live" if target else None, } ) diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index 26fca461..e6776453 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -19,7 +19,7 @@ from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse from artemis.core.diagnostics import readiness_engine -from artemis.runtime import DeviceExecutionLock, device_pool +from artemis.runtime import DeviceExecutionLock, device_pool, ios_device_pool try: from admin_console.core.state import state @@ -113,6 +113,18 @@ async def run_task(request: RunRequest): "total_queued": len(state.queue_tasks), } + platform = str(request.platform or "android").strip().lower() or "android" + if platform not in ("android", "ios"): + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) + if platform == "ios" and request.locked_app_package: + raise HTTPException( + status_code=400, + detail="locked_app_package is unavailable for iOS simulator tasks.", + ) + # Reject an explicit unknown/offline target before running the more # expensive readiness probe. Besides producing a stable SDK response, # this avoids probing the currently active device for a serial that can @@ -121,7 +133,12 @@ async def run_task(request: RunRequest): # fail downstream with a clear error instead. if request.device_serial: try: - rejection = await device_pool.validate_explicit_serial_async(request.device_serial) + if platform == "ios": + rejection = await ios_device_pool.validate_explicit_serial_async( + request.device_serial + ) + else: + rejection = await device_pool.validate_explicit_serial_async(request.device_serial) except Exception: rejection = None if rejection: @@ -140,30 +157,35 @@ async def run_task(request: RunRequest): # With no explicit serial the probe itself resolves a live target (it # prefers the diagnostics target preference, then any unlocked ready # device); the verified serial is bound below. + # iOS skips the Android screen-lock probe entirely: the Xcode driver + # validates and boots the pinned simulator at execution time. target_serial = request.device_serial - device_probe = await readiness_engine.run_device_submission_probe(target_serial=target_serial) - if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: - locked_serial = ( - device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" - ) - detail = ( - f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace( - " ", " " - ).strip() - if device_probe.summary == "Device Locked" - else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace( - " ", " " - ).strip() + if platform != "ios": + device_probe = await readiness_engine.run_device_submission_probe( + target_serial=target_serial ) - raise HTTPException(status_code=409, detail=detail) + if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: + locked_serial = ( + device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" + ) + detail = ( + f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace( + " ", " " + ).strip() + if device_probe.summary == "Device Locked" + else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace( + " ", " " + ).strip() + ) + raise HTTPException(status_code=409, detail=detail) - if device_probe and device_probe.metadata.get("active_device"): - verified_serial = device_probe.metadata["active_device"].get("serial") - # Only auto-selected targets may be re-bound to the probed device. An - # explicitly requested serial is never silently replaced -- if it is - # invalid, enqueue_tasks rejects the submission with a clear error. - if verified_serial and not request.device_serial: - target_serial = verified_serial + if device_probe and device_probe.metadata.get("active_device"): + verified_serial = device_probe.metadata["active_device"].get("serial") + # Only auto-selected targets may be re-bound to the probed device. An + # explicitly requested serial is never silently replaced -- if it is + # invalid, enqueue_tasks rejects the submission with a clear error. + if verified_serial and not request.device_serial: + target_serial = verified_serial return await task_queue_service.enqueue_tasks( incoming_goals, @@ -178,6 +200,8 @@ async def run_task(request: RunRequest): ingress=request.ingress or "frontend", session_id=request.session_id, conversation_id=request.conversation_id, + platform=platform, + ios_workspace=request.ios_workspace, ) @@ -199,9 +223,11 @@ async def get_run_defaults(): @router.get("/api/devices") async def list_devices(): - """List all connected Android devices with their busy / idle status.""" - devices = await device_pool.list_devices_async() - return {"devices": [d.to_dict() for d in devices]} + """List all connected devices (Android + iOS simulators) with lock status.""" + android_devices, ios_devices = await asyncio.gather( + device_pool.list_devices_async(), ios_device_pool.list_devices_async() + ) + return {"devices": [d.to_dict() for d in (*android_devices, *ios_devices)]} @router.post("/api/stop") diff --git a/apps/admin_console/schemas/task_schema.py b/apps/admin_console/schemas/task_schema.py index 144a9720..6843dba4 100644 --- a/apps/admin_console/schemas/task_schema.py +++ b/apps/admin_console/schemas/task_schema.py @@ -29,6 +29,10 @@ class RunRequest(BaseModel): locked_app_package: str | None = None app_path: str | None = None device_serial: str | None = None + # Target platform: "android" (default) or "ios". For iOS, device_serial is a + # simulator UDID and ios_workspace optionally scopes the Xcode approval grant. + platform: str | None = None + ios_workspace: str | None = None ingress: str | None = "frontend" session_id: str | None = None conversation_id: str | None = None diff --git a/apps/admin_console/services/device_stream_service.py b/apps/admin_console/services/device_stream_service.py index 6455a3e7..77ee9061 100644 --- a/apps/admin_console/services/device_stream_service.py +++ b/apps/admin_console/services/device_stream_service.py @@ -15,7 +15,9 @@ """Device Live Screen Streaming Service. Provides real-time, low-latency device screen frames over HTTP MJPEG and WebSocket. -Operates concurrently with ADB agent actions with zero interference. +Android frames come from ``adb exec-out screencap``; iOS Simulator frames come +from the native ``xcrun simctl io screenshot`` capture. The target is +picked per frame so the stream follows whichever platform is under automation. """ import asyncio @@ -23,7 +25,11 @@ import subprocess import time from collections.abc import AsyncGenerator +from pathlib import Path +from artemis.config.paths import get_temp_dir +from artemis.runtime import DeviceExecutionLock, ios_device_pool +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.toolchain import find_adb logger = logging.getLogger("artemis.stream_service") @@ -40,7 +46,39 @@ def __init__(self): self._is_capturing = False self._capture_task: asyncio.Task | None = None + async def get_stream_target(self) -> dict[str, str] | None: + """Pick the device the stream should follow right now. + + An actively locked iOS simulator wins (it is the device under + automation), then a connected Android device, then the single + unambiguous booted simulator. Returns ``{"platform", "serial"}``. + """ + try: + for owner in DeviceExecutionLock.get_active_owners().values(): + if ( + owner + and getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE + and owner.device_id + ): + return {"platform": "ios", "serial": str(owner.device_id)} + except Exception as exc: + logger.debug(f"[StreamService] iOS lock-owner scan failed: {exc}") + + serial = await self._android_serial() + if serial: + return {"platform": "android", "serial": serial} + + udid = await ios_device_pool.select_device_async() + if udid: + return {"platform": "ios", "serial": udid} + return None + async def get_device_serial(self) -> str | None: + """Serial of the currently streamable device, regardless of platform.""" + target = await self.get_stream_target() + return target["serial"] if target else None + + async def _android_serial(self) -> str | None: """Find the currently connected active ADB device serial.""" try: adb_bin = find_adb() @@ -60,27 +98,65 @@ async def get_device_serial(self) -> str | None: logger.warning(f"Error checking adb devices: {e}") return None + async def _capture_android(self, serial: str | None) -> bytes | None: + adb_bin = find_adb() + cmd = ( + [adb_bin, "-s", serial, "exec-out", "screencap", "-p"] + if serial + else [adb_bin, "exec-out", "screencap", "-p"] + ) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout if proc.returncode == 0 and len(stdout) > 1000 else None + + async def _capture_ios(self, udid: str) -> bytes | None: + """One PNG frame from the simulator's framebuffer via simctl. + + ``simctl io screenshot`` only writes files (``-`` is not a stdout + sink on Xcode 27), so frames stream through one reused temp path. + """ + # Reused path per UDID: the loop is serialized, and simctl writes the + # file itself, so no per-frame temp file churn is needed. + frame_path = Path(get_temp_dir("streams")) / f"ios_stream_{udid}.png" + try: + proc = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + udid, + "screenshot", + "--type=png", + str(frame_path), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + await proc.communicate() + if proc.returncode == 0 and frame_path.exists(): + data = frame_path.read_bytes() + return data if len(data) > 1000 else None + finally: + frame_path.unlink(missing_ok=True) + return None + async def _capture_loop(self): """Background frame capture loop that runs while listeners > 0.""" logger.info("[StreamService] Starting live screen capture loop...") while self._active_listeners > 0: try: start_t = time.time() - serial = await self.get_device_serial() - adb_bin = find_adb() - cmd = ( - [adb_bin, "-s", serial, "exec-out", "screencap", "-p"] - if serial - else [adb_bin, "exec-out", "screencap", "-p"] - ) - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, _ = await proc.communicate() - if proc.returncode == 0 and len(stdout) > 1000: - self._latest_frame = stdout + target = await self.get_stream_target() + frame = None + if target is not None: + if target["platform"] == "ios": + frame = await self._capture_ios(target["serial"]) + else: + frame = await self._capture_android(target["serial"]) + if frame is not None: + self._latest_frame = frame self._last_frame_time = time.time() elapsed = time.time() - start_t diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index b99b4c91..0cefab5c 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -52,6 +52,7 @@ request_cancel, trace_store, ) +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE, IosTarget logger = logging.getLogger(__name__) @@ -216,14 +217,16 @@ def _enforce_sync() -> None: ).start() @staticmethod - def _task_target(task_item: dict[str, Any]) -> AdbTarget: + def _task_target(task_item: dict[str, Any]) -> AdbTarget | IosTarget: + serial = task_item.get("device_serial") + if str(task_item.get("platform") or "android").lower() == "ios": + return IosTarget(serial=str(serial) if serial else None) endpoint_data = task_item.get("adb_endpoint") endpoint = ( AdbEndpoint.from_mapping(endpoint_data) if isinstance(endpoint_data, dict) else current_adb_endpoint() ) - serial = task_item.get("device_serial") return AdbTarget(endpoint=endpoint, serial=str(serial) if serial else None) @classmethod @@ -498,7 +501,7 @@ def _build_worker_invocation( sess_id: Any, goal: str, profile: str, - target: AdbTarget, + target: AdbTarget | IosTarget, ) -> tuple[list[str], dict[str, str]]: """Assemble the worker subprocess command line and environment.""" expected_output = task_item.get("expected_output") @@ -525,7 +528,7 @@ def _build_worker_invocation( env["ARTEMIS_SESSION_ID"] = str(sess_id) env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend")) env["ARTEMIS_TASK_WORKER"] = "1" - target.endpoint.apply_to_environment(env) + target.apply_to_environment(env) env[DeviceExecutionLock.LOCK_SCOPE_ENV] = target.lock_scope queue_ticket = task_item.get("queue_ticket") if queue_ticket: @@ -556,7 +559,14 @@ def _build_worker_invocation( if app_path: cmd.extend(["--app-path", str(app_path)]) device_serial = task_item.get("device_serial") - if device_serial: + if isinstance(target, IosTarget): + cmd.extend(["--platform", "ios"]) + ios_workspace = task_item.get("ios_workspace") + if ios_workspace: + cmd.extend(["--ios-workspace", str(ios_workspace)]) + if device_serial: + cmd.extend(["--device-serial", str(device_serial)]) + elif device_serial: cmd.extend(["--device-serial", str(device_serial)]) env["ADB_DEVICE_SERIAL"] = str(device_serial) return cmd, env @@ -569,7 +579,7 @@ def _register_worker_run( sess_id: Any, goal: str, profile: str, - target: AdbTarget, + target: AdbTarget | IosTarget, proc: asyncio.subprocess.Process, ) -> None: """Record the spawned worker in shared state and hand it the device reservation.""" @@ -580,7 +590,9 @@ def _register_worker_run( "process": proc, "device_id": str(device_serial) if device_serial else None, "lock_key": target.lock_key if device_serial else None, - "adb_endpoint": target.endpoint.to_dict(), + "platform": target.platform, + "target": target.to_dict(), + "adb_endpoint": (target.endpoint.to_dict() if isinstance(target, AdbTarget) else None), "goal": goal, "profile": profile, } @@ -963,7 +975,9 @@ def _find_duplicate_submission( return None @classmethod - async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str, Any] | None: + async def _reject_unavailable_device( + cls, device_serial: str | None, platform: str = "android" + ) -> dict[str, Any] | None: """Return the rejection response for an unattached explicit serial, if any.""" # Strict device binding: reject an explicitly requested serial that is not # attached and authorized, instead of silently running on another device. @@ -971,9 +985,14 @@ async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str # the task can proceed and fail downstream with a clear no-device error. if device_serial: try: - from artemis.runtime import device_pool + if platform == "ios": + from artemis.runtime import ios_device_pool + + rejection = await ios_device_pool.validate_explicit_serial_async(device_serial) + else: + from artemis.runtime import device_pool - rejection = await device_pool.validate_explicit_serial_async(device_serial) + rejection = await device_pool.validate_explicit_serial_async(device_serial) except Exception: rejection = None if rejection: @@ -1004,6 +1023,8 @@ def _create_queue_item( conversation_id: str | None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ) -> dict[str, Any]: """Reserve a device slot and build one pending queue item for a goal.""" sess_id = single_session_id if single_session_id else str(uuid.uuid4()) @@ -1015,7 +1036,7 @@ def _create_queue_item( device_id=assigned_serial or "pending", session_id=sess_id, ingress=ingress, - lock_scope=endpoint.identity, + lock_scope=IOS_LOCK_SCOPE if platform == "ios" else endpoint.identity, ) return { "session_id": sess_id, @@ -1029,6 +1050,8 @@ def _create_queue_item( "app_path": app_path, "device_serial": assigned_serial, "adb_endpoint": endpoint.to_dict(), + "platform": platform, + "ios_workspace": ios_workspace, "ingress": ingress, "conversation_id": conversation_id, "status": "pending", @@ -1052,13 +1075,20 @@ async def enqueue_tasks( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, ) -> dict[str, Any]: """Enqueues one or more goals and wakes up the background worker. ``verification_level`` and ``explorer_mode`` are Pro-profile tuning knobs forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``; they are normalised here so the queue item and the CLI see one spelling. + + ``platform="ios"`` enqueues a native iOS Simulator task: the serial is a + simulator UDID, the device lock is scoped under ``ios``, and the worker + runs with ``--platform ios`` instead of ADB bindings. """ + platform = str(platform or "android").strip().lower() or "android" verification_level = ( str(verification_level).strip().lower() or None if verification_level else None ) @@ -1075,17 +1105,22 @@ async def enqueue_tasks( if duplicate_response is not None: return duplicate_response - rejection_response = await cls._reject_unavailable_device(device_serial) + rejection_response = await cls._reject_unavailable_device(device_serial, platform) if rejection_response is not None: return rejection_response single_session_id = session_id if (session_id and len(goals) == 1) else None if not device_serial: - # Device enumeration may block on ADB. - from artemis.runtime import device_pool - try: - device_serial = await device_pool.select_device_async() + if platform == "ios": + from artemis.runtime import ios_device_pool + + device_serial = await ios_device_pool.select_device_async() + else: + # Device enumeration may block on ADB. + from artemis.runtime import device_pool + + device_serial = await device_pool.select_device_async() except Exception: device_serial = None for i, goal in enumerate(goals): @@ -1105,6 +1140,8 @@ async def enqueue_tasks( conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, ) state.queue_items.append(task_item) enqueued_tasks.append(task_item) diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 3c43c9cf..cb7e78ce 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -96,8 +96,39 @@ def _uiautomator_fix(serial: str | None) -> list[str]: ] -def fix_for_error(error: str | None, serial: str | None) -> list[str]: +def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: + """iOS Simulator repair steps (simctl/Xcode instead of adb/UIAutomator).""" + s = serial or "" + text = (error or "").lower() + if "busy" in text or "another task" in text: + return [ + "Wait for the running task to finish, or stop it with mobile_manage_task(action='stop', ...).", + "Then rerun the smoke test.", + ] + if "approv" in text or "authoriz" in text or "permission" in text: + return [ + "Approve Artemis's agent access in Xcode (the first run opens a workspace approval prompt; choose persistent approval).", + "Confirm the permitted agent and workspace under Xcode > Settings > Agentic Tools.", + ] + if "xcode" in text or "macos" in text or "darwin" in text: + return [ + "Install Xcode 27 or later and select it: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", + ] + if "simulator" in text or "udid" in text or "boot" in text or "not available" in text: + return [ + f"List simulators: xcrun simctl list devices; boot one with: xcrun simctl boot {s}", + "Create a simulator in Xcode > Settings > Platforms if none exist.", + ] + return [ + "Check native capture manually: xcrun simctl io booted screenshot /tmp/ios.png", + 'Verify Xcode MCP access: echo \'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}\' | xcrun mcpbridge', + ] + + +def fix_for_error(error: str | None, serial: str | None, platform: str = "android") -> list[str]: """Map an error string to concrete repair steps (substring match, case-insensitive).""" + if platform.lower() == "ios": + return _ios_fix_for_error(error, serial) if not error: return [] text = error.lower() @@ -260,15 +291,20 @@ def _decoded_length(b64: Any) -> int | None: async def smoke_test_device( - device_serial: str | None = None, timeout_seconds: float = 20.0 + device_serial: str | None = None, + timeout_seconds: float = 20.0, + platform: str = "android", ) -> dict[str, Any]: """Observe the device exactly like ``mobile_get_device_state`` and report a verdict. + ``platform="ios"`` exercises the native Xcode 27 simulator path + (``simctl`` + mcpbridge session) instead of ADB/UIAutomator. + Never raises. Returns:: {"ok": bool, "serial": str | None, "elapsed_seconds": float, "screenshot_bytes": int | None, "element_count": int | None, - "hierarchy_backend": "helper" | "uiautomator" | None, + "hierarchy_backend": "helper" | "uiautomator" | "xcode" | None, "error": str | None, "fix": list[str]} """ started = time.monotonic() @@ -289,7 +325,9 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any result["elapsed_seconds"] = round(time.monotonic() - started, 3) result["error"] = error result["ok"] = error is None - result["fix"] = fix_for_error(cause if cause is not None else error, result["serial"]) + result["fix"] = fix_for_error( + cause if cause is not None else error, result["serial"], platform + ) return result busy = _find_busy_owner(requested_serial) @@ -303,11 +341,17 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any " skipping the screen capture so the running task is not disturbed." ) + is_ios = platform.lower() == "ios" try: from artemis.mcp import adb_server + # Keep the Android call shape identical; the platform kwarg only + # exists for iOS. controller = await _run_in_daemon_thread( - lambda: adb_server._get_controller(device_serial=device_serial), + lambda: adb_server._get_controller( + device_serial=device_serial, + **({"target_platform": platform} if is_ios else {}), + ), timeout_seconds, "controller-init", ) @@ -329,8 +373,15 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any remaining = max(0.5, timeout_seconds - (time.monotonic() - started)) try: + + async def _observe() -> Any: + if is_ios: + # The native Xcode session opens lazily on first use. + await controller._driver.connect() + return await controller.get_screen_data() + device_data = await _run_in_daemon_thread( - lambda: _run_coroutine_blocking(controller.get_screen_data), + lambda: _run_coroutine_blocking(_observe), remaining, "screen-data", ) @@ -344,9 +395,14 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None)) result["element_count"] = _count_elements(getattr(device_data, "elements", None)) - from artemis.clients.screen_client_factory import describe_backend + if is_ios: + result["hierarchy_backend"] = "xcode" + else: + from artemis.clients.screen_client_factory import describe_backend - result["hierarchy_backend"] = describe_backend(getattr(controller_ctx, "ui_adb_client", None)) + result["hierarchy_backend"] = describe_backend( + getattr(controller_ctx, "ui_adb_client", None) + ) if result["screenshot_bytes"] is None: return _finish("Screen capture returned no screenshot data") diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index 97f9d20c..a46393a7 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -27,6 +27,7 @@ LLMCredentialsProbe, VisionOCRProbe, ) +from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe from artemis.core.diagnostics.probes.runtime_probe import ( PythonRuntimeProbe, SystemConfigProbe, @@ -67,6 +68,7 @@ def __init__(self): self._credentials_probe = LLMCredentialsProbe() self._ocr_probe = VisionOCRProbe() self._adb_probe = AdbDeviceProbe() + self._ios_probe = IosSimulatorProbe() self._report_cache: SystemReadinessReport | None = None self._report_cache_time = 0.0 self._report_cache_generation = -1 @@ -80,6 +82,7 @@ def __init__(self): self.register_probe(self._credentials_probe) self.register_probe(self._ocr_probe) self.register_probe(self._adb_probe) + self.register_probe(self._ios_probe) def register_probe(self, probe: BaseProbe) -> None: """Register a new diagnostic probe.""" diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py new file mode 100644 index 00000000..d81e45c7 --- /dev/null +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -0,0 +1,142 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator Readiness Probe. + +Informational only (``is_blocker = False``): iOS support is opt-in per task, +so a missing Xcode toolchain must never block the Android default path. The +probe reports whether this host could run an iOS Simulator task today — +macOS, Xcode 27+, and at least one available simulator. +""" + +import asyncio +import re + +from artemis.core.diagnostics.probes.base import BaseProbe +from artemis.core.diagnostics.schema import ( + ProbeAction, + ProbeCategory, + ProbeResult, + ProbeStatus, +) +from artemis.drivers.ios.discovery import ( + list_ios_simulators, + run_xcrun, + simctl_available, +) + + +class IosSimulatorProbe(BaseProbe): + """Reports iOS Simulator automation capability (Xcode 27+ on macOS).""" + + @property + def probe_id(self) -> str: + return "ios_simulators" + + @property + def category(self) -> ProbeCategory: + return ProbeCategory.DEVICE + + @property + def is_blocker(self) -> bool: + return False + + async def probe(self) -> ProbeResult: + if not simctl_available(): + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Simulator", + status=ProbeStatus.SKIPPED, + is_blocker=self.is_blocker, + summary="Unavailable", + description=( + "iOS Simulator automation requires macOS with Xcode 27+; " + "this host cannot enumerate simulators." + ), + metadata={"platform_supported": False}, + ) + + xcode_ok = False + xcode_version: str | None = None + try: + raw = await asyncio.wait_for(run_xcrun("xcodebuild", "-version"), timeout=15.0) + match = re.search(rb"Xcode\s+(\d+(?:\.\d+)*)", raw) + if match: + xcode_version = match.group(1).decode() + xcode_ok = int(xcode_version.split(".")[0]) >= 27 + except (OSError, RuntimeError, TimeoutError): + xcode_version = None + + simulators = await list_ios_simulators() + booted = [d for d in (simulators or []) if d.get("state") == "Booted"] + metadata = { + "platform_supported": True, + "xcode_version": xcode_version, + "xcode_27_or_newer": xcode_ok, + "simulator_count": len(simulators or []), + "booted_udids": [d.get("udid") for d in booted], + "simulators": simulators or [], + } + + if not xcode_ok: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Simulator", + status=ProbeStatus.WARN, + is_blocker=self.is_blocker, + summary="Xcode 27+ Required", + description=( + "Native iOS automation requires Xcode 27 or later" + + (f" (found {xcode_version})." if xcode_version else "; none detected.") + ), + metadata=metadata, + actions=[ + ProbeAction( + action_type="hint", + label="Select Xcode 27", + payload="sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", + ), + ], + ) + + if not simulators: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Simulator", + status=ProbeStatus.WARN, + is_blocker=self.is_blocker, + summary="No Simulators", + description=( + "Xcode is installed but no available iOS simulators were found. " + "Create one in Xcode > Settings > Platforms or via simctl." + ), + metadata=metadata, + ) + + summary = f"{len(booted)} Booted" if booted else f"{len(simulators)} Available" + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Simulator", + status=ProbeStatus.PASS, + is_blocker=self.is_blocker, + summary=summary, + description=( + f"Xcode {xcode_version} with {len(simulators)} available iOS " + f"simulator(s){' (' + str(len(booted)) + ' booted)' if booted else ''}. " + "Submit a task with platform='ios' and a simulator UDID to use them." + ), + metadata=metadata, + ) diff --git a/artemis/core/diagnostics/readiness.py b/artemis/core/diagnostics/readiness.py index 8f8f0853..c197c374 100644 --- a/artemis/core/diagnostics/readiness.py +++ b/artemis/core/diagnostics/readiness.py @@ -41,8 +41,9 @@ "integration_host": 2, "gemini_api_key": 3, "android_adb": 4, - "toolchain": 5, - "vision_ocr_key": 6, + "ios_simulators": 5, + "toolchain": 6, + "vision_ocr_key": 7, } diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py new file mode 100644 index 00000000..40221293 --- /dev/null +++ b/artemis/drivers/ios/discovery.py @@ -0,0 +1,117 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared ``simctl`` device enumeration for iOS Simulator support. + +Both the Xcode driver (target validation) and the runtime pools (discovery, +explicit-serial validation, auto-selection) read the same +``xcrun simctl list devices --json`` output through this module so parsing +stays in one place. Every function fails closed to ``None``/``[]`` on +missing tooling so Android-only hosts never see iOS errors. +""" + +import asyncio +import json +import shutil +import subprocess +import sys +from typing import Any + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +SIMCTL_QUERY_TIMEOUT = 15.0 + + +async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: + """Run argv directly, reporting native errors and reaping cancelled children.""" + process = await asyncio.create_subprocess_exec( + "xcrun", + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except (TimeoutError, asyncio.CancelledError): + if process.returncode is None: + process.kill() + await process.communicate() + raise + if process.returncode: + raise RuntimeError( + f"xcrun {' '.join(arguments[:3])} failed: {stderr.decode(errors='replace').strip()}" + ) + return stdout + + +def simctl_available() -> bool: + """Whether this host can enumerate iOS simulators at all.""" + return sys.platform == "darwin" and shutil.which("xcrun") is not None + + +def parse_simctl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Flatten ``simctl list devices --json`` into available iOS entries.""" + return [ + { + "udid": device.get("udid"), + "name": device.get("name"), + "state": device.get("state"), + "runtime": runtime, + } + for runtime, entries in payload.get("devices", {}).items() + if ".iOS-" in runtime + for device in entries + if device.get("isAvailable") and device.get("udid") + ] + + +async def list_ios_simulators() -> list[dict[str, Any]] | None: + """All available iOS simulators, or ``None`` when enumeration fails.""" + if not simctl_available(): + return None + try: + raw = await run_xcrun("simctl", "list", "devices", "--json", timeout=SIMCTL_QUERY_TIMEOUT) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"simctl device enumeration failed: {exc}") + return None + try: + return parse_simctl_devices(json.loads(raw)) + except (ValueError, TypeError) as exc: + logger.debug(f"simctl device list parse failed: {exc}") + return None + + +def list_ios_simulators_sync() -> list[dict[str, Any]] | None: + """Synchronous variant for non-async callers (e.g. replay device lists).""" + if not simctl_available(): + return None + try: + completed = subprocess.run( + ["xcrun", "simctl", "list", "devices", "--json"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=SIMCTL_QUERY_TIMEOUT, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.debug(f"simctl device enumeration failed: {exc}") + return None + if completed.returncode != 0: + return None + try: + return parse_simctl_devices(json.loads(completed.stdout)) + except (ValueError, TypeError) as exc: + logger.debug(f"simctl device list parse failed: {exc}") + return None diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index 10f213b8..71407d5f 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -26,6 +26,7 @@ from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, XcodeBridge +from artemis.drivers.ios.discovery import parse_simctl_devices, run_xcrun from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy from artemis.drivers.ios.recording import IosRecordingSession, IosScreenRecorder from third_party.mobile_use.controllers.types import ElementQuery @@ -35,29 +36,6 @@ logger = get_logger(__name__) -async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: - """Run argv directly, reporting native errors and reaping cancelled children.""" - process = await asyncio.create_subprocess_exec( - "xcrun", - *arguments, - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - try: - stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) - except (TimeoutError, asyncio.CancelledError): - if process.returncode is None: - process.kill() - await process.communicate() - raise - if process.returncode: - raise RuntimeError( - f"xcrun {' '.join(arguments[:3])} failed: {stderr.decode(errors='replace').strip()}" - ) - return stdout - - class XcodeSimulatorDriver(BaseDeviceDriver): """One explicitly selected iOS simulator and one native interaction session. @@ -122,13 +100,7 @@ async def _resolve_device(self) -> dict[str, Any]: "Native iOS interaction requires Xcode 27 or later. Set DEVELOPER_DIR to select it." ) devices = json.loads(await run_xcrun("simctl", "list", "devices", "--json")) - available = [ - device - for runtime, entries in devices.get("devices", {}).items() - if ".iOS-" in runtime - for device in entries - if device.get("isAvailable") - ] + available = parse_simctl_devices(devices) if self._device_id == "booted": candidates = [device for device in available if device.get("state") == "Booted"] if len(candidates) != 1: diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 225ba98f..4ba4d436 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -38,6 +38,9 @@ async def run_batch_tasks( delay_seconds: float = 5.0, verification_level: str | None = None, explorer_pro_mode: str | None = None, + platform: str = "android", + device_serial: str | None = None, + ios_workspace: str | None = None, ) -> None: """Executes a list of automation tasks sequentially. @@ -49,6 +52,9 @@ async def run_batch_tasks( 'strict') for the Pro profile; ignored by Flash. explorer_pro_mode: Explorer tier ('flash', 'pro', 'ultra') behind ``ask_explorer`` under the Pro profile; ignored by Flash. + platform: 'android' (default) or 'ios' (local iOS Simulator). + device_serial: Android serial or iOS simulator UDID for all tasks. + ios_workspace: Xcode project/workspace for first-run iOS approval. """ if not os.environ.get("ARTEMIS_TASK_INGRESS"): os.environ["ARTEMIS_TASK_INGRESS"] = "cli" @@ -59,6 +65,15 @@ async def run_batch_tasks( config_builder.with_verification_level(verification_level) if explorer_pro_mode is not None: config_builder.with_explorer(pro_mode=explorer_pro_mode) + if platform.lower() == "ios": + config_builder.for_ios_simulator( + device_id=device_serial or "booted", + workspace_path=ios_workspace, + ) + elif device_serial: + from artemis.context import DevicePlatform + + config_builder.for_device(DevicePlatform.ANDROID, device_serial) config = config_builder.build() agent = Agent(config=config) @@ -162,8 +177,39 @@ def batch_command( help="Explorer tier behind ask_explorer under the Pro profile ('flash', 'pro', 'ultra').", ), ] = None, + platform: Annotated[ + str, + typer.Option( + "--platform", + help="Target mobile platform: 'android' (default) or 'ios' (local iOS simulator).", + ), + ] = "android", + device_serial: Annotated[ + str | None, + typer.Option( + "--device-serial", + help="Android serial or iOS simulator UDID for every task in the batch.", + ), + ] = None, + ios_workspace: Annotated[ + Path | None, + typer.Option( + "--ios-workspace", + help="Existing Xcode project/workspace to request iOS first-run agent approval.", + ), + ] = None, ) -> None: """Execute multiple automation tasks in sequence.""" + platform = platform.strip().lower() + if platform not in ("android", "ios"): + typer.secho( + f"Error: --platform must be 'android' or 'ios' (got '{platform}').", + fg=typer.colors.RED, + ) + raise typer.Exit(1) + if ios_workspace is not None and platform != "ios": + typer.secho("Error: --ios-workspace requires --platform ios.", fg=typer.colors.RED) + raise typer.Exit(1) task_list: list[str] = [] if tasks_file: @@ -222,8 +268,11 @@ def batch_command( resp = submit_batch_to_daemon( task_list, profile=profile, + device_serial=device_serial, verification_level=verification_level, explorer_mode=explorer_pro_mode, + platform=platform, + ios_workspace=str(ios_workspace) if ios_workspace else None, base_url=base_url, ) if resp and resp.get("tasks"): @@ -274,5 +323,8 @@ def batch_command( delay_seconds=delay, verification_level=verification_level, explorer_pro_mode=explorer_pro_mode, + platform=platform, + device_serial=device_serial, + ios_workspace=str(ios_workspace) if ios_workspace else None, ) ) diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 11204934..1ff07c4d 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -390,6 +390,10 @@ def run_command( app_path=app_path, session_id=target_sid, ingress="cli", + platform=platform.value, + ios_workspace=str(ios_workspace) if ios_workspace else None, + verification_level=verification_level, + explorer_mode=explorer_pro_mode, base_url=base_url, ) if resp and resp.get("tasks"): diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 14da4d5c..5a3e962a 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -115,12 +115,41 @@ def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: _CONTROLLERS: dict[str, Any] = {} -def _get_controller(device_serial: str | None = None): - """Lazy-load device controller on-demand, caching per device serial.""" +def _get_controller(device_serial: str | None = None, target_platform: str | None = None): + """Lazy-load device controller on-demand, caching per device serial. + + ``target_platform="ios"`` builds a native iOS Simulator controller: the + serial is a simulator UDID (or "booted"), the context carries + ``DevicePlatform.IOS`` so the factory selects the Xcode driver, and the + cache key is namespaced so a UDID can never collide with an Android + serial. Callers must ``await driver.connect()`` before interacting -- + the native session opens lazily on first use. + """ global _GLOBAL_CONTROLLER, _CONTROLLERS target_serial = ( device_serial or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL") ) + if (target_platform or "").lower() == "ios": + cache_key = f"ios:{target_serial or 'booted'}" + if cache_key in _CONTROLLERS: + return _CONTROLLERS[cache_key] + logger.info("Initializing lazy iOS simulator controller...") + ctx = ArtemisContext( + trace_id="mcp-session", + device=DeviceContext( + host_platform=platform.os_type.name, + mobile_platform=DevicePlatform.IOS, + device_id=target_serial or "booted", + device_width=1206, + device_height=2622, + ), + ) + controller = UnifiedMobileController(ctx=ctx) + _CONTROLLERS[cache_key] = controller + if _GLOBAL_CONTROLLER is None: + _GLOBAL_CONTROLLER = controller + return controller + if target_serial and target_serial in _CONTROLLERS: return _CONTROLLERS[target_serial] if not target_serial and _GLOBAL_CONTROLLER is not None: diff --git a/artemis/runtime/__init__.py b/artemis/runtime/__init__.py index 9821beec..08106288 100644 --- a/artemis/runtime/__init__.py +++ b/artemis/runtime/__init__.py @@ -21,13 +21,16 @@ ) from artemis.runtime.adb_endpoint import ( ADB_ENDPOINT_ID_ENV, + IOS_LOCK_SCOPE, AdbEndpoint, AdbSession, AdbTarget, InvalidAdbEndpoint, + IosTarget, current_adb_endpoint, ) from artemis.runtime.device_pool import DevicePool, DeviceStatus, device_pool +from artemis.runtime.ios_device_pool import IosDevicePool, ios_device_pool from artemis.runtime.process_probe import pid_is_alive from artemis.runtime.daemon_client import ( ensure_daemon_running, @@ -66,6 +69,7 @@ __all__ = [ "ADB_ENDPOINT_ID_ENV", + "IOS_LOCK_SCOPE", "AdbEndpoint", "AdbSession", "AdbTarget", @@ -75,6 +79,8 @@ "DevicePool", "DeviceStatus", "InvalidAdbEndpoint", + "IosDevicePool", + "IosTarget", "clear_cancel_request", "clear_server_info", "current_adb_endpoint", @@ -87,6 +93,7 @@ "get_server_status", "is_artemis_daemon", "is_daemon_running", + "ios_device_pool", "is_cancel_requested", "is_port_in_use", "pid_is_alive", diff --git a/artemis/runtime/adb_endpoint.py b/artemis/runtime/adb_endpoint.py index 9729f96a..986a3c11 100644 --- a/artemis/runtime/adb_endpoint.py +++ b/artemis/runtime/adb_endpoint.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Immutable ADB endpoint and task target primitives. +"""Immutable device endpoint and task target primitives. -The selected endpoint is a user preference. An :class:`AdbTarget` is an -execution snapshot. Keeping those concepts separate prevents a queued or -running task from silently moving to another ADB server when the preference -changes in the Admin Console. +The selected endpoint is a user preference. An :class:`AdbTarget` or +:class:`IosTarget` is an execution snapshot. Keeping those concepts separate +prevents a queued or running task from silently moving to another ADB server +or simulator when the preference changes in the Admin Console. """ from __future__ import annotations @@ -120,6 +120,10 @@ class AdbTarget: endpoint: AdbEndpoint serial: str | None = None + @property + def platform(self) -> str: + return "android" + @property def lock_scope(self) -> str: return self.endpoint.identity @@ -128,8 +132,58 @@ def lock_scope(self) -> str: def lock_key(self) -> str: return f"{self.lock_scope}/{self.serial or 'pending'}" + def apply_to_environment( + self, + environment: MutableMapping[str, str] | None = None, + ) -> MutableMapping[str, str]: + return self.endpoint.apply_to_environment(environment) + + def to_dict(self) -> dict[str, Any]: + return { + "platform": self.platform, + "endpoint": self.endpoint.to_dict(), + "serial": self.serial, + } + + +# Execution lock scope shared by every iOS simulator task. The device UDID +# distinguishes simulators inside the scope; "ios" keeps an iOS lock file +# namespaced away from any Android serial of the same text. +IOS_LOCK_SCOPE = "ios" + + +@dataclass(frozen=True, slots=True) +class IosTarget: + """An iOS simulator UDID bound to the local CoreSimulator service.""" + + serial: str | None = None + + @property + def platform(self) -> str: + return "ios" + + @property + def lock_scope(self) -> str: + return IOS_LOCK_SCOPE + + @property + def lock_key(self) -> str: + return f"{self.lock_scope}/{self.serial or 'pending'}" + + def apply_to_environment( + self, + environment: MutableMapping[str, str] | None = None, + ) -> MutableMapping[str, str]: + target = environment if environment is not None else os.environ + # LOCK_SCOPE_ENV names the execution scope generically: the ADB + # endpoint identity for Android, the platform tag for iOS simulators. + target[ADB_ENDPOINT_ID_ENV] = self.lock_scope + # An iOS worker never touches ADB: a stale serial must not leak in. + target.pop("ADB_DEVICE_SERIAL", None) + return target + def to_dict(self) -> dict[str, Any]: - return {"endpoint": self.endpoint.to_dict(), "serial": self.serial} + return {"platform": self.platform, "serial": self.serial} class AdbSession: diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 85471489..45a17029 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -221,6 +221,8 @@ def submit_task_to_daemon( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, base_url: str | None = None, timeout: float = 15.0, ) -> dict[str, Any] | None: @@ -230,6 +232,9 @@ def submit_task_to_daemon( ``explorer_mode`` ('flash' | 'pro' | 'ultra') are the Pro-profile tuning knobs of ``/api/run``; they are forwarded verbatim and ignored by Flash. + ``platform="ios"`` submits an iOS Simulator task: ``device_serial`` is a + simulator UDID and ``ios_workspace`` optionally scopes Xcode approval. + Returns the response JSON dict if successfully enqueued, or None on error. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" @@ -246,6 +251,8 @@ def submit_task_to_daemon( "session_id": session_id, "ingress": ingress, "conversation_id": conversation_id, + "platform": platform, + "ios_workspace": ios_workspace, } try: @@ -345,13 +352,16 @@ def submit_batch_to_daemon( ingress: str = "cli", verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, base_url: str | None = None, timeout: float = 15.0, ) -> dict[str, Any] | None: """Submit a batch of goals to the running Daemon. ``verification_level`` / ``explorer_mode`` apply to every goal of the batch - (see :func:`submit_task_to_daemon`). + (see :func:`submit_task_to_daemon`). ``platform="ios"`` targets an iOS + Simulator; ``device_serial`` is then the simulator UDID. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" payload = { @@ -361,6 +371,8 @@ def submit_batch_to_daemon( "ingress": ingress, "verification_level": verification_level, "explorer_mode": explorer_mode, + "platform": platform, + "ios_workspace": ios_workspace, } try: data = json.dumps(payload).encode("utf-8") diff --git a/artemis/runtime/device_pool.py b/artemis/runtime/device_pool.py index c67aa538..388fe826 100644 --- a/artemis/runtime/device_pool.py +++ b/artemis/runtime/device_pool.py @@ -37,7 +37,7 @@ class DeviceStatus: """State and lock allocation metadata for a connected device.""" serial: str - state: str # "device", "offline", "unauthorized", etc. + state: str # "device", "offline", "unauthorized", "Shutdown", etc. model: str | None = None product: str | None = None is_emulator: bool = False @@ -46,6 +46,7 @@ class DeviceStatus: active_task_desc: str | None = None active_session_id: str | None = None acquired_at: str | None = None + platform: str = "android" def to_dict(self) -> dict[str, Any]: return { @@ -59,6 +60,7 @@ def to_dict(self) -> dict[str, Any]: "active_task_desc": self.active_task_desc, "active_session_id": self.active_session_id, "acquired_at": self.acquired_at, + "platform": self.platform, } diff --git a/artemis/runtime/ios_device_pool.py b/artemis/runtime/ios_device_pool.py new file mode 100644 index 00000000..eaeb3b3f --- /dev/null +++ b/artemis/runtime/ios_device_pool.py @@ -0,0 +1,151 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator device pool: discovery, lock status, and target selection. + +The iOS counterpart of :class:`~artemis.runtime.device_pool.DevicePool`. +Enumeration rides ``xcrun simctl list devices``; lock ownership shares the +same :class:`DeviceExecutionLock` registry under the ``ios`` scope so a +simulator and an Android serial can never share a lock identity. All methods +fail open on missing Xcode tooling or an enumeration error, mirroring the +Android pool's admission contract. +""" + +from __future__ import annotations + +from artemis.drivers.ios.discovery import list_ios_simulators, list_ios_simulators_sync +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.device_lock import DeviceExecutionLock +from artemis.runtime.device_pool import DeviceStatus +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +# simctl states the driver can use as-is ("device"/Booted) or boot itself +# ("Shutdown"). Anything else (e.g. "Creating") is rejected on explicit picks. +ACCEPTABLE_STATES = frozenset({"device", "Shutdown"}) + + +class IosDevicePool: + """Discovers iOS simulators and reports their Artemis lock state.""" + + @staticmethod + def _build_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: + active_owners = DeviceExecutionLock.get_active_owners() + # Owner map keys are only scope-prefixed for multi-owner collisions, so + # match by owner payload: an iOS lock always carries lock_scope="ios", + # and an Android lock on the same text must never mark a sim busy. + ios_owners = { + DeviceExecutionLock._normalize_device_id(o.device_id): o + for o in active_owners.values() + if o and getattr(o, "lock_scope", None) == IOS_LOCK_SCOPE + } + devices: list[DeviceStatus] = [] + for device in raw_devices: + udid = str(device.get("udid") or "") + runtime = str(device.get("runtime") or "") + state = ( + "device" + if device.get("state") == "Booted" + else str(device.get("state") or "unknown") + ) + owner = ios_owners.get(DeviceExecutionLock._normalize_device_id(udid)) + devices.append( + DeviceStatus( + serial=udid, + state=state, + model=device.get("name"), + product=runtime.removeprefix("com.apple.CoreSimulator.SimRuntime.").replace( + "-", " " + ), + is_emulator=True, + is_busy=owner is not None, + active_pid=owner.pid if owner else None, + active_task_desc=owner.description if owner else None, + active_session_id=owner.session_id if owner else None, + acquired_at=owner.acquired_at if owner else None, + platform="ios", + ) + ) + return devices + + async def list_devices_async(self) -> list[DeviceStatus]: + """All available simulators; an enumeration failure returns an empty list.""" + return self._build_statuses(await list_ios_simulators() or []) + + async def try_list_devices_async(self) -> list[DeviceStatus] | None: + """Like list_devices_async, but ``None`` when simctl could not answer.""" + raw = await list_ios_simulators() + return None if raw is None else self._build_statuses(raw) + + async def validate_explicit_serial_async(self, requested_serial: str) -> str | None: + """Reject an explicitly requested UDID, or ``None`` when usable. + + Mirrors the Android validator: only a successful, non-empty + enumeration may reject. ``Shutdown`` simulators are valid targets — + the driver boots them on connect. + """ + devices = await self.try_list_devices_async() + if not devices: + return None + by_udid = {d.serial.lower(): d for d in devices} + match = by_udid.get(str(requested_serial).lower()) + if match is None: + return ( + f"iOS simulator '{requested_serial}' is not available. " + f"Known simulators: {sorted(d.serial for d in devices)}." + ) + if match.state not in ACCEPTABLE_STATES: + return ( + f"iOS simulator '{requested_serial}' is in state '{match.state}' " + "and cannot be used." + ) + return None + + def validate_explicit_serial(self, requested_serial: str) -> str | None: + """Synchronous validator for non-async admission paths (MCP tools).""" + devices = list_ios_simulators_sync() + if not devices: + return None + statuses = {d["udid"].lower(): d.get("state") for d in devices} + state = statuses.get(str(requested_serial).lower()) + if state is None: + return ( + f"iOS simulator '{requested_serial}' is not available. " + f"Known simulators: {sorted(statuses)}." + ) + if state != "Booted" and state != "Shutdown": + return f"iOS simulator '{requested_serial}' is in state '{state}' and cannot be used." + return None + + async def select_device_async(self, preferred_serial: str | None = None) -> str | None: + """Pick a simulator UDID for task execution. + + An explicit serial wins as-is. Otherwise prefer an idle booted + simulator, then any booted one, then the only available simulator + when none are booted (the driver boots it on connect). ``None`` + when no simulator exists; strict ``booted`` ambiguity rejection is + the driver's job for users who literally target "booted". + """ + if preferred_serial: + return preferred_serial + devices = await self.try_list_devices_async() or [] + booted = [d for d in devices if d.state == "device"] + if len(booted) == 1: + return booted[0].serial + if not booted and len(devices) == 1: + return devices[0].serial + idle_booted = [d for d in booted if not d.is_busy] + return idle_booted[0].serial if idle_booted else (booted[0].serial if booted else None) + + +ios_device_pool = IosDevicePool() diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index 47ea8354..091e4b4d 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -47,6 +47,7 @@ from artemis.data_engine.engine import DataEngine from artemis.graph.state import State from artemis.runtime import DeviceExecutionLock, trace_store +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.runtime.cancel_requests import watch_for_cancel_request from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix from artemis.sdk.types.agent import AgentConfig @@ -207,6 +208,7 @@ async def _ios_operation(self): max_concurrency=getattr(self._config, "max_concurrency", None), session_id=self._session_id, ingress="sdk", + lock_scope=IOS_LOCK_SCOPE, ) queue_cancel_event = threading.Event() acquire_task = asyncio.create_task( diff --git a/docs/ios.md b/docs/ios.md index 9ced6a7a..6dd1b1a9 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -2,9 +2,10 @@ Artemis can run Flash and Pro tasks on iOS simulators on macOS with Xcode 27 or newer. Select iOS explicitly: the existing Android defaults still apply on a -Mac. The first implementation supports the standalone CLI and embedded Python -SDK. The web console, Artemis daemon, remote `artemis-client`, and Artemis MCP -server currently use the Android device path. +Mac. iOS tasks work through the CLI, embedded Python SDK, Artemis Daemon task +queue, Admin Console, and the Artemis MCP tools — including native screen +recording, video analysis/replay, device discovery, live screen streaming, and +per-device locking that cannot collide with Android targets. The iOS driver uses tools included with Xcode and Artemis's existing Python MCP dependency. It communicates with `xcrun mcpbridge` using an initialized MCP @@ -113,10 +114,7 @@ from artemis.sdk import Agent from artemis.sdk.builders import AgentConfigBuilder config = ( - AgentConfigBuilder() - .for_ios_simulator("") - .with_default_profile("flash") - .build() + AgentConfigBuilder().for_ios_simulator("").with_default_profile("flash").build() ) agent = Agent(config=config) ``` @@ -144,6 +142,33 @@ The generic builder also accepts imported from `artemis.context`. Supplying only a `device_serial` without an iOS configuration retains the existing Android selection behavior. +## Daemon, web console, and batch submission + +Without `--standalone`, `artemis run --platform ios` forwards +`platform`/`ios_workspace`/`device_serial` to the running Artemis Daemon, which +queues the task against the simulator under the shared `ios` lock scope and +spawns the worker with `--platform ios` — no ADB endpoint or Android readiness +probe is involved. `artemis batch` accepts the same `--platform`, +`--device-serial`, and `--ios-workspace` flags for goal lists. + +The Admin Console `/api/run` accepts `platform: "ios"`, a simulator UDID in +`device_serial`, and an optional `ios_workspace`; `/api/devices` lists Android +devices and iOS simulators together, each tagged with its `platform`. The live +screen view (`/api/stream/device-live`) streams simulator frames captured with +`simctl io screenshot` when an iOS task holds the lock or a simulator is the +only viable target; `/api/stream/device-state` reports the `platform` of the +streamed device. Replay preserves the recorded session's `mobile_platform`, so +an iOS trace replays through the Xcode driver on a simulator (the device +picker retargets iOS replays to a chosen UDID). + +`mobile_run_task` accepts `platform="ios"`, `device_serial=`, and +`ios_workspace=`; it validates the UDID against `simctl` rather than ADB +and queues the runner under the `ios` lock scope. `mobile_get_device_state` +and `mobile_diagnose` accept the same `platform` switch — the latter runs a +native screenshot/hierarchy smoke test on the simulator. The legacy +`Android_ADB_Controller` actuator server (tap/swipe/type tools) remains +Android-only. + A minimal run looks like: ```python @@ -224,8 +249,9 @@ recorded frames. inventory is empty. - **Multiple booted simulators:** supply `--device-serial` with the intended simulator's UDID. -- **Android daemon or device checks appear:** include `--platform ios - --standalone`; iOS tasks currently run in the embedded process. +- **A queued iOS task runs on the wrong surface:** confirm the submission + carried `platform: "ios"` (CLI `--platform ios`, web request `platform`, + or the MCP `platform` argument); tasks default to Android. ## Contributing and validation diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index a6f377ce..bcdc11d5 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -121,6 +121,8 @@ async def run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_pro_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ): """Executes the mobile automation agent task and logs all actions/results. @@ -154,7 +156,8 @@ async def run_task( print("--------------------------------------------------") agent = None - adb_path = device_utils.resolve_adb_path() + is_ios = platform.lower() == "ios" + adb_path = None if is_ios else device_utils.resolve_adb_path() target_serial = device_serial try: @@ -168,9 +171,14 @@ async def run_task( from artemis.sdk.builders import Builders from artemis.sdk.types import AgentProfile - connected_devices = device_utils.get_connected_devices(adb_path) + connected_devices = [] if is_ios else device_utils.get_connected_devices(adb_path) - if device_serial: + if is_ios: + # The iOS driver resolves "booted" against `simctl` at connect + # time; an explicit UDID was already validated at admission. + target_serial = device_serial or "booted" + print(f"✅ Targeting iOS simulator: '{target_serial}'.") + elif device_serial: target_serial = device_serial if connected_devices and device_serial not in connected_devices: print( @@ -221,7 +229,12 @@ async def run_task( if settings.ADB_HOST: config_builder.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) - if target_serial: + if is_ios: + config_builder.for_ios_simulator( + device_id=target_serial or "booted", + workspace_path=ios_workspace, + ) + elif target_serial: from artemis.context import DevicePlatform config_builder.for_device(DevicePlatform.ANDROID, target_serial) @@ -434,6 +447,16 @@ async def run_task( "--explorer-pro-mode", help="Pro-profile Explorer perception version: 'flash', 'pro' or 'ultra'", ) + parser.add_argument( + "--platform", + default="android", + choices=["android", "ios"], + help="Target platform ('android' default; 'ios' targets a Simulator via Xcode)", + ) + parser.add_argument( + "--ios-workspace", + help="Xcode project/workspace path for first-run iOS approval (iOS only)", + ) args = parser.parse_args() @@ -449,5 +472,7 @@ async def run_task( device_serial=args.device_serial, verification_level=args.verification_level, explorer_pro_mode=args.explorer_pro_mode, + platform=args.platform, + ios_workspace=args.ios_workspace, ) ) diff --git a/mcp_server/tools/device_state.py b/mcp_server/tools/device_state.py index 76b682bf..0df3c64f 100644 --- a/mcp_server/tools/device_state.py +++ b/mcp_server/tools/device_state.py @@ -31,7 +31,11 @@ @mcp.tool() -async def mobile_get_device_state(view_type: str, device_serial: str | None = None) -> str: +async def mobile_get_device_state( + view_type: str, + device_serial: str | None = None, + platform: str | None = None, +) -> str: """Real-time mobile device state observer (for debugging and validation). Retrieves a real-time screenshot or a simplified UI element tree from the @@ -44,20 +48,32 @@ async def mobile_get_device_state(view_type: str, device_serial: str | None = No returns the image's local file URI. - "hierarchy": returns the simplified text-labeled element list — exactly what the automation subagent sees when making decisions. - device_serial: Optional device serial (e.g. "emulator-5554") to inspect - a specific device; omitted → the default connected device. With - several devices attached, confirm the target with the user - (`adb devices -l` lists serials). + device_serial: Optional device serial (e.g. "emulator-5554", or an iOS + simulator UDID with platform="ios") to inspect a specific device; + omitted → the default connected device. + platform: "android" (default) or "ios". For iOS, device_serial is a + simulator UDID; omit it to use the single booted simulator. """ try: - controller = _get_controller(device_serial=device_serial) - device_width = controller.ctx.device.device_width - device_height = controller.ctx.device.device_height + controller = _get_controller( + device_serial=device_serial, + **({"target_platform": platform} if platform else {}), + ) + is_ios = str(getattr(controller.ctx.device, "mobile_platform", "android")).lower() == "ios" except Exception as e: - return f"Error: Failed to initialize/lock Android device controller: {e}" + return f"Error: Failed to initialize/lock device controller: {e}" try: + if is_ios: + # The native Xcode session opens lazily; connect before observing. + await controller._driver.connect() device_data = await controller.get_screen_data() + if is_ios: + # Real capture dims supersede the provisional ctx placeholders. + controller.ctx.device.device_width = device_data.width + controller.ctx.device.device_height = device_data.height + device_width = controller.ctx.device.device_width + device_height = controller.ctx.device.device_height latest_screenshot_b64 = device_data.base64 xml_hierarchy = device_data.elements @@ -69,7 +85,8 @@ async def mobile_get_device_state(view_type: str, device_serial: str | None = No safe_device_id = "".join( [c if c.isalnum() or c in ("-", "_") else "_" for c in device_id] ) - screenshot_filename = f"live_screenshot_{safe_device_id}.jpg" + ext = "png" if is_ios else "jpg" + screenshot_filename = f"live_screenshot_{safe_device_id}.{ext}" screenshot_path = os.path.join(project_root, screenshot_filename) with open(screenshot_path, "wb") as f: diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index 4f336463..cd7197ba 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -467,14 +467,37 @@ def _probe_unavailable(serial: str | None, error: str) -> dict[str, Any]: } -async def _device_smoke_test(device_serial: str | None) -> dict[str, Any]: +async def _device_smoke_test( + device_serial: str | None, platform: str = "android" +) -> dict[str, Any]: """Drive the device end to end (screenshot + UI hierarchy) through the shared smoke test.""" - return await smoke_test_device(device_serial) + return await smoke_test_device(device_serial, platform=platform) async def _run_device_probe( - adb_result: ProbeResult | None, requested_device: str | None + adb_result: ProbeResult | None, + requested_device: str | None, + ios_result: ProbeResult | None = None, + platform: str = "android", ) -> dict[str, Any]: + if platform == "ios": + simulators = (ios_result.metadata.get("simulators") if ios_result else None) or [] + ready_udids = [str(d.get("udid")) for d in simulators if d.get("udid")] + if requested_device and requested_device not in ready_udids: + return _probe_unavailable( + requested_device, + f"requested iOS simulator '{requested_device}' is not available; nothing to probe", + ) + if not ready_udids: + return _probe_unavailable(None, "no available iOS simulator found; nothing to probe") + serial = requested_device or (ready_udids[0] if len(ready_udids) == 1 else None) + try: + return await _device_smoke_test(serial, platform="ios") + except Exception as exc: + return _probe_unavailable( + serial, f"iOS smoke test raised {exc.__class__.__name__}: {exc}" + ) + devices = (adb_result.metadata.get("devices") if adb_result else None) or [] ready = [str(d.get("serial")) for d in devices if d.get("state") == "device"] if requested_device and requested_device not in ready: @@ -732,6 +755,7 @@ def _next_steps( tasks: dict[str, list[dict[str, Any]]], device_probe: dict[str, Any] | None, accessibility_helper: dict[str, Any] | None = None, + platform: str = "android", ) -> list[str]: steps: list[str] = [] needs_restart = False @@ -773,16 +797,20 @@ def _next_steps( steps.extend(credential_lines) needs_restart = needs_restart or credential_restart - steps.extend( - _device_steps( - adb_result, - attempt_fix=attempt_fix, - requested_device=requested_device, - emulator=emulator, - launch_requested=bool(launch_steps), - tasks=tasks, + if platform != "ios": + # ADB-specific guidance (busy devices, AVD hints, authorization) does + # not apply to simulator targets; the ios_simulators probe's own + # steps cover iOS findings. + steps.extend( + _device_steps( + adb_result, + attempt_fix=attempt_fix, + requested_device=requested_device, + emulator=emulator, + launch_requested=bool(launch_steps), + tasks=tasks, + ) ) - ) steps.extend(launch_steps) steps.extend(_device_probe_steps(device_probe)) steps.extend(_helper_steps(accessibility_helper, attempt_fix=attempt_fix)) @@ -975,19 +1003,24 @@ async def _run_extras( launch_avd: str | None, verify_credentials: bool, probe_device: bool, + platform: str = "android", ) -> dict[str, Any]: """Optional, slower work: emulator launch, live key checks, device smoke test, lock state.""" adb_result = _find(results, "android_adb") + ios_result = _find(results, "ios_simulators") launch_steps: list[str] = [] if launch_avd: emulator, launch_steps = await _handle_launch_avd(launch_avd, adb_result) else: emulator = _emulator_status() - helper_serial = _helper_target(adb_result, requested_device) + # The on-device Accessibility Helper is Android-only; iOS observes natively. + helper_serial = _helper_target(adb_result, requested_device) if platform != "ios" else None credentials, device_probe, accessibility_helper = await asyncio.gather( _verify_credentials(_find(results, "gemini_api_key")) if verify_credentials else _none(), - _run_device_probe(adb_result, requested_device) if probe_device else _none(), + _run_device_probe(adb_result, requested_device, ios_result, platform) + if probe_device + else _none(), asyncio.to_thread(_helper_status, helper_serial) if helper_serial else _none(), ) return { @@ -1000,10 +1033,23 @@ async def _run_extras( } -def _requested_device_ready(results: list[ProbeResult], requested_device: str | None) -> bool: - """A caller-named device counts as a blocker: attached and authorized, or not ready.""" +def _requested_device_ready( + results: list[ProbeResult], + requested_device: str | None, + platform: str = "android", +) -> bool: + """A caller-named device counts as a blocker: attached and usable, or not ready.""" if not requested_device: return True + if platform == "ios": + ios_result = _find(results, "ios_simulators") + if ios_result is None: + return False + simulators = ios_result.metadata.get("simulators") or [] + return any( + d.get("udid") == requested_device and d.get("state") in ("Booted", "Shutdown") + for d in simulators + ) adb_result = _find(results, "android_adb") if adb_result is None: return False @@ -1018,9 +1064,19 @@ def _verdict( credentials: list[dict[str, Any]] | None, device_probe: dict[str, Any] | None, accessibility_helper: dict[str, Any] | None = None, + platform: str = "android", ) -> str: - verdict = base_verdict(results) - if verdict == "blocked" or not _requested_device_ready(results, requested_device): + if platform == "ios": + # iOS tasks never touch ADB: the iOS simulator probe takes the device + # gate and android_adb drops out of the blocker set for this call. + ios_result = _find(results, "ios_simulators") + adjusted = [r for r in results if r.id != "android_adb"] + verdict = base_verdict(adjusted) + if verdict == "blocked" or ios_result is None or ios_result.status is not ProbeStatus.PASS: + return "blocked" + else: + verdict = base_verdict(results) + if verdict == "blocked" or not _requested_device_ready(results, requested_device, platform): return "blocked" primary = _primary_credential(credentials) if primary is not None and not primary.get("valid"): @@ -1096,6 +1152,7 @@ async def mobile_diagnose( launch_avd: str | None = None, verify_credentials: bool = False, probe_device: bool = False, + platform: str | None = None, ) -> dict[str, Any]: """Diagnoses why ARTEMIS cannot run tasks from this IDE and returns the fixes. @@ -1177,10 +1234,16 @@ async def mobile_diagnose( really start. Use it when the checks pass but tasks still fail on the device, or the screen stays black. A failed probe makes the verdict "blocked" and lists the fix. + platform: 'android' (default) or 'ios'. With 'ios', `device_serial` + is a simulator UDID and `probe_device` exercises the native Xcode + path; ADB-only extras (AVD launch, accessibility helper) are skipped. """ fixes_applied: list[dict[str, Any]] = [] + platform_name = (platform or "android").strip().lower() requested_device = device_serial.strip() if device_serial and device_serial.strip() else None - avd_name = launch_avd.strip() if launch_avd and launch_avd.strip() else None + avd_name = ( + launch_avd.strip() if launch_avd and launch_avd.strip() and platform_name != "ios" else None + ) try: report, host = await asyncio.wait_for( collect_readiness(), timeout=DIAGNOSIS_TIMEOUT_SECONDS @@ -1201,6 +1264,7 @@ async def mobile_diagnose( launch_avd=avd_name, verify_credentials=verify_credentials, probe_device=probe_device, + platform=platform_name, ), timeout=DIAGNOSIS_TIMEOUT_SECONDS, ) @@ -1216,6 +1280,7 @@ async def mobile_diagnose( credentials=credentials, device_probe=device_probe, accessibility_helper=accessibility_helper, + platform=platform_name, ) device = _compact_device(report) if device is not None and accessibility_helper is not None: @@ -1236,6 +1301,7 @@ async def mobile_diagnose( tasks=extras["tasks"], device_probe=device_probe, accessibility_helper=accessibility_helper, + platform=platform_name, ), "checks": [_render_check(r) for r in sort_by_fix_order(results)], "host": _compact_host(host.metadata), diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index d30c42eb..23e21b5a 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -32,6 +32,7 @@ DeviceExecutionLock, device_pool, ensure_daemon_running, + ios_device_pool, submit_task_to_daemon, trace_store, ) @@ -144,17 +145,22 @@ def _start_spawn_watchdog( ).start() -def _validate_device_serial(device_serial: str) -> dict[str, Any] | None: - """Reject a task whose explicitly requested device is not attached and authorized. +def _validate_device_serial(device_serial: str, platform: str = "android") -> dict[str, Any] | None: + """Reject a task whose explicitly requested device is not attached and usable. Returns a failure response dict when the serial must be rejected, or None when - the device is usable. If device enumeration itself fails (no adb available), + the device is usable. If device enumeration itself fails (no adb/xcrun), validation is skipped rather than blocking task submission. """ try: - # The shared validator fails open on an indeterminate/empty enumeration: + # The shared validators fail open on an indeterminate/empty enumeration: # the task proceeds and fails downstream with a clear no-device error. - detail = device_pool.validate_explicit_serial(device_serial) + if platform == "ios": + detail = ios_device_pool.validate_explicit_serial(device_serial) + inspect_hint = "Run `xcrun simctl list devices` to inspect simulators" + else: + detail = device_pool.validate_explicit_serial(device_serial) + inspect_hint = "Run `adb devices -l` to inspect attached hardware" except Exception: return None if detail is None: @@ -164,8 +170,8 @@ def _validate_device_serial(device_serial: str) -> dict[str, Any] | None: "error": detail, "message": ( f"{detail} Task rejected to prevent execution on an unintended device. " - "Run `adb devices -l` to inspect attached hardware, then resubmit with a " - "valid serial (or omit device_serial for automatic selection)." + f"{inspect_hint}, then resubmit with a " + "valid identifier (or omit device_serial for automatic selection)." ), } @@ -212,8 +218,10 @@ def mobile_run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ) -> dict[str, Any]: - """Starts an autonomous mobile UI automation subagent on a connected Android device. + """Starts an autonomous mobile UI automation subagent on a connected mobile device. Delegates a mobile workflow to a background agent. Non-blocking: returns immediately with `trace_id` (for `mobile_manage_task` / `mobile_inspect_trace`), @@ -282,6 +290,17 @@ def mobile_run_task( if model.lower() not in ("flash", "pro"): raise ValueError(f"Invalid model '{model}'. Must be either 'Flash' or 'Pro'.") canonical_model = "Flash" if model.lower() == "flash" else "Pro" + # 0a. Validate and normalize platform: 'android' (ADB) or 'ios' (Xcode Simulator). + platform = (platform or "android").strip().lower() + if platform not in ("android", "ios"): + raise ValueError(f"Invalid platform '{platform}'. Must be 'android' or 'ios'.") + if platform == "ios" and locked_app_package: + return { + "trace_id": None, + "status": "failed", + "error": "locked_app_package is Android-only; iOS tasks cannot lock to an app package.", + "message": "Task rejected: 'locked_app_package' requires the Android platform.", + } # 0b. Validate the Pro tuning knobs before any trace exists so a typo is a # plain tool error rather than a failed trace on disk. verification_level, explorer_mode = _normalize_pro_tuning(verification_level, explorer_mode) @@ -303,7 +322,7 @@ def mobile_run_task( # different device than the caller asked for. Runs after init_trace so the # rejection carries a trace_id like every other response of this tool. if device_serial: - rejection = _validate_device_serial(device_serial) + rejection = _validate_device_serial(device_serial, platform) if rejection: trace_store.update_trace_status(trace_id, "failed", error=rejection["error"]) return {"trace_id": trace_id, **rejection} @@ -325,6 +344,8 @@ def mobile_run_task( conversation_id=conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, base_url=base_url, ) if resp and resp.get("status") == "rejected": @@ -442,6 +463,10 @@ def mobile_run_task( } if device_serial: reserve_kwargs["device_id"] = device_serial + if platform == "ios": + # iOS workers queue under the ios__ scope so they never collide + # with an Android device sharing the same textual identifier. + reserve_kwargs["lock_scope"] = "ios" queue_ticket = DeviceExecutionLock.reserve(**reserve_kwargs) # 5. Spawn the background task runner as an independent subprocess @@ -458,6 +483,8 @@ def mobile_run_task( canonical_model, "--conversation-id", conversation_id or "", + "--platform", + platform, ] if locked_app_package: cmd.extend(["--locked-app-package", locked_app_package]) @@ -467,6 +494,8 @@ def mobile_run_task( cmd.extend(["--expected-output-desc", expected_output_desc]) if device_serial: cmd.extend(["--device-serial", device_serial]) + if platform == "ios" and ios_workspace: + cmd.extend(["--ios-workspace", ios_workspace]) if verification_level: cmd.extend(["--verification-level", verification_level]) if explorer_mode: @@ -476,8 +505,11 @@ def mobile_run_task( env["ARTEMIS_SESSION_ID"] = trace_id env["ARTEMIS_TASK_INGRESS"] = "mcp" if device_serial: - env["ADB_DEVICE_SERIAL"] = device_serial - env["ARTEMIS_DEVICE_ID"] = device_serial + if platform == "ios": + env["ARTEMIS_DEVICE_ID"] = device_serial + else: + env["ADB_DEVICE_SERIAL"] = device_serial + env["ARTEMIS_DEVICE_ID"] = device_serial env[DeviceExecutionLock.QUEUE_TICKET_ENV] = queue_ticket try: ipc_port = read_ipc_port() @@ -499,6 +531,8 @@ def mobile_run_task( "session_id": trace_id, "ingress": "mcp", } + if platform == "ios": + transfer_kwargs["lock_scope"] = "ios" if device_serial: transfer_kwargs["device_id"] = device_serial diff --git a/tests/unit/core/test_readiness.py b/tests/unit/core/test_readiness.py index 0cfb0c77..37c93037 100644 --- a/tests/unit/core/test_readiness.py +++ b/tests/unit/core/test_readiness.py @@ -131,6 +131,7 @@ def test_sort_by_fix_order_follows_check_order_and_appends_unknown_ids(): "integration_host", "gemini_api_key", "android_adb", + "ios_simulators", "toolchain", "vision_ocr_key", ] diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py new file mode 100644 index 00000000..ad18ced4 --- /dev/null +++ b/tests/unit/test_ios_parity.py @@ -0,0 +1,727 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator platform-parity tests: discovery, pool, locks, queue, routing. + +Every test is hermetic — ``simctl``/``xcrun``/adb are never invoked; device +enumeration is stubbed at the discovery boundary. +""" + +import asyncio +import os +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.drivers.ios.discovery import ( + list_ios_simulators, + list_ios_simulators_sync, + parse_simctl_devices, +) +from artemis.runtime.adb_endpoint import AdbEndpoint, AdbTarget, IOS_LOCK_SCOPE, IosTarget +from artemis.runtime.device_lock import DeviceExecutionLock, DeviceLockOwner +from artemis.runtime.ios_device_pool import IosDevicePool + + +@pytest.fixture(autouse=True) +def isolated_lock_directory(tmp_path, monkeypatch): + monkeypatch.setattr( + "artemis.runtime.device_lock.get_temp_dir", + lambda _name: tmp_path, + ) + + +SIMCTL_PAYLOAD = { + "devices": { + "com.apple.CoreSimulator.SimRuntime.iOS-26-0": [ + { + "udid": "AAAA-1111", + "name": "iPhone 17 Pro", + "state": "Booted", + "isAvailable": True, + }, + { + "udid": "BBBB-2222", + "name": "iPhone 17", + "state": "Shutdown", + "isAvailable": True, + }, + { + "udid": "CCCC-3333", + "name": "iPad Unavailable", + "state": "Shutdown", + "isAvailable": False, + }, + ], + "com.apple.CoreSimulator.SimRuntime.watchOS-26-0": [ + { + "udid": "DDDD-4444", + "name": "Watch", + "state": "Shutdown", + "isAvailable": True, + } + ], + "com.apple.CoreSimulator.SimRuntime.iOS-26-1": [ + { + "udid": "", + "name": "No UDID", + "state": "Shutdown", + "isAvailable": True, + } + ], + } +} + +SIM_LIST = parse_simctl_devices(SIMCTL_PAYLOAD) + + +# --------------------------------------------------------------------------- # +# discovery +# --------------------------------------------------------------------------- # + + +def test_parse_simctl_devices_filters_ios_available_only(): + devices = parse_simctl_devices(SIMCTL_PAYLOAD) + udids = {d["udid"] for d in devices} + assert udids == {"AAAA-1111", "BBBB-2222"} + by_udid = {d["udid"]: d for d in devices} + assert by_udid["AAAA-1111"]["state"] == "Booted" + assert by_udid["AAAA-1111"]["name"] == "iPhone 17 Pro" + assert "iOS-26-0" in by_udid["BBBB-2222"]["runtime"] + + +def test_parse_simctl_devices_empty_payload(): + assert parse_simctl_devices({}) == [] + assert parse_simctl_devices({"devices": {}}) == [] + + +@pytest.mark.asyncio +async def test_list_ios_simulators_returns_none_on_failure(monkeypatch): + monkeypatch.setattr( + "artemis.drivers.ios.discovery.run_xcrun", + AsyncMock(side_effect=RuntimeError("no xcode")), + ) + assert await list_ios_simulators() is None + + +@pytest.mark.asyncio +async def test_list_ios_simulators_parses_json(monkeypatch): + import json + + monkeypatch.setattr( + "artemis.drivers.ios.discovery.run_xcrun", + AsyncMock(return_value=json.dumps(SIMCTL_PAYLOAD).encode()), + ) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + devices = await list_ios_simulators() + assert {d["udid"] for d in devices} == {"AAAA-1111", "BBBB-2222"} + + +def test_list_ios_simulators_sync_skips_without_simctl(monkeypatch): + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: False) + assert list_ios_simulators_sync() is None + + +def test_list_ios_simulators_sync_parses_json(monkeypatch): + import json + import subprocess + + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps(SIMCTL_PAYLOAD).encode() + ) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + monkeypatch.setattr("artemis.drivers.ios.discovery.subprocess.run", lambda *a, **k: completed) + devices = list_ios_simulators_sync() + assert {d["udid"] for d in devices} == {"AAAA-1111", "BBBB-2222"} + + +# --------------------------------------------------------------------------- # +# IosTarget / lock scope +# --------------------------------------------------------------------------- # + + +def test_ios_target_lock_scope_and_key(): + target = IosTarget(serial="AAAA-1111") + assert target.platform == "ios" + assert target.lock_scope == IOS_LOCK_SCOPE + assert target.lock_key == f"{IOS_LOCK_SCOPE}/AAAA-1111" + assert target.to_dict() == {"platform": "ios", "serial": "AAAA-1111"} + + +def test_ios_target_apply_to_environment_sets_scope_and_scrubs_adb(monkeypatch): + env = {"ADB_DEVICE_SERIAL": "emulator-5554", "ADB_HOST": "10.0.0.1"} + IosTarget(serial="AAAA-1111").apply_to_environment(env) + assert env[DeviceExecutionLock.LOCK_SCOPE_ENV] == IOS_LOCK_SCOPE + # iOS workers never touch ADB: a stale serial must not leak in. + assert "ADB_DEVICE_SERIAL" not in env + + +def test_ios_and_android_locks_do_not_collide(): + """A UDID and an ADB serial with identical text lock independently.""" + shared_id = "emulator-5554" + ios_lock = DeviceExecutionLock(shared_id, "ios task", lock_scope=IOS_LOCK_SCOPE) + android_lock = DeviceExecutionLock(shared_id, "android task") + assert ios_lock.clean_device_id != android_lock.clean_device_id + + ios_lock.acquire() + try: + owners = list(DeviceExecutionLock.get_active_owners().values()) + ios_owner = next(o for o in owners if o.device_id == shared_id) + assert ios_owner.lock_scope == IOS_LOCK_SCOPE + # Android lock for the same text is unaffected and acquirable. + android_lock.acquire(blocking=False) + try: + owners = list(DeviceExecutionLock.get_active_owners().values()) + by_scope = {o.lock_scope: o for o in owners if o.device_id == shared_id} + assert IOS_LOCK_SCOPE in by_scope + assert len(by_scope) == 2 + finally: + android_lock.release() + finally: + ios_lock.release() + + +def test_get_active_owner_scoped_lookup_isolates_platform(monkeypatch): + monkeypatch.delenv(DeviceExecutionLock.LOCK_SCOPE_ENV, raising=False) + shared_id = "emulator-5554" + ios_lock = DeviceExecutionLock(shared_id, "ios task", lock_scope=IOS_LOCK_SCOPE) + ios_lock.acquire() + try: + owner = DeviceExecutionLock.get_active_owner(shared_id, lock_scope=IOS_LOCK_SCOPE) + assert owner is not None and owner.description == "ios task" + # An unscoped (Android) lookup must not see the iOS owner. + assert DeviceExecutionLock.get_active_owner(shared_id, lock_scope=None) is None + finally: + ios_lock.release() + + +# --------------------------------------------------------------------------- # +# IosDevicePool +# --------------------------------------------------------------------------- # + + +def _pool_with_devices(monkeypatch, devices=None): + import importlib + + pool = IosDevicePool() + devices = SIM_LIST if devices is None else devices + # artemis.runtime.ios_device_pool resolves to the singleton instance in the + # package namespace; patch the module object itself. + module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(devices))) + monkeypatch.setattr(module, "list_ios_simulators_sync", lambda: list(devices)) + return pool + + +@pytest.mark.asyncio +async def test_ios_pool_statuses_carry_platform_and_runtime(monkeypatch): + pool = _pool_with_devices(monkeypatch) + devices = await pool.list_devices_async() + by_serial = {d.serial: d for d in devices} + booted = by_serial["AAAA-1111"] + assert booted.platform == "ios" + assert booted.state == "device" # Booted maps to the generic ready state + assert booted.is_emulator is True + assert "iOS 26 0" in booted.product + shutdown = by_serial["BBBB-2222"] + assert shutdown.state == "Shutdown" + + +@pytest.mark.asyncio +async def test_ios_pool_busy_status_uses_ios_lock_scope(monkeypatch): + pool = _pool_with_devices(monkeypatch) + lock = DeviceExecutionLock("AAAA-1111", "ios task", lock_scope=IOS_LOCK_SCOPE) + lock.acquire() + try: + devices = await pool.list_devices_async() + by_serial = {d.serial: d for d in devices} + assert by_serial["AAAA-1111"].is_busy is True + assert by_serial["AAAA-1111"].active_task_desc == "ios task" + assert by_serial["BBBB-2222"].is_busy is False + finally: + lock.release() + + +@pytest.mark.asyncio +async def test_ios_pool_android_lock_does_not_mark_ios_busy(monkeypatch): + """An Android lock on the same text must not mark the simulator busy.""" + pool = _pool_with_devices(monkeypatch) + lock = DeviceExecutionLock("AAAA-1111", "android task") + lock.acquire() + try: + devices = await pool.list_devices_async() + by_serial = {d.serial: d for d in devices} + assert by_serial["AAAA-1111"].is_busy is False + finally: + lock.release() + + +@pytest.mark.asyncio +async def test_ios_pool_validate_explicit_serial(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert await pool.validate_explicit_serial_async("AAAA-1111") is None + assert await pool.validate_explicit_serial_async("bbbb-2222") is None + rejection = await pool.validate_explicit_serial_async("NOPE-9999") + assert rejection is not None and "not available" in rejection + # Shutdown sims are valid; unavailable devices were filtered at parse time. + assert await pool.validate_explicit_serial_async("CCCC-3333") is not None + + +@pytest.mark.asyncio +async def test_ios_pool_validate_fails_open_on_enumeration_error(monkeypatch): + pool = IosDevicePool() + import importlib + + module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=None)) + assert await pool.validate_explicit_serial_async("ANY") is None + + +def test_ios_pool_validate_explicit_serial_sync(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert pool.validate_explicit_serial("AAAA-1111") is None + rejection = pool.validate_explicit_serial("NOPE-9999") + assert rejection is not None and "not available" in rejection + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_prefers_booted_and_idle(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert await pool.select_device_async() == "AAAA-1111" + # Explicit preference always wins. + assert await pool.select_device_async(preferred_serial="BBBB-2222") == "BBBB-2222" + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_boots_single_shutdown_sim(monkeypatch): + pool = _pool_with_devices(monkeypatch, [d for d in SIM_LIST if d["udid"] == "BBBB-2222"]) + assert await pool.select_device_async() == "BBBB-2222" + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_skips_busy_booted(monkeypatch): + two_booted = [ + {**SIM_LIST[0]}, + {**SIM_LIST[1], "state": "Booted"}, + ] + pool = _pool_with_devices(monkeypatch, two_booted) + lock = DeviceExecutionLock("AAAA-1111", "ios task", lock_scope=IOS_LOCK_SCOPE) + lock.acquire() + try: + assert await pool.select_device_async() == "BBBB-2222" + finally: + lock.release() + + +# --------------------------------------------------------------------------- # +# Task queue routing +# --------------------------------------------------------------------------- # + + +def test_task_target_routes_ios_to_ios_target(): + from apps.admin_console.services.task_queue_service import TaskQueueService + + target = TaskQueueService._task_target({"platform": "ios", "device_serial": "AAAA-1111"}) + assert isinstance(target, IosTarget) + assert target.serial == "AAAA-1111" + + +def test_task_target_defaults_to_android_adb_target(): + from apps.admin_console.services.task_queue_service import TaskQueueService + + target = TaskQueueService._task_target({"device_serial": "emulator-5554"}) + assert isinstance(target, AdbTarget) + assert target.serial == "emulator-5554" + + +def test_build_worker_invocation_ios(monkeypatch): + from apps.admin_console.services.task_queue_service import TaskQueueService + + monkeypatch.delenv("ADB_DEVICE_SERIAL", raising=False) + cmd, env = TaskQueueService._build_worker_invocation( + { + "platform": "ios", + "device_serial": "AAAA-1111", + "ios_workspace": "/proj/App.xcodeproj", + }, + run_key="rk", + sess_id="sess-1", + goal="open settings", + profile="flash", + target=IosTarget(serial="AAAA-1111"), + ) + assert "--platform" in cmd + assert cmd[cmd.index("--platform") + 1] == "ios" + assert cmd[cmd.index("--device-serial") + 1] == "AAAA-1111" + assert cmd[cmd.index("--ios-workspace") + 1] == "/proj/App.xcodeproj" + assert env[DeviceExecutionLock.LOCK_SCOPE_ENV] == IOS_LOCK_SCOPE + assert "ADB_DEVICE_SERIAL" not in env + + +def test_build_worker_invocation_android_unchanged(): + from apps.admin_console.services.task_queue_service import TaskQueueService + + endpoint = AdbEndpoint.local() + target = AdbTarget(endpoint=endpoint, serial="emulator-5554") + cmd, env = TaskQueueService._build_worker_invocation( + {"device_serial": "emulator-5554"}, + run_key="rk", + sess_id="sess-1", + goal="open settings", + profile="flash", + target=target, + ) + assert "--platform" not in cmd + assert cmd[cmd.index("--device-serial") + 1] == "emulator-5554" + assert env["ADB_DEVICE_SERIAL"] == "emulator-5554" + assert env[DeviceExecutionLock.LOCK_SCOPE_ENV] == endpoint.identity + + +@pytest.mark.asyncio +async def test_enqueue_tasks_ios_carries_platform_and_lock_scope(monkeypatch): + from apps.admin_console.services.task_queue_service import TaskQueueService + from apps.admin_console.core.state import state + + monkeypatch.setattr(TaskQueueService, "ensure_worker_running", staticmethod(lambda: None)) + monkeypatch.setattr( + TaskQueueService, + "_find_duplicate_submission", + classmethod(lambda cls, *a, **k: None), + ) + monkeypatch.setattr( + IosDevicePool, + "validate_explicit_serial_async", + AsyncMock(return_value=None), + ) + state.queue_items.clear() + try: + resp = await TaskQueueService.enqueue_tasks( + ["open settings"], + "flash", + None, + None, + None, + None, + "AAAA-1111", + "frontend", + session_id="sess-1", + conversation_id=None, + platform="ios", + ios_workspace="/proj/App.xcodeproj", + ) + item = state.queue_items[-1] + assert item["platform"] == "ios" + assert item["ios_workspace"] == "/proj/App.xcodeproj" + assert item["device_serial"] == "AAAA-1111" + # The queue reservation was scoped ios — check via the target builder. + target = TaskQueueService._task_target(item) + assert target.lock_scope == IOS_LOCK_SCOPE + assert not resp.get("rejected") + finally: + state.queue_items.clear() + + +# --------------------------------------------------------------------------- # +# Stream service target resolution +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_stream_target_prefers_active_ios_owner(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + owner = DeviceLockOwner( + pid=os.getpid(), + process_created_at=0.0, + token="t", + device_id="AAAA-1111", + description="ios task", + acquired_at="now", + lock_scope=IOS_LOCK_SCOPE, + ) + monkeypatch.setattr( + DeviceExecutionLock, + "get_active_owners", + classmethod(lambda cls: {"ios__AAAA-1111": owner}), + ) + service = DeviceStreamService() + target = await service.get_stream_target() + assert target == {"platform": "ios", "serial": "AAAA-1111"} + + +@pytest.mark.asyncio +async def test_stream_target_falls_back_to_android(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {})) + service = DeviceStreamService() + monkeypatch.setattr(service, "_android_serial", AsyncMock(return_value="emulator-5554")) + monkeypatch.setattr(IosDevicePool, "select_device_async", AsyncMock(return_value=None)) + target = await service.get_stream_target() + assert target == {"platform": "android", "serial": "emulator-5554"} + + +@pytest.mark.asyncio +async def test_stream_target_falls_back_to_booted_sim(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {})) + service = DeviceStreamService() + monkeypatch.setattr(service, "_android_serial", AsyncMock(return_value=None)) + monkeypatch.setattr(IosDevicePool, "select_device_async", AsyncMock(return_value="AAAA-1111")) + target = await service.get_stream_target() + assert target == {"platform": "ios", "serial": "AAAA-1111"} + + +@pytest.mark.asyncio +async def test_stream_target_none_without_devices(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {})) + service = DeviceStreamService() + monkeypatch.setattr(service, "_android_serial", AsyncMock(return_value=None)) + monkeypatch.setattr(IosDevicePool, "select_device_async", AsyncMock(return_value=None)) + assert await service.get_stream_target() is None + + +# --------------------------------------------------------------------------- # +# MCP task runner platform routing +# --------------------------------------------------------------------------- # + + +def test_mcp_validate_device_serial_ios_uses_ios_pool(monkeypatch): + from mcp_server.tools import task_runner + + ios_validate = MagicMock(return_value=None) + adb_validate = MagicMock(return_value="should not be called") + monkeypatch.setattr(task_runner.ios_device_pool, "validate_explicit_serial", ios_validate) + monkeypatch.setattr(task_runner.device_pool, "validate_explicit_serial", adb_validate) + assert task_runner._validate_device_serial("AAAA-1111", "ios") is None + ios_validate.assert_called_once_with("AAAA-1111") + adb_validate.assert_not_called() + + +def test_mcp_validate_device_serial_ios_rejection(monkeypatch): + from mcp_server.tools import task_runner + + monkeypatch.setattr( + task_runner.ios_device_pool, + "validate_explicit_serial", + lambda serial: f"iOS simulator '{serial}' is not available.", + ) + result = task_runner._validate_device_serial("NOPE", "ios") + assert result is not None and result["status"] == "failed" + assert "simctl" in result["message"] + + +# --------------------------------------------------------------------------- # +# Daemon client payloads +# --------------------------------------------------------------------------- # + + +def test_submit_task_to_daemon_forwards_platform(monkeypatch): + from artemis.runtime import daemon_client + + captured = {} + + class FakeResponse: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return b'{"status": "enqueued"}' + + def fake_urlopen(req, timeout=0): + import json + + captured.update(json.loads(req.data.decode())) + return FakeResponse() + + monkeypatch.setattr("artemis.runtime.daemon_client.urllib.request.urlopen", fake_urlopen) + resp = daemon_client.submit_task_to_daemon( + "open settings", + device_serial="AAAA-1111", + platform="ios", + ios_workspace="/proj/App.xcodeproj", + ) + assert resp == {"status": "enqueued"} + assert captured["platform"] == "ios" + assert captured["ios_workspace"] == "/proj/App.xcodeproj" + assert captured["device_serial"] == "AAAA-1111" + + +def test_submit_batch_to_daemon_forwards_platform(monkeypatch): + from artemis.runtime import daemon_client + + captured = {} + + class FakeResponse: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return b'{"status": "enqueued"}' + + def fake_urlopen(req, timeout=0): + import json + + captured.update(json.loads(req.data.decode())) + return FakeResponse() + + monkeypatch.setattr("artemis.runtime.daemon_client.urllib.request.urlopen", fake_urlopen) + daemon_client.submit_batch_to_daemon( + ["a", "b"], + device_serial="AAAA-1111", + platform="ios", + ios_workspace="/proj/App.xcodeproj", + ) + assert captured["platform"] == "ios" + assert captured["ios_workspace"] == "/proj/App.xcodeproj" + + +# --------------------------------------------------------------------------- # +# Replay context platform preservation +# --------------------------------------------------------------------------- # + + +def test_replay_device_context_preserves_ios_platform(): + from apps.admin_console.replay_manager import ReplayManager + from artemis.context import DevicePlatform + + ctx = ReplayManager._replay_device_context( + { + "mobile_platform": "ios", + "device_id": "AAAA-1111", + "device_width": 1206, + "device_height": 2622, + }, + override_device_id="BBBB-2222", + ) + assert ctx.mobile_platform == DevicePlatform.IOS + assert ctx.device_id == "BBBB-2222" + assert ctx.device_width == 1206 + + +def test_replay_device_context_android_ignores_override(): + from apps.admin_console.replay_manager import ReplayManager + from artemis.context import DevicePlatform + + ctx = ReplayManager._replay_device_context( + {"mobile_platform": "android", "device_id": "emulator-5554"}, + override_device_id="AAAA-1111", + ) + assert ctx.mobile_platform == DevicePlatform.ANDROID + # Android replay keeps the recorded device; the picker stays decorative. + assert ctx.device_id == "emulator-5554" + + +def test_replay_device_context_missing_platform_defaults_android(): + from apps.admin_console.replay_manager import ReplayManager + from artemis.context import DevicePlatform + + ctx = ReplayManager._replay_device_context({"device_id": "emulator-5554"}, None) + assert ctx.mobile_platform == DevicePlatform.ANDROID + assert ctx.device_id == "emulator-5554" + + +# --------------------------------------------------------------------------- # +# iOS readiness probe +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_ios_probe_skipped_without_simctl(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: False) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.SKIPPED + assert result.is_blocker is False + + +@pytest.mark.asyncio +async def test_ios_probe_warns_on_old_xcode(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.run_xcrun", + AsyncMock(return_value=b"Xcode 16.4\nBuild version 16F6"), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", + AsyncMock(return_value=SIM_LIST), + ) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.WARN + assert result.metadata["xcode_27_or_newer"] is False + + +@pytest.mark.asyncio +async def test_ios_probe_passes_with_xcode27_and_sims(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.run_xcrun", + AsyncMock(return_value=b"Xcode 27.0\nBuild version 27A266a"), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", + AsyncMock(return_value=SIM_LIST), + ) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.PASS + assert result.metadata["simulator_count"] == 2 + assert result.metadata["booted_udids"] == ["AAAA-1111"] + assert result.is_blocker is False + + +@pytest.mark.asyncio +async def test_ios_probe_warns_without_simulators(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.run_xcrun", + AsyncMock(return_value=b"Xcode 27.0"), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", + AsyncMock(return_value=[]), + ) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.WARN + assert "No Simulators" in result.summary + + +def test_ios_probe_registered_in_engine(): + from artemis.core.diagnostics.engine import readiness_engine + + assert "ios_simulators" in readiness_engine._probes + assert readiness_engine._probes["ios_simulators"].is_blocker is False diff --git a/third_party/mobile_use/sdk/agent.py b/third_party/mobile_use/sdk/agent.py index 5efc1822..57b7e568 100644 --- a/third_party/mobile_use/sdk/agent.py +++ b/third_party/mobile_use/sdk/agent.py @@ -66,6 +66,7 @@ from artemis.graph.graph import get_graph from artemis.graph.state import State from artemis.runtime import DeviceExecutionLock +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix from artemis.sdk.types.agent import AgentConfig from artemis.utils.startup_progress import publish_startup_progress @@ -472,7 +473,14 @@ async def _execute_task_logic(): or getattr(task, "id", None) or getattr(getattr(task, "request", None), "task_name", None) ) - active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id) + # iOS simulators lock under the "ios" scope so a UDID can never + # collide with an Android serial carrying the same text. + lock_scope = ( + IOS_LOCK_SCOPE if context.device.mobile_platform == DevicePlatform.IOS else None + ) + active_owner = DeviceExecutionLock.get_active_owner( + self._device_context.device_id, lock_scope=lock_scope + ) already_held = ( active_owner is not None and active_owner.pid == os.getpid() @@ -491,6 +499,7 @@ async def _execute_task_logic(): max_concurrency=effective_max, session_id=str(sess_id) if sess_id else None, ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", + lock_scope=lock_scope, ) ) try: From 0c4f132e7ed75c68c7561d115ba55e5f1d5d5803 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 07:16:04 +0900 Subject: [PATCH 07/32] fix: let iOS tasks run through the daemon queue The run command still carried a pre-queue guard that rejected iOS workers and forced every iOS invocation into standalone mode, so daemon-submitted iOS tasks died at worker startup with "The daemon and device queue support Android only". Drop the stale checks; iOS now daemon-routes like Android (--standalone still forces local), and the worker path reaches execute_task with platform/ios_workspace intact. Verified live: /api/run platform=ios now spawns the iOS worker and reaches agent init, stopping only at the missing GOOGLE_API_KEY. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/interfaces/cli/commands/run.py | 6 ---- tests/unit/test_ios_integration.py | 46 ++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 1ff07c4d..925e82b7 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -352,14 +352,8 @@ def run_command( raise typer.BadParameter( "iOS Simulator support is local only; cloud mode targets Android." ) - if is_worker: - raise typer.BadParameter( - "The daemon and device queue support Android only; run iOS tasks locally." - ) if locked_app_package: raise typer.BadParameter("--locked-app is unavailable for iOS simulator tasks.") - is_standalone = True - console.print("[dim]Running the iOS simulator task in standalone mode.[/dim]") ensure_video_recording_available(with_video_recording_tools) # All platforms route through unified Artemis Daemon unless specifically configured as standalone diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py index 5398db3c..c7d9a8ff 100644 --- a/tests/unit/test_ios_integration.py +++ b/tests/unit/test_ios_integration.py @@ -361,7 +361,16 @@ def test_cli_ios_runs_locally_without_android_status_or_daemon(monkeypatch): monkeypatch.delenv("ARTEMIS_TASK_WORKER", raising=False) monkeypatch.delenv("ARTEMIS_DEVICE_QUEUE_TICKET", raising=False) result = CliRunner().invoke( - app, ["run", "--platform", "ios", "--device-serial", "booted", "Open Settings"] + app, + [ + "run", + "--platform", + "ios", + "--standalone", + "--device-serial", + "booted", + "Open Settings", + ], ) assert result.exit_code == 0, result.output assert execute.call_args.kwargs["platform"] == DevicePlatform.IOS @@ -375,7 +384,15 @@ def test_cli_forwards_ios_workspace(monkeypatch, tmp_path): monkeypatch.setattr(run_module, "execute_task", execute) result = CliRunner().invoke( app, - ["run", "--platform", "ios", "--ios-workspace", str(project), "Open Settings"], + [ + "run", + "--platform", + "ios", + "--standalone", + "--ios-workspace", + str(project), + "Open Settings", + ], ) assert result.exit_code == 0, result.output assert execute.call_args.kwargs["ios_workspace_path"] == project @@ -405,7 +422,9 @@ def test_cli_approval_error_exits_2_with_guidance_panel(monkeypatch): ) ), ) - result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) + result = CliRunner().invoke( + app, ["run", "--platform", "ios", "--standalone", "Open Settings"] + ) assert result.exit_code == 2, result.output assert "Xcode Approval Required" in result.output assert "Always Allow" in result.output @@ -414,16 +433,27 @@ def test_cli_approval_error_exits_2_with_guidance_panel(monkeypatch): assert "GEMINI" not in result.output -@pytest.mark.parametrize("env_name", ["ARTEMIS_CLOUD_MODE", "ARTEMIS_TASK_WORKER"]) -def test_cli_rejects_ios_android_services_before_run(monkeypatch, env_name): +def test_cli_rejects_ios_cloud_mode_before_run(monkeypatch): execute = AsyncMock() monkeypatch.setattr(run_module, "execute_task", execute) - monkeypatch.setenv(env_name, "1") + monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1") result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) assert result.exit_code != 0 execute.assert_not_called() +def test_cli_allows_ios_queue_worker(monkeypatch): + """iOS daemon workers are supported; only cloud mode is rejected.""" + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setenv("ARTEMIS_TASK_WORKER", "1") + monkeypatch.setenv("ARTEMIS_DEVICE_QUEUE_TICKET", "ticket-1") + result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) + assert result.exit_code == 0, result.output + execute.assert_awaited_once() + assert execute.call_args.kwargs["platform"] == DevicePlatform.IOS + + def test_ios_probe_and_pro_tool_gates(native_driver, monkeypatch): from artemis.agents.checker.checker import probes_enabled from artemis.tools import index @@ -738,7 +768,9 @@ async def test_context_disconnect_driver_swallows_expected_cleanup_errors(native def test_cli_ios_accepts_video_flag_in_standalone(monkeypatch, flag): execute = AsyncMock() monkeypatch.setattr(run_module, "execute_task", execute) - result = CliRunner().invoke(app, ["run", "--platform", "ios", flag, "Open Settings"]) + result = CliRunner().invoke( + app, ["run", "--platform", "ios", "--standalone", flag, "Open Settings"] + ) assert result.exit_code == 0, result.output execute.assert_awaited_once() From 789f2f6fd9212d1ba9fc37b8d8c944da497c5d46 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 15:20:43 +0900 Subject: [PATCH 08/32] feat: add iOS simulator support to the console UI The console frontend was Android-only even though the backend already listed iOS simulators and accepted platform=ios runs: - connectedDevices merges ios_simulators probe metadata (udid/name/ state/runtime) into the device list, tagged platform=ios - Device chips render phone_iphone + an iOS badge; the hero card shows the selected simulator with its runtime - /api/system/devices/select accepts platform; iOS validates the UDID via the iOS device pool instead of retargeting the ADB probe - iOS selection is tracked client-side and satisfies the run button device gate; runTask sends platform=ios + device_serial when a simulator is selected - Sessions/queue items carry platform for an iOS tag in the session list Android selection/readiness/submission paths are unchanged. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/admin_console/routers/system.py | 26 +++- .../legacy-agent-stream.component.html | 2 +- .../src/app/core/models/session.model.ts | 2 + .../src/app/core/models/system.model.ts | 2 + .../src/app/pages/home/home.component.html | 44 +++--- .../src/app/pages/home/home.component.scss | 10 ++ .../src/app/pages/home/home.component.ts | 8 +- .../src/app/services/agent.service.spec.ts | 47 +++++++ .../src/app/services/agent.service.ts | 12 +- .../src/app/services/system.service.spec.ts | 133 ++++++++++++++++++ .../src/app/services/system.service.ts | 58 ++++++-- tests/unit/test_ios_parity.py | 79 ++++++++++- 12 files changed, 390 insertions(+), 33 deletions(-) diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index c8ce1834..70b1ace9 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -93,9 +93,13 @@ def _require_local_lifecycle_request(request: Request) -> None: class SelectDeviceRequest(BaseModel): - """Payload to select an active target Android device.""" + """Payload to select an active target device.""" - serial: str = Field(description="Serial number or identifier of the Android device to select") + serial: str = Field(description="Serial number or identifier of the device to select") + platform: str = Field( + default="android", + description="Device platform: 'android' (default) or 'ios' (simulator UDID)", + ) @router.get("/readiness", response_model=SystemReadinessReport) @@ -106,11 +110,27 @@ async def get_system_readiness(force: bool = False) -> SystemReadinessReport: @router.post("/devices/select") async def select_active_device(request: SelectDeviceRequest): - """Select the active Android device or emulator for subsequent automated tasks.""" + """Select the active device or iOS simulator for subsequent automated tasks.""" serial = request.serial.strip() if not serial: raise HTTPException(status_code=400, detail="Device serial cannot be empty.") + platform = (request.platform or "android").strip().lower() + if platform == "ios": + # iOS has no ADB probe to retarget; validate the UDID and let the + # task payload's device_serial + platform carry the binding. + from artemis.runtime import ios_device_pool + + rejection = await ios_device_pool.validate_explicit_serial_async(serial) + if rejection: + raise HTTPException(status_code=400, detail=rejection) + return {"status": "success", "selected_serial": serial, "platform": "ios"} + if platform != "android": + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) + readiness_engine.set_probe_target_serial(serial) # Return updated readiness report = await readiness_engine.run_all(force_refresh=True) diff --git a/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html b/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html index 44ec4120..c7bd726b 100644 --- a/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +++ b/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html @@ -35,7 +35,7 @@

👣 Real-time Agent Stream (Legacy Card View)

} diff --git a/apps/showcase_ui/src/app/core/models/session.model.ts b/apps/showcase_ui/src/app/core/models/session.model.ts index ef322d1c..8733ac3d 100644 --- a/apps/showcase_ui/src/app/core/models/session.model.ts +++ b/apps/showcase_ui/src/app/core/models/session.model.ts @@ -30,6 +30,7 @@ export interface TaskQueueItem { start_time?: number; device_serial?: string | null; device_id?: string | null; + platform?: 'android' | 'ios' | string | null; } export interface Session { @@ -44,6 +45,7 @@ export interface Session { device_serial?: string | null; device_id?: string | null; device_info?: any; + platform?: 'android' | 'ios' | string | null; } export interface AgentStatusResponse { diff --git a/apps/showcase_ui/src/app/core/models/system.model.ts b/apps/showcase_ui/src/app/core/models/system.model.ts index cf15406f..cfa52957 100644 --- a/apps/showcase_ui/src/app/core/models/system.model.ts +++ b/apps/showcase_ui/src/app/core/models/system.model.ts @@ -34,6 +34,8 @@ export interface DeviceInfo { is_locked: boolean | null; is_emulator: boolean; installed_packages?: string[]; + /** 'android' (default) or 'ios' (iOS Simulator; serial is the UDID). */ + platform?: 'android' | 'ios'; } export interface AdbServerEndpoint { diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index 060a01d2..303176c4 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -714,26 +714,32 @@

Android Device & Emulator Connection

- phone_android + {{ selectedTarget()?.platform === 'ios' ? 'phone_iphone' : 'phone_android' }}
- {{ activeDevice()?.model || activeDevice()?.serial }} + {{ selectedTarget()?.model || selectedTarget()?.serial }}
- @if (activeDevice()?.is_emulator) { + @if (selectedTarget()?.platform === 'ios') { + iOS Simulator + · + } @else if (selectedTarget()?.is_emulator) { Virtual Device · } - {{ activeDevice()?.serial }} - @if (activeDevice()?.screen_resolution) { + {{ selectedTarget()?.serial }} + @if (selectedTarget()?.screen_resolution) { · - {{ activeDevice()?.screen_resolution }} + {{ selectedTarget()?.screen_resolution }} } - @if (activeDevice()?.android_version) { + @if (selectedTarget()?.platform === 'ios' && selectedTarget()?.product) { + · + {{ selectedTarget()?.product }} + } @else if (selectedTarget()?.android_version) { · - Android {{ activeDevice()?.android_version }} + Android {{ selectedTarget()?.android_version }} } @if (isRemoteAdbServer()) { · @@ -758,14 +764,17 @@

Android Device & Emulator Connection

@@ -1569,14 +1578,17 @@

No Android Virtual Devices (AVD) Detected

diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index c1a5e68d..bb00547b 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -1811,6 +1811,16 @@ $text-light: #94a3b8; font-size: 14px; } + .ios-badge { + font-size: 9px; + font-weight: 700; + padding: 1px 5px; + border-radius: 4px; + background: #eef2ff; + color: #4f46e5; + letter-spacing: 0.3px; + } + .active-check { color: $emerald; font-size: 10.5px; diff --git a/apps/showcase_ui/src/app/pages/home/home.component.ts b/apps/showcase_ui/src/app/pages/home/home.component.ts index e189b9ce..1a30a247 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.ts +++ b/apps/showcase_ui/src/app/pages/home/home.component.ts @@ -371,6 +371,10 @@ export class HomeComponent implements OnInit, OnDestroy { // Device information public activeDevice = computed(() => this.systemService.activeDevice()); public connectedDevices = computed(() => this.systemService.connectedDevices()); + public selectedDeviceSerial = computed(() => this.systemService.selectedDeviceSerial()); + public selectedTarget = computed( + () => this.systemService.selectedIosDevice() ?? this.activeDevice() + ); public installedAvds = computed(() => this.systemService.installedAvds()); public emulatorPath = computed(() => this.systemService.emulatorPath()); public isEmulatorInPath = computed(() => this.systemService.isEmulatorInPath()); @@ -1008,8 +1012,8 @@ export class HomeComponent implements OnInit, OnDestroy { this.systemService.dismissEmulatorStatus().subscribe(); } - public selectTargetDevice(serial: string): void { - this.systemService.selectDevice(serial).subscribe(); + public selectTargetDevice(dev: DeviceInfo): void { + this.systemService.selectDevice(dev.serial, dev.platform ?? 'android').subscribe(); } public getEmulatorCommand(avdName: string): string { diff --git a/apps/showcase_ui/src/app/services/agent.service.spec.ts b/apps/showcase_ui/src/app/services/agent.service.spec.ts index c17e334d..980d065b 100644 --- a/apps/showcase_ui/src/app/services/agent.service.spec.ts +++ b/apps/showcase_ui/src/app/services/agent.service.spec.ts @@ -180,6 +180,53 @@ describe('AgentService live LLM retry timeline', () => { expect(selectSpy).toHaveBeenCalledWith('new-session', false); }); + it('sends platform and device_serial when an iOS simulator is selected', () => { + const service = createServiceWithoutPolling(); + let postedPayload: any = null; + (service as any).http = { + post: (_url: string, payload: any) => { + postedPayload = payload; + return of({ tasks: [] }); + } + }; + (service as any).systemService = { + selectedIosDevice: () => ({ serial: 'UDID-1', platform: 'ios' }) + }; + service.agentStatus = signal('idle'); + service.runningSessionId = signal(null); + service.userPinnedSessionId = signal(null); + (service as any).sessions = signal([]); + spyOn(service, 'selectSession'); + + service.runTask('test goal').subscribe(); + + expect(postedPayload.platform).toBe('ios'); + expect(postedPayload.device_serial).toBe('UDID-1'); + }); + + it('keeps the Android payload unchanged when no iOS simulator is selected', () => { + const service = createServiceWithoutPolling(); + let postedPayload: any = null; + (service as any).http = { + post: (_url: string, payload: any) => { + postedPayload = payload; + return of({ tasks: [] }); + } + }; + (service as any).systemService = { selectedIosDevice: () => null }; + service.agentStatus = signal('idle'); + service.runningSessionId = signal(null); + service.userPinnedSessionId = signal(null); + (service as any).sessions = signal([]); + spyOn(service, 'selectSession'); + + service.runTask('test goal').subscribe(); + + expect(postedPayload.goal).toBe('test goal'); + expect('platform' in postedPayload).toBeFalse(); + expect('device_serial' in postedPayload).toBeFalse(); + }); + it('keeps the paused state when the backend says there is nothing to resume', () => { const service = createServiceWithoutPolling(); (service as any).http = { post: () => of({ status: 'not_paused' }) }; diff --git a/apps/showcase_ui/src/app/services/agent.service.ts b/apps/showcase_ui/src/app/services/agent.service.ts index b063c99b..878876f8 100644 --- a/apps/showcase_ui/src/app/services/agent.service.ts +++ b/apps/showcase_ui/src/app/services/agent.service.ts @@ -23,6 +23,7 @@ import { ProTuningDefaults, ProTuningOptions } from '../core/models/pro-tuning.m import { StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE, PersistedCheckerStream, StreamSegment } from '../core/models/stream.model'; import { extractStepReplayFrames } from '../utils/action-formatter.util'; import { persistedStreamToSegments } from '../utils/stream-aggregator.util'; +import { SystemService } from './system.service'; export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice }; const SESSION_CACHE_KEY = 'artemis.sessions.v1'; @@ -70,6 +71,7 @@ interface SessionVideoResponse { export class AgentService { private http = inject(HttpClient); private zone = inject(NgZone); + private systemService = inject(SystemService); private activePauseCardKey: string | null = null; // Signals to expose state to components @@ -158,7 +160,8 @@ export class AgentService { start_time: at.acquired_at ? (new Date(at.acquired_at).getTime() / 1000) : (Date.now() / 1000), status: 'running', model_info: this.activeModel() || undefined, - device_serial: at.device_id || null + device_serial: at.device_id || null, + platform: at.platform || null }; sessionMap.set(sid, newSession); this.activeSessionTracking.set(sid, newSession); @@ -430,6 +433,13 @@ export class AgentService { }; this.pendingStartupProgress.set([submittedEvent]); const payload: any = { goal, profile }; + const iosDevice = this.systemService?.selectedIosDevice?.() ?? null; + if (iosDevice) { + // iOS submissions carry the simulator UDID + platform explicitly; + // Android keeps resolving through the ADB probe target. + payload.platform = 'ios'; + payload.device_serial = iosDevice.serial; + } if (expectedOutput && expectedOutput.trim()) { payload.expected_output = expectedOutput.trim(); } diff --git a/apps/showcase_ui/src/app/services/system.service.spec.ts b/apps/showcase_ui/src/app/services/system.service.spec.ts index bd2c6d51..8c9f7fb0 100644 --- a/apps/showcase_ui/src/app/services/system.service.spec.ts +++ b/apps/showcase_ui/src/app/services/system.service.spec.ts @@ -145,6 +145,139 @@ describe('SystemService readiness polling', () => { expect(service.adbServerStatus()).toBeNull(); }); + it('merges iOS simulators into the device list with platform tags', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(60), + probes: [ + { + id: 'android_adb', + category: 'device', + title: 'Android Device', + status: 'pass', + is_blocker: true, + summary: 'Device Ready', + description: '', + metadata: { + devices: [ + { + serial: 'emulator-5554', + state: 'device', + model: 'Pixel_8', + product: 'sdk', + android_version: '14', + screen_resolution: '1080x2400', + is_locked: false, + is_emulator: true + } + ] + }, + actions: [] + }, + { + id: 'ios_simulators', + category: 'device', + title: 'iOS Simulator', + status: 'pass', + is_blocker: false, + summary: '1 Booted', + description: '', + metadata: { + simulators: [ + { + udid: 'E1D9F1D1-04E5-4E95-801F-830B854FD3E2', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + }, + { + udid: 'AAAA1111-2222-3333-4444-555566667777', + name: 'iPhone 17', + state: 'Shutdown', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-26-0', + isAvailable: true + } + ] + }, + actions: [] + } + ] + }); + + const devices = service.connectedDevices(); + expect(devices.length).toBe(3); + expect(devices[0].platform).toBe('android'); + const booted = devices.find(d => d.serial === 'E1D9F1D1-04E5-4E95-801F-830B854FD3E2'); + expect(booted?.platform).toBe('ios'); + expect(booted?.model).toBe('iPhone 18 Pro'); + expect(booted?.state).toBe('device'); + expect(booted?.product).toBe('iOS 27 0'); + expect(booted?.is_emulator).toBeTrue(); + const shutdown = devices.find(d => d.serial === 'AAAA1111-2222-3333-4444-555566667777'); + expect(shutdown?.state).toBe('Shutdown'); + }); + + it('selects an iOS simulator by posting its platform and tracks it locally', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(70), + probes: [ + { + id: 'ios_simulators', + category: 'device', + title: 'iOS Simulator', + status: 'pass', + is_blocker: false, + summary: '1 Booted', + description: '', + metadata: { + simulators: [ + { + udid: 'UDID-1', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + } + ] + }, + actions: [] + } + ] + }); + + service.selectDevice('UDID-1', 'ios').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'UDID-1', platform: 'ios' }); + request.flush({ status: 'success', selected_serial: 'UDID-1', platform: 'ios' }); + + expect(service.selectedIosDevice()?.serial).toBe('UDID-1'); + expect(service.selectedDeviceSerial()).toBe('UDID-1'); + expect(service.isDeviceReady()).toBeTrue(); + }); + + it('clears the iOS selection when an Android device is selected instead', () => { + service.selectedIosDevice.set({ + serial: 'UDID-1', + state: 'device', + model: 'iPhone', + product: 'iOS 27 0', + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: true, + platform: 'ios' + }); + + service.selectDevice('emulator-5554', 'android').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'emulator-5554', platform: 'android' }); + request.flush({ status: 'success', selected_serial: 'emulator-5554', report: report(80) }); + + expect(service.selectedIosDevice()).toBeNull(); + }); + it('restores the standard local ADB server explicitly', () => { service.useLocalAdbServer(true).subscribe(); diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index b69697d9..3186ee21 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -56,6 +56,7 @@ export class SystemService { public configProbe = computed(() => this.probes().find(p => p.id === 'system_config') || null); public toolchainProbe = computed(() => this.probes().find(p => p.id === 'toolchain') || null); public adbProbe = computed(() => this.probes().find(p => p.id === 'android_adb') || null); + public iosProbe = computed(() => this.probes().find(p => p.id === 'ios_simulators') || null); public llmProbe = computed(() => this.probes().find(p => p.id === 'gemini_api_key' || p.id === 'llm_api_key') || null); public geminiProbe = computed(() => this.llmProbe()); public ocrProbe = computed(() => this.probes().find(p => p.id === 'vision_ocr_key' || p.id === 'ocr_api_key') || null); @@ -85,7 +86,8 @@ export class SystemService { public isDeviceReady = computed(() => { const adb = this.adbProbe(); - return adb?.status === 'pass'; + // A selected iOS simulator satisfies the device requirement on its own. + return adb?.status === 'pass' || this.selectedIosDevice() != null; }); // Step-level counting matching the 3-step onboarding guide @@ -122,15 +124,44 @@ export class SystemService { } - // Device list from metadata + // Device list from metadata: Android devices plus available iOS simulators. public connectedDevices = computed(() => { - const meta = this.adbProbe()?.metadata; - if (meta && Array.isArray(meta['devices'])) { - return meta['devices'] as DeviceInfo[]; + const devices: DeviceInfo[] = []; + const adbMeta = this.adbProbe()?.metadata; + if (adbMeta && Array.isArray(adbMeta['devices'])) { + for (const d of adbMeta['devices'] as DeviceInfo[]) { + devices.push({ ...d, platform: d.platform ?? 'android' }); + } } - return []; + const iosMeta = this.iosProbe()?.metadata; + if (iosMeta && Array.isArray(iosMeta['simulators'])) { + for (const s of iosMeta['simulators'] as any[]) { + if (!s?.udid) continue; + devices.push({ + serial: s.udid, + state: s.state === 'Booted' ? 'device' : (s.state || 'unknown'), + model: s.name ?? null, + product: String(s.runtime ?? '') + .replace('com.apple.CoreSimulator.SimRuntime.', '') + .replace(/-/g, ' ') || null, + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: true, + platform: 'ios', + }); + } + } + return devices; }); + // iOS selection lives client-side: iOS has no ADB probe to retarget, so + // the chosen simulator is carried in the run payload instead. + public selectedIosDevice = signal(null); + public selectedDeviceSerial = computed( + () => this.selectedIosDevice()?.serial ?? this.activeDevice()?.serial ?? null + ); + // Installed local AVD emulators from metadata public installedAvds = computed(() => { const meta = this.adbProbe()?.metadata; @@ -496,13 +527,19 @@ export class SystemService { } /** - * Select a specific connected device serial as active target + * Select a specific connected device or iOS simulator as active target */ - public selectDevice(serial: string): Observable { + public selectDevice(serial: string, platform: 'android' | 'ios' = 'android'): Observable { this.isLoading.set(true); - return this.http.post('/api/system/devices/select', { serial }).pipe( + return this.http.post('/api/system/devices/select', { serial, platform }).pipe( tap({ next: (res) => { + if (platform === 'ios' && res?.status === 'success') { + const dev = this.connectedDevices().find(d => d.serial === serial) ?? null; + this.selectedIosDevice.set(dev); + } else if (platform === 'android') { + this.selectedIosDevice.set(null); + } if (res?.report) { this.applyReadinessReport(res.report); } @@ -510,6 +547,9 @@ export class SystemService { }, error: (err) => { console.error('Failed to select active device:', err); + if (platform === 'ios') { + this.selectedIosDevice.set(null); + } this.isLoading.set(false); } }) diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index ad18ced4..c556170a 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -20,7 +20,7 @@ import asyncio import os -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -725,3 +725,80 @@ def test_ios_probe_registered_in_engine(): assert "ios_simulators" in readiness_engine._probes assert readiness_engine._probes["ios_simulators"].is_blocker is False + + +# --------------------------------------------------------------------------- # +# web: platform-aware device selection +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_select_device_ios_validates_udid_and_skips_adb_target(monkeypatch): + from unittest.mock import AsyncMock + + from apps.admin_console.routers import system as system_router + from apps.admin_console.routers.system import SelectDeviceRequest + + validate = AsyncMock(return_value=None) + monkeypatch.setattr(IosDevicePool, "validate_explicit_serial_async", validate) + set_target = Mock() + monkeypatch.setattr(system_router.readiness_engine, "set_probe_target_serial", set_target) + + result = await system_router.select_active_device( + SelectDeviceRequest(serial="AAAA-1111", platform="ios") + ) + + assert result["status"] == "success" + assert result["selected_serial"] == "AAAA-1111" + assert result["platform"] == "ios" + validate.assert_awaited_once_with("AAAA-1111") + set_target.assert_not_called() + + +@pytest.mark.asyncio +async def test_select_device_ios_rejects_unknown_udid(monkeypatch): + from unittest.mock import AsyncMock + + from fastapi import HTTPException + + from apps.admin_console.routers.system import SelectDeviceRequest, select_active_device + + monkeypatch.setattr( + IosDevicePool, + "validate_explicit_serial_async", + AsyncMock(return_value="iOS Simulator UDID 'NOPE' was not found."), + ) + + with pytest.raises(HTTPException) as exc_info: + await select_active_device(SelectDeviceRequest(serial="NOPE", platform="ios")) + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_select_device_rejects_unknown_platform(): + from fastapi import HTTPException + + from apps.admin_console.routers.system import SelectDeviceRequest, select_active_device + + with pytest.raises(HTTPException) as exc_info: + await select_active_device(SelectDeviceRequest(serial="dev-1", platform="tvos")) + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_select_device_android_unchanged(monkeypatch): + from unittest.mock import AsyncMock + + from apps.admin_console.routers import system as system_router + from apps.admin_console.routers.system import SelectDeviceRequest + + set_target = Mock() + report = object() + monkeypatch.setattr(system_router.readiness_engine, "set_probe_target_serial", set_target) + monkeypatch.setattr(system_router.readiness_engine, "run_all", AsyncMock(return_value=report)) + + result = await system_router.select_active_device(SelectDeviceRequest(serial="emulator-5554")) + + assert result["status"] == "success" + assert result["selected_serial"] == "emulator-5554" + set_target.assert_called_once_with("emulator-5554") From 3bf5581aff78c0b90d7c5ba2d83824ecd74abae8 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 15:31:12 +0900 Subject: [PATCH 09/32] fix: surface iOS simulators in the console device guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connected-device panel only rendered when the ADB probe passed, so on hosts without an Android device the iOS simulators could never be selected — and selecting one was what made the panel appear: - Show an "iOS Simulators" chip panel inside the no-device guide state whenever the ios_simulators probe reports simulators, independent of ADB state - Selecting a sim flips the device step to ready and shows the hero card - Retitle step 3 "Device & Emulator Connection" (Android via ADB or iOS Simulator via Xcode 27+) Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/app/pages/home/home.component.html | 40 ++++++++++++++++- .../src/app/pages/home/home.component.scss | 44 +++++++++++++++++++ .../src/app/pages/home/home.component.ts | 1 + .../src/app/services/system.service.ts | 4 ++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index 303176c4..63f18cc0 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -672,9 +672,9 @@

AI Model Setup

-

Android Device & Emulator Connection

+

Device & Emulator Connection

-

Local emulator, attached phone, wireless device, or another ADB server

+

Android device or emulator via ADB, or an iOS Simulator (Xcode 27+)

+ } +
+
+ }
@@ -1856,6 +1856,23 @@

System Ready for Autonomous Tasks

Pro + + + @if (selectedTarget(); as tgt) { + + }
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index f19e8059..29190574 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -4110,6 +4110,55 @@ $text-light: #94a3b8; .bar-left { display: flex; align-items: center; + gap: 8px; + + .target-device-chip { + display: inline-flex; + align-items: center; + gap: 6px; + height: 30px; + padding: 0 10px; + background: #ffffff; + border: 1px solid $card-border; + border-radius: 15px; + font-family: inherit; + font-size: 11.5px; + font-weight: 600; + color: $text-secondary; + cursor: pointer; + transition: all 0.15s ease; + + .target-icon { + font-size: 15px; + } + + .target-name { + max-width: 160px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .target-plat { + font-size: 9px; + font-weight: 700; + padding: 1px 5px; + border-radius: 4px; + background: #f1f5f9; + color: $text-muted; + letter-spacing: 0.3px; + } + + &.is-ios { + border-color: #c7d2fe; + color: #4338ca; + + .target-plat { + background: #eef2ff; + color: #4f46e5; + } + } + } .model-toggle-capsule { display: inline-flex; From 1a742996c7cbdd08998e4f5b9c1c6529bd90c438 Mon Sep 17 00:00:00 2001 From: James Lee Date: Sun, 4 Oct 2026 16:39:12 +0900 Subject: [PATCH 11/32] feat: split device setup into Android and iOS platform tabs The device guide card mixed Android-only controls (Restart ADB, connection methods, AVD management) with the iOS simulator picker, and the merged chip list made the run target ambiguous. - Add platform tabs at the top of the card; auto-select the tab that has devices (or the platform of the currently selected simulator) - Scope Restart ADB, Change Connection, Android hero/switcher, connection methods, AVD list, and locked/unauthorized states to the Android tab - Give iOS its own panel: selected-simulator hero, simulator chip picker, and an empty state explaining the Xcode 27+ requirement - Selecting a chip also switches the active tab so the chosen target stays visible; ready banner text is platform-neutral Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/app/pages/home/home.component.html | 201 ++++++++++++------ .../src/app/pages/home/home.component.scss | 79 ++++++- .../src/app/pages/home/home.component.ts | 24 +++ 3 files changed, 237 insertions(+), 67 deletions(-) diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index aaa40221..a07ba38b 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -677,6 +677,7 @@

Device & Emulator Connection

Android device or emulator via ADB, or an iOS Simulator (Xcode 27+)

+ @if (deviceGuidePlatform() === 'android') { + }
+ +
+ + +
+ + @if (deviceGuidePlatform() === 'android') { + - @if (isDeviceReady()) { + @if (androidDeviceReady()) {
- {{ selectedTarget()?.platform === 'ios' ? 'phone_iphone' : 'phone_android' }} + phone_android
- {{ selectedTarget()?.model || selectedTarget()?.serial }} + {{ activeDevice()?.model || activeDevice()?.serial }}
- @if (selectedTarget()?.platform === 'ios') { - iOS Simulator - · - } @else if (selectedTarget()?.is_emulator) { + @if (activeDevice()?.is_emulator) { Virtual Device · } - {{ selectedTarget()?.serial }} - @if (selectedTarget()?.screen_resolution) { + {{ activeDevice()?.serial }} + @if (activeDevice()?.screen_resolution) { · - {{ selectedTarget()?.screen_resolution }} + {{ activeDevice()?.screen_resolution }} } - @if (selectedTarget()?.platform === 'ios' && selectedTarget()?.product) { - · - {{ selectedTarget()?.product }} - } @else if (selectedTarget()?.android_version) { + @if (activeDevice()?.android_version) { · - Android {{ selectedTarget()?.android_version }} + Android {{ activeDevice()?.android_version }} } @if (isRemoteAdbServer()) { · @@ -756,11 +782,11 @@

Device & Emulator Connection

- @if (connectedDevices().length > 1) { + @if (androidDevices().length > 1) {
Select Active Device:
- @for (dev of connectedDevices(); track dev.serial) { + @for (dev of androidDevices(); track dev.serial) { - } -
-
- }
- @if (connectedDevices().length > 1) { + @if (androidDevices().length > 1) {
Select Active Device:
- @for (dev of connectedDevices(); track dev.serial) { + @for (dev of androidDevices(); track dev.serial) {
} + } @else { + + +
+ + + @if (selectedIosDevice(); as sim) { +
+
+
+ + phone_iphone + +
+
+
+ {{ sim.model || sim.serial }} +
+
+ iOS Simulator + · + {{ sim.serial }} + @if (sim.product) { + · + {{ sim.product }} + } +
+
+
+ + Connected +
+
+
+ } + + + @if (iosDevices().length > 0) { +
+
+ phone_iphone +
+ iOS Simulators + + {{ iosDevices().length }} available — select one to run iOS tasks + +
+
+
+ @for (dev of iosDevices(); track dev.serial) { + + } +
+
+ } @else { +
+ phone_iphone +
+
No iOS Simulators Available
+
+ {{ iosProbe()?.description || 'iOS Simulator automation requires macOS with Xcode 27+ and at least one available simulator.' }} +
+
+
+ } + +
+ + } +
@@ -1734,7 +1805,7 @@

No Android Virtual Devices (AVD) Detected

System Ready for Autonomous Tasks

-

All dependencies, AI model credentials, and Android device are verified and ready to run.

+

All dependencies, AI model credentials, and a target device are verified and ready to run.

@@ -1763,13 +1763,10 @@

No Android Virtual Devices (AVD) Detected

(click)="selectTargetDevice(dev)" > - phone_iphone + {{ dev.model?.includes('iPad') ? 'tablet_mac' : 'phone_iphone' }} - {{ dev.model || dev.serial }} - @if (dev.state !== 'device') { - ({{ dev.state }}) - } - iOS + {{ dev.model || dev.serial }} + @if (dev.serial === selectedDeviceSerial()) { ✓ Active } diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index a7211a06..14c3c4db 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -1761,6 +1761,10 @@ $text-light: #94a3b8; margin-left: auto; flex-shrink: 0; + &.status-idle { + color: #b45309; + } + .status-dot { width: 7px; height: 7px; @@ -1768,6 +1772,12 @@ $text-light: #94a3b8; background: #10b981; box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); animation: statusDotPulse 2s infinite cubic-bezier(0.4, 0, 0.6, 1); + + &.dot-warn { + background: #f59e0b; + box-shadow: none; + animation: none; + } } } @@ -1811,20 +1821,24 @@ $text-light: #94a3b8; font-size: 14px; } - .ios-badge { - font-size: 9px; - font-weight: 700; - padding: 1px 5px; - border-radius: 4px; - background: #eef2ff; - color: #4f46e5; - letter-spacing: 0.3px; + .chip-name { + color: $text-main; } - .sim-state { - font-size: 10px; - color: $text-secondary; - font-weight: 500; + .sim-state-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: #94a3b8; + flex-shrink: 0; + + &.booted { + background: #10b981; + } + + &.booting { + background: #f59e0b; + } } .active-check { diff --git a/apps/showcase_ui/src/app/services/system.service.spec.ts b/apps/showcase_ui/src/app/services/system.service.spec.ts index 8c9f7fb0..40748721 100644 --- a/apps/showcase_ui/src/app/services/system.service.spec.ts +++ b/apps/showcase_ui/src/app/services/system.service.spec.ts @@ -212,7 +212,7 @@ describe('SystemService readiness polling', () => { expect(booted?.platform).toBe('ios'); expect(booted?.model).toBe('iPhone 18 Pro'); expect(booted?.state).toBe('device'); - expect(booted?.product).toBe('iOS 27 0'); + expect(booted?.product).toBe('iOS 27.0'); expect(booted?.is_emulator).toBeTrue(); const shutdown = devices.find(d => d.serial === 'AAAA1111-2222-3333-4444-555566667777'); expect(shutdown?.state).toBe('Shutdown'); @@ -262,7 +262,7 @@ describe('SystemService readiness polling', () => { serial: 'UDID-1', state: 'device', model: 'iPhone', - product: 'iOS 27 0', + product: 'iOS 27.0', android_version: null, screen_resolution: null, is_locked: null, diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index 19b8b069..7b795268 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -143,7 +143,7 @@ export class SystemService { model: s.name ?? null, product: String(s.runtime ?? '') .replace('com.apple.CoreSimulator.SimRuntime.', '') - .replace(/-/g, ' ') || null, + .replace(/^([A-Za-z]+)-(.+)$/, (_m: string, name: string, ver: string) => `${name} ${ver.replace(/-/g, '.')}`) || null, android_version: null, screen_resolution: null, is_locked: null, From 1c58c628c2fe896c82d03a9b1a2d7211b84d6922 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 00:51:22 +0900 Subject: [PATCH 15/32] docs: reflect iOS daemon/console integration in README scope lines The feature section and roadmap entry predated the web/daemon/MCP integration commits and still described iOS as standalone-only; only physical devices remain planned. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.md | 9 +++++---- README_CN.md | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f63feec3..8620cbf7 100644 --- a/README.md +++ b/README.md @@ -106,9 +106,10 @@ cd artemis ### iOS Simulator (macOS, Xcode 27+) -Standalone CLI and embedded Python SDK tasks can target iOS simulators through -Xcode's native MCP device-interaction tools. See the [iOS setup and usage guide](./docs/ios.md) -for prerequisites, Xcode access approval, and current limits: +CLI, embedded Python SDK, web console, and daemon tasks can target iOS +simulators through Xcode's native MCP device-interaction tools. See the +[iOS setup and usage guide](./docs/ios.md) for prerequisites, Xcode access +approval, and current limits: ```bash bash scripts/setup_ios_env.sh @@ -312,7 +313,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [ ] **iOS Platform Expansion**: [Standalone iOS Simulator support](./docs/ios.md) is available with Xcode 27+; physical devices and web/daemon integration remain planned. +- [ ] **iOS Platform Expansion**: [iOS Simulator support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming; physical devices remain planned. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index fce858bd..b2e198b9 100644 --- a/README_CN.md +++ b/README_CN.md @@ -108,7 +108,7 @@ cd artemis ### iOS 模拟器（macOS，Xcode 27+） -独立 CLI 和嵌入式 Python SDK 可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器。 +CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器。 安装要求、Xcode 权限审批和当前限制见 [iOS 使用指南（英文）](./docs/ios.md)： ```bash @@ -309,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 路线图 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [ ] **iOS 跨平台支持**：[独立模式 iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+；真机和 Web/Daemon 集成仍在规划中。 +- [ ] **iOS 跨平台支持**：[iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏；真机仍在规划中。 - [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。 From 98c07b266c670f77bfb2db82ff865c5b341a5ed9 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 01:35:19 +0900 Subject: [PATCH 16/32] feat: add physical iPhone/iPad support via devicectl Extend the iOS driver family to paired physical devices. Device routing keys off the requested UDID: serials resolving to a CoreDevice physical entry get PhysicalIosDriver, while simulators and unknown serials keep the existing XcodeSimulatorDriver path. - discovery.py: devicectl list devices JSON parsing (current and deprecated property shapes) with the same fail-closed caching contract as simctl - PhysicalIosDriver subclasses the simulator driver to inherit the Xcode DeviceInteraction session (screenshots, hierarchy, taps, swipes, text), overriding only lifecycle: devicectl install/launch/terminate/openURL/apps and pairing/connectivity checks with actionable errors - connect() is decomposed into seams (_require_ios_host, _prepare_device, _start_interaction_session, _validate_session_device) so the physical driver asserts deviceIsSimulator=false without duplicating session setup - PhysicalIosRecorder polls devicectl screenshots into timestamped frames and assembles MP4 segments with the ffconcat demuxer; rotation and seal boundaries roll segments, matching the simulator manifest contract - IosDevicePool enumerates physical devices for the console/queue, validates explicit serials (paired + connected), and never auto-selects hardware - CLI help, ios.md, and both READMEs updated for physical requirements and recording limitations Physical automation requires Xcode 27+, a paired/trusted device with Developer Mode enabled, and a device-signed .app/.ipa for installs. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.md | 7 +- README_CN.md | 6 +- artemis/core/diagnostics/probes/ios_probe.py | 41 +- artemis/drivers/factory.py | 51 +- artemis/drivers/ios/discovery.py | 148 +++++ artemis/drivers/ios/physical_driver.py | 349 ++++++++++++ artemis/drivers/ios/physical_recording.py | 429 ++++++++++++++ artemis/drivers/ios/xcode_driver.py | 111 ++-- artemis/interfaces/cli/commands/run.py | 4 +- artemis/runtime/ios_device_pool.py | 107 +++- docs/ios.md | 125 ++-- tests/unit/drivers/test_ios_physical.py | 565 +++++++++++++++++++ tests/unit/test_ios_parity.py | 17 +- 13 files changed, 1822 insertions(+), 138 deletions(-) create mode 100644 artemis/drivers/ios/physical_driver.py create mode 100644 artemis/drivers/ios/physical_recording.py create mode 100644 tests/unit/drivers/test_ios_physical.py diff --git a/README.md b/README.md index 8620cbf7..1a3b381d 100644 --- a/README.md +++ b/README.md @@ -104,10 +104,11 @@ cd artemis > **Tip**: Opens `http://localhost:8000` in your default browser with a device connection wizard, live screen mirroring, prompt sandbox, and execution replays. You can also run directly from CLI: `uv run artemis run "Open Settings, find Battery and tell me current level" --profile flash`. -### iOS Simulator (macOS, Xcode 27+) +### iOS Devices (macOS, Xcode 27+) CLI, embedded Python SDK, web console, and daemon tasks can target iOS -simulators through Xcode's native MCP device-interaction tools. See the +simulators and paired physical iPhones/iPads through Xcode's native MCP +device-interaction tools plus `devicectl` lifecycle on hardware. See the [iOS setup and usage guide](./docs/ios.md) for prerequisites, Xcode access approval, and current limits: @@ -313,7 +314,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [ ] **iOS Platform Expansion**: [iOS Simulator support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming; physical devices remain planned. +- [x] **iOS Platform Expansion**: [iOS support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming, covering simulators and paired physical devices. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index b2e198b9..3df106e7 100644 --- a/README_CN.md +++ b/README_CN.md @@ -106,9 +106,9 @@ cd artemis > **提示**：启动后将自动在默认浏览器中打开 Web 控制台（`http://localhost:8000`），提供设备连接向导、实时投屏、任务演练与状态回放面板。你也可以通过命令行直接运行：`uv run artemis run "打开系统设置，找到电池选项并告诉我当前电量" --profile flash`。 -### iOS 模拟器（macOS，Xcode 27+） +### iOS 设备（macOS，Xcode 27+） -CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器。 +CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器，真机另经 devicectl 管理生命周期。 安装要求、Xcode 权限审批和当前限制见 [iOS 使用指南（英文）](./docs/ios.md)： ```bash @@ -309,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 路线图 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [ ] **iOS 跨平台支持**：[iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏；真机仍在规划中。 +- [x] **iOS 跨平台支持**：[iOS 支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏，含模拟器与已配对真机。 - [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。 diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py index d81e45c7..5eaaf057 100644 --- a/artemis/core/diagnostics/probes/ios_probe.py +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -29,6 +29,8 @@ ProbeStatus, ) from artemis.drivers.ios.discovery import ( + is_physical_ios, + list_core_devices, list_ios_simulators, run_xcrun, simctl_available, @@ -79,6 +81,14 @@ async def probe(self) -> ProbeResult: simulators = await list_ios_simulators() booted = [d for d in (simulators or []) if d.get("state") == "Booted"] + core_devices = await list_core_devices() + physical = [d for d in (core_devices or []) if is_physical_ios(d)] + connected = [ + d + for d in physical + if d.get("pairing_state") == "paired" + and d.get("connection_state") == "connected" + ] metadata = { "platform_supported": True, "xcode_version": xcode_version, @@ -86,6 +96,10 @@ async def probe(self) -> ProbeResult: "simulator_count": len(simulators or []), "booted_udids": [d.get("udid") for d in booted], "simulators": simulators or [], + "physical_device_count": len(physical), + "connected_physical_devices": [ + {"udid": d.get("udid"), "name": d.get("name")} for d in connected + ], } if not xcode_ok: @@ -110,33 +124,40 @@ async def probe(self) -> ProbeResult: ], ) - if not simulators: + if not simulators and not connected: return ProbeResult( id=self.probe_id, category=self.category, - title="iOS Simulator", + title="iOS Devices", status=ProbeStatus.WARN, is_blocker=self.is_blocker, - summary="No Simulators", + summary="No Devices", description=( - "Xcode is installed but no available iOS simulators were found. " - "Create one in Xcode > Settings > Platforms or via simctl." + "Xcode is installed but no available iOS simulators or connected " + "physical devices were found. Create a simulator in Xcode > " + "Settings > Platforms, or attach a paired iPhone/iPad." ), metadata=metadata, ) - summary = f"{len(booted)} Booted" if booted else f"{len(simulators)} Available" + parts = [] + if booted: + parts.append(f"{len(booted)} simulator(s) booted") + elif simulators: + parts.append(f"{len(simulators)} simulator(s) available") + if connected: + parts.append(f"{len(connected)} physical device(s) connected") + summary = ", ".join(parts) return ProbeResult( id=self.probe_id, category=self.category, - title="iOS Simulator", + title="iOS Devices", status=ProbeStatus.PASS, is_blocker=self.is_blocker, summary=summary, description=( - f"Xcode {xcode_version} with {len(simulators)} available iOS " - f"simulator(s){' (' + str(len(booted)) + ' booted)' if booted else ''}. " - "Submit a task with platform='ios' and a simulator UDID to use them." + f"Xcode {xcode_version}: {summary}. " + "Submit a task with platform='ios' and a device UDID to use them." ), metadata=metadata, ) diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index f4055eb4..5c67d7ad 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -31,21 +31,50 @@ logger = get_logger(__name__) +def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: + """Select the simulator or physical-device driver for an iOS target. + + The selection keys off the requested UDID, not names: a serial that + resolves to a CoreDevice physical device gets ``PhysicalIosDriver``; + anything else keeps the simulator path so its existing validation and + error messages still apply. + """ + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS support is local only; cloud mode targets Android.") + config = getattr(ctx, "agent_config", None) + workspace_path = ( + getattr(config, "ios_workspace_path", None) if config is not None else None + ) + device_id = ctx.device.device_id + if device_id and device_id != "booted": + from artemis.drivers.ios.discovery import ( + find_physical_ios_device_sync, + list_ios_simulators_sync, + ) + + simulators = list_ios_simulators_sync() + if simulators is None or not any( + device.get("udid", "").lower() == device_id.lower() for device in simulators + ): + if find_physical_ios_device_sync(device_id) is not None: + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + return PhysicalIosDriver( + device_id=device_id, + width=ctx.device.device_width, + height=ctx.device.device_height, + workspace_path=workspace_path, + ) + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + return XcodeSimulatorDriver(device_id=device_id, workspace_path=workspace_path) + + def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: """Instantiates the appropriate BaseDeviceDriver based on the runtime context.""" mobile_platform = getattr(ctx.device, "mobile_platform", "android") if mobile_platform == "ios": - if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - raise ValueError("iOS Simulator support is local only; cloud mode targets Android.") - from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver - - config = getattr(ctx, "agent_config", None) - return XcodeSimulatorDriver( - device_id=ctx.device.device_id, - workspace_path=( - getattr(config, "ios_workspace_path", None) if config is not None else None - ), - ) + return _create_ios_driver(ctx) if mobile_platform not in {"android", "mock"}: raise ValueError(f"Unsupported mobile platform: {mobile_platform}") diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py index 7baa2605..105bbe99 100644 --- a/artemis/drivers/ios/discovery.py +++ b/artemis/drivers/ios/discovery.py @@ -161,3 +161,151 @@ def list_ios_simulators_sync( except (ValueError, TypeError) as exc: logger.debug(f"simctl device list parse failed: {exc}") return None + + +# --- CoreDevice (physical iPhone/iPad) enumeration ------------------------- + +DEVICECTL_QUERY_TIMEOUT = 20.0 +_core_device_cache: list[dict[str, Any]] | None = None +_core_device_cache_time = 0.0 + + +def clear_core_device_cache() -> None: + """Drop the cached CoreDevice enumeration.""" + global _core_device_cache, _core_device_cache_time + _core_device_cache = None + _core_device_cache_time = 0.0 + + +def _cache_core_devices(devices: list[dict[str, Any]]) -> list[dict[str, Any]]: + global _core_device_cache, _core_device_cache_time + _core_device_cache = devices + _core_device_cache_time = time.monotonic() + return devices + + +def _cached_core_devices() -> list[dict[str, Any]] | None: + if _core_device_cache is None: + return None + if time.monotonic() - _core_device_cache_time > _SIMULATOR_CACHE_TTL: + return None + return _core_device_cache + + +def _device_property(device: dict[str, Any], section: str, key: str) -> Any: + """Read a property across devicectl's current and deprecated JSON shapes. + + Xcode marks ``hardwareProperties``/``deviceProperties``/``connectionProperties`` + deprecated in favor of a nested ``properties`` dictionary; accept both. + """ + value = device.get(section, {}).get(key) + if value is not None: + return value + return device.get("properties", {}).get(section, {}).get(key) + + +def parse_devicectl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Flatten ``devicectl list devices --json-output -`` into device entries.""" + devices = [] + for device in payload.get("result", {}).get("devices", []): + udid = _device_property(device, "hardwareProperties", "udid") + if not udid: + continue + devices.append( + { + "udid": udid, + "name": _device_property(device, "deviceProperties", "name"), + "os_version": _device_property( + device, "deviceProperties", "osVersionNumber" + ), + "platform": _device_property(device, "hardwareProperties", "platform"), + "reality": _device_property(device, "hardwareProperties", "reality"), + "product_type": _device_property( + device, "hardwareProperties", "productType" + ), + "connection_state": _device_property( + device, "connectionProperties", "tunnelState" + ), + "pairing_state": _device_property( + device, "connectionProperties", "pairingState" + ), + "visibility": device.get("visibilityClass"), + } + ) + return devices + + +async def list_core_devices( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """All devices known to CoreDevice (physical and simulated), cached briefly.""" + if not simctl_available(): + return None + if not force_refresh: + cached = _cached_core_devices() + if cached is not None: + return cached + try: + raw = await run_xcrun( + "devicectl", "list", "devices", "--json-output", "-", timeout=DEVICECTL_QUERY_TIMEOUT + ) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"devicectl device enumeration failed: {exc}") + return None + try: + return _cache_core_devices(parse_devicectl_devices(json.loads(raw))) + except (ValueError, TypeError) as exc: + logger.debug(f"devicectl device list parse failed: {exc}") + return None + + +def list_core_devices_sync( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """Synchronous variant for the driver factory and other sync callers.""" + if not simctl_available(): + return None + if not force_refresh: + cached = _cached_core_devices() + if cached is not None: + return cached + try: + completed = subprocess.run( + ["xcrun", "devicectl", "list", "devices", "--json-output", "-"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.debug(f"devicectl device enumeration failed: {exc}") + return None + if completed.returncode != 0: + return None + try: + return _cache_core_devices(parse_devicectl_devices(json.loads(completed.stdout))) + except (ValueError, TypeError) as exc: + logger.debug(f"devicectl device list parse failed: {exc}") + return None + + +def is_physical_ios(device: dict[str, Any]) -> bool: + """Whether a CoreDevice entry is a physical iPhone/iPad (not a simulator).""" + return ( + device.get("platform") == "iOS" + and device.get("reality") == "physical" + ) + + +def find_physical_ios_device_sync(identifier: str) -> dict[str, Any] | None: + """Match a physical iOS device by UDID or exact name, or ``None``.""" + devices = list_core_devices_sync() + if devices is None or not identifier: + return None + needle = identifier.lower() + for device in devices: + if not is_physical_ios(device): + continue + if device.get("udid", "").lower() == needle or device.get("name") == identifier: + return device + return None diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py new file mode 100644 index 00000000..aad6b299 --- /dev/null +++ b/artemis/drivers/ios/physical_driver.py @@ -0,0 +1,349 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iPhone/iPad support via ``devicectl`` plus Xcode's native session. + +Lifecycle operations (install, launch, terminate, app list, URL open) run +through ``xcrun devicectl``; observation and input reuse the same Xcode 27 +``DeviceInteraction*`` session as the simulator driver — Apple's interaction +layer distinguishes ``deviceIsSimulator`` per session and drives paired +physical devices identically once the device is attached, trusted, and in +Developer Mode. Recording polls ``devicectl`` screenshots instead of +``simctl io recordVideo``, which does not exist for hardware. + +Prerequisites surface as actionable errors: the device must appear in +``devicectl list devices`` as ``paired`` and ``connected`` (USB or network); +unpaired, offline, or non-iOS targets fail before any session starts. +""" + +import json +from pathlib import Path +import plistlib +import re +from typing import Any + +from artemis.drivers.ios.discovery import ( + is_physical_ios, + list_core_devices, + run_xcrun, +) +from artemis.drivers.ios.physical_recording import ( + IosPhysicalRecordingSession, + PhysicalIosRecorder, +) +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session, remove_active_session + +logger = get_logger(__name__) + +DEVICECTL_QUERY_TIMEOUT = 30.0 +DEVICECTL_LAUNCH_TIMEOUT = 60.0 +DEVICECTL_INSTALL_TIMEOUT = 300.0 + + +class PhysicalIosDriver(XcodeSimulatorDriver): + """One paired physical iOS device and one native interaction session. + + Inherits screenshots, hierarchy, taps, swipes, text, and keys from + ``XcodeSimulatorDriver`` — the Xcode device-interaction session is shared. + Only lifecycle plumbing differs: CoreDevice handles discovery, app + install/launch/terminate, URL opening, and frame capture. + """ + + def __init__( + self, + device_id: str, + width: int = 0, + height: int = 0, + *, + workspace_path: str | Path | None = None, + ): + super().__init__( + device_id=device_id, width=width, height=height, workspace_path=workspace_path + ) + self._physical_recorder: PhysicalIosRecorder | None = None + self._launched_pids: dict[str, int] = {} + + async def _resolve_device(self) -> dict[str, Any]: + self._validate_workspace() + if self._device_id in ("", "booted"): + raise ValueError( + "Physical iOS devices require --device-serial ; " + "'booted' only selects simulators. Find UDIDs via 'xcrun devicectl list devices'." + ) + await self._require_ios_host() + devices = await list_core_devices(force_refresh=True) + if devices is None: + raise RuntimeError( + "Could not enumerate physical devices; 'xcrun devicectl list devices' failed." + ) + needle = self._device_id.lower() + matches = [ + device + for device in devices + if is_physical_ios(device) + and ( + device.get("udid", "").lower() == needle + or device.get("name") == self._device_id + ) + ] + if not matches: + simulator = [ + device + for device in devices + if device.get("udid", "").lower() == needle + and device.get("reality") == "simulated" + ] + if simulator: + raise ValueError( + f"UDID {self._device_id} is an iOS Simulator, not a physical device." + ) + raise ValueError( + f"No paired physical iOS device matches {self._device_id!r}. " + "Attach it, trust this Mac, and verify 'xcrun devicectl list devices'." + ) + candidate = matches[0] + self._device_id = candidate["udid"] + return candidate + + async def _prepare_device(self, candidate: dict[str, Any]) -> None: + """Verify the paired device is reachable instead of booting it.""" + if candidate.get("pairing_state") != "paired": + raise RuntimeError( + f"iOS device {self._device_id} is not paired. Connect it and tap Trust." + ) + if candidate.get("connection_state") != "connected": + raise RuntimeError( + f"iOS device {self._device_id} ({candidate.get('name') or 'unknown'}) is not " + "connected. Attach it over USB or ensure network pairing is reachable; " + "on iOS 16+ also enable Developer Mode in Settings > Privacy & Security." + ) + + def _session_label(self) -> str: + return f"Artemis Device {self._device_id[:8]}" + + def _validate_session_device(self, session: dict[str, Any]) -> None: + if session.get("deviceIsSimulator"): + raise RuntimeError( + "Xcode opened a simulator session for a physical UDID; refusing to interact." + ) + if session.get("deviceUUID", "").lower() != self._device_id.lower(): + raise RuntimeError("Xcode selected a different device; refusing to interact.") + + async def launch_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + raw = await run_xcrun( + "devicectl", + "device", + "process", + "launch", + "--device", + self._device_id, + "--terminate-existing", + "--json-output", + "-", + package_name, + timeout=DEVICECTL_LAUNCH_TIMEOUT, + ) + pid = self._launched_pid(raw) + if pid is not None: + self._launched_pids[package_name] = pid + return True + + @staticmethod + def _launched_pid(raw: bytes) -> int | None: + """Best-effort process id from devicectl's launch JSON or text output.""" + try: + payload = json.loads(raw) + except (ValueError, TypeError): + payload = None + if isinstance(payload, dict): + result = payload.get("result", {}) + for key in ("processIdentifier", "pid"): + pid = result.get(key) or payload.get(key) + if isinstance(pid, int): + return pid + if isinstance(pid, str) and pid.isdigit(): + return int(pid) + match = re.search(r"pid[:= ]+(\d+)", raw.decode(errors="replace")) + return int(match.group(1)) if match else None + + async def _resolve_pid(self, package_name: str) -> int: + """Find a running process id for the bundle, tracking launches first.""" + tracked = self._launched_pids.get(package_name) + if tracked is not None: + return tracked + try: + raw = await run_xcrun( + "devicectl", + "device", + "info", + "processes", + "--device", + self._device_id, + "--json-output", + "-", + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + payload = json.loads(raw) + for process in payload.get("result", {}).get("runningProcesses", []): + executable = str( + process.get("executable") or process.get("bundleIdentifier") or "" + ) + if package_name in executable and isinstance( + process.get("processIdentifier"), int + ): + return process["processIdentifier"] + except (OSError, RuntimeError, TimeoutError, ValueError, TypeError) as exc: + logger.debug(f"devicectl process enumeration failed: {exc}") + raise ValueError( + f"No running process found for {package_name!r} on {self._device_id}; " + "launch it with launch_app before stopping." + ) + + async def stop_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + pid = await self._resolve_pid(package_name) + await run_xcrun( + "devicectl", + "device", + "process", + "terminate", + "--device", + self._device_id, + "--pid", + str(pid), + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + self._launched_pids.pop(package_name, None) + return True + + async def install_app(self, app_path: Path) -> str: + self._require_connected() + path = app_path.expanduser().resolve() + if path.suffix == ".ipa": + if not path.is_file(): + raise ValueError("iOS installation requires an existing .ipa file.") + # The bundle id lives inside the archive; the filename stem is the + # best available label until the app is installed and listed. + bundle = path.stem + elif path.suffix == ".app" and path.is_dir(): + with (path / "Info.plist").open("rb") as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + if not isinstance(bundle, str) or not bundle: + raise ValueError("The .app has no CFBundleIdentifier in Info.plist.") + else: + raise ValueError( + "Physical iOS installation requires a signed .app directory or .ipa built " + "for a device (arm64) with a valid provisioning profile." + ) + async with self._operation_lock: + self._require_connected() + await run_xcrun( + "devicectl", + "device", + "install", + "app", + "--device", + self._device_id, + str(path), + timeout=DEVICECTL_INSTALL_TIMEOUT, + ) + return bundle + + async def list_apps(self) -> dict[str, str]: + async with self._operation_lock: + self._require_connected() + raw = await run_xcrun( + "devicectl", + "device", + "info", + "apps", + "--device", + self._device_id, + "--json-output", + "-", + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + payload = json.loads(raw) + apps = payload.get("result", {}).get("apps", []) + result = {} + for app in apps: + bundle = app.get("bundleIdentifier") or app.get("bundleID") + if not bundle: + continue + result[bundle] = app.get("name") or bundle + return result + + async def open_url(self, url: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun( + "devicectl", + "device", + "process", + "openURL", + "--device", + self._device_id, + url, + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + return True + + async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: + raise NotImplementedError("Android shell commands are unavailable on iOS devices.") + + # --- Recording (devicectl screenshot polling) --- + + @property + def recording_session(self) -> IosPhysicalRecordingSession | None: + if self._physical_recorder is None: + return None + return self._physical_recorder.session + + async def start_video_recording( + self, output_dir: Path | None = None, max_duration_seconds: int = 900 + ) -> None: + self._require_connected() + if self._physical_recorder is None: + self._physical_recorder = PhysicalIosRecorder(self._device_id) + await self._physical_recorder.start(output_dir, max_duration_seconds) + + async def seal_recording_segment(self, through_time: float | None = None) -> None: + if self._physical_recorder is not None: + await self._physical_recorder.seal(through_time) + + async def stop_video_recording(self) -> str | None: + if self._physical_recorder is None: + return None + path = await self._physical_recorder.stop() + return str(path) if path is not None else None + + async def disconnect(self) -> None: + try: + if self._physical_recorder is not None: + session = self._physical_recorder.session + if session is not None: + if session.is_active: + try: + await self._physical_recorder.stop() + except Exception as exc: + logger.error( + f"Physical iOS recording finalization failed during disconnect: {exc}" + ) + if get_active_session(self._device_id) is session: + remove_active_session(self._device_id) + finally: + await super().disconnect() diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py new file mode 100644 index 00000000..8d92cf45 --- /dev/null +++ b/artemis/drivers/ios/physical_recording.py @@ -0,0 +1,429 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iOS device recording via polled ``devicectl`` screenshots. + +``simctl io recordVideo`` is simulator-only, so physical capture polls +``xcrun devicectl device capture screenshot`` into timestamped PNG frames and +assembles each contiguous run of equal-sized frames into a CFR MP4 with the +ffconcat demuxer, preserving real per-frame timing. Segments roll on rotation +(dimension change), max duration, or capture failures. Physical capture has no +audio and a lower frame rate than simulator recording (~1-3 fps bounded by +devicectl round-trip time). +""" + +import asyncio +from io import BytesIO +from pathlib import Path +import tempfile +import time +from typing import Any +from uuid import uuid4 + +from PIL import Image + +from artemis.config.paths import get_temp_dir +from artemis.drivers.ios.discovery import run_xcrun +from artemis.drivers.ios.recording import IosRecordingSession +from artemis.utils.video import get_ffmpeg_path, probe_video_segment, write_recording_manifest +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +CAPTURE_TIMEOUT_SECONDS = 15.0 +WATCHDOG_INTERVAL_SECONDS = 0.5 +MAX_CONSECUTIVE_FAILURES = 5 + + +async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: + """Seam for segment assembly so tests can intercept the ffmpeg process.""" + process = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _stdout, stderr = await process.communicate() + return process.returncode, stderr + + +class IosPhysicalRecordingSession(IosRecordingSession): + """Recording session for polled physical-device frame capture.""" + + poll_task: asyncio.Task | None = None + frames_dir: Path | None = None + frames: list[dict[str, Any]] = [] + frame_index: int = 0 + + +class PhysicalIosRecorder: + """Owns one screenshot-poll recording lifecycle for one physical UDID.""" + + def __init__(self, device_id: str): + self._device_id = device_id + self._session: IosPhysicalRecordingSession | None = None + self._output_dir: Path | None = None + self._lock = asyncio.Lock() + self._max_duration_seconds = 900 + self._consecutive_failures = 0 + + @property + def session(self) -> IosPhysicalRecordingSession | None: + """The latest session, kept after stop/failure for error reporting.""" + return self._session + + def _segment_output_path(self, session: IosPhysicalRecordingSession, index: int) -> Path: + assert self._output_dir is not None + return self._output_dir / f"segment_{index:04d}.mp4" + + async def _capture_frame(self, session: IosPhysicalRecordingSession) -> None: + """Append one timestamped screenshot frame to the open segment.""" + assert session.frames_dir is not None + session.frame_index += 1 + path = session.frames_dir / f"frame_{session.frame_index:06d}.png" + await run_xcrun( + "devicectl", + "device", + "capture", + "screenshot", + "--device", + self._device_id, + "--destination", + str(path), + timeout=CAPTURE_TIMEOUT_SECONDS, + ) + captured_at = time.monotonic() + if not path.exists() or path.stat().st_size == 0: + raise RuntimeError("devicectl produced no screenshot frame") + with Image.open(path) as image: + width, height = image.size + if session.capture_width is None: + session.capture_width, session.capture_height = width, height + session.segment_started_monotonic = captured_at + session.segment_started_at = time.time() + if session.anchor_monotonic is None: + session.anchor_monotonic = captured_at + session.start_time = session.segment_started_at + session.frames.append({"path": path, "at": captured_at}) + if (width, height) != (session.capture_width, session.capture_height): + await self._roll(session, captured_at, "rotation") + + async def _poll_loop(self, session: IosPhysicalRecordingSession) -> None: + """Capture frames back-to-back; devicectl's own latency paces the loop.""" + try: + while session.is_active: + failed = False + async with self._lock: + if not session.is_active: + return + try: + await self._capture_frame(session) + self._consecutive_failures = 0 + except asyncio.CancelledError: + raise + except (OSError, RuntimeError, TimeoutError) as exc: + failed = True + self._consecutive_failures += 1 + session.errors.append(f"Frame capture failed: {exc}") + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + session.errors.append( + "Frame capture failure limit reached; session stopped" + ) + session.is_active = False + return + # Yield every iteration: callers and the watchdog share this loop. + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS if failed else 0) + except asyncio.CancelledError: + return + + async def _start_segment(self, session: IosPhysicalRecordingSession) -> None: + assert self._output_dir is not None + session.frames = [] + session.frame_index = 0 + session.capture_width = session.capture_height = None + session.segment_started_monotonic = None + session.frames_dir = self._output_dir / f"frames_{session.segment_index:04d}" + session.frames_dir.mkdir(parents=True, exist_ok=True) + + async def _poll_first_frame(self, session: IosPhysicalRecordingSession) -> None: + """Block until the first frame anchors the recording timeline.""" + deadline = time.monotonic() + CAPTURE_TIMEOUT_SECONDS * 2 + while session.anchor_monotonic is None: + if time.monotonic() > deadline: + raise RuntimeError( + "Timed out waiting for the first devicectl screenshot frame." + ) + if not session.is_active: + raise RuntimeError( + session.errors[-1] + if session.errors + else "Frame capture stopped before the first frame." + ) + await asyncio.sleep(0.05) + + def _seal_current_segment( + self, session: IosPhysicalRecordingSession, end_monotonic: float + ) -> dict[str, Any] | None: + frames = [f for f in session.frames if f["at"] < end_monotonic] + session.frames = [f for f in session.frames if f["at"] >= end_monotonic] + if not frames: + return None + record = { + "frames": frames, + "output_path": self._segment_output_path(session, session.segment_index), + "start": max(0.0, frames[0]["at"] - (session.anchor_monotonic or 0.0)), + "end": max(0.0, end_monotonic - (session.anchor_monotonic or 0.0)), + "width": session.capture_width, + "height": session.capture_height, + "generation": session.generation, + "conversion_done": False, + "conversion_error": None, + } + session.segments.append(record) + session.sealed_until = max(session.sealed_until, record["end"]) + session.generation += 1 + session.segment_index += 1 + return record + + def _write_concat(self, session: IosPhysicalRecordingSession, record: dict[str, Any]) -> Path: + """Write an ffconcat list preserving real capture timestamps.""" + frames = record["frames"] + end_monotonic = record["end"] + (session.anchor_monotonic or 0.0) + lines = ["ffconcat version 1.0"] + for index, frame in enumerate(frames): + lines.append(f"file '{Path(frame['path']).as_posix()}'") + following = frames[index + 1]["at"] if index + 1 < len(frames) else end_monotonic + lines.append(f"duration {max(0.001, following - frame['at']):.3f}") + # The concat demuxer ignores the final frame's duration unless the file + # line repeats, so list the last frame twice to keep its tail visible. + lines.append(f"file '{Path(frames[-1]['path']).as_posix()}'") + concat_path = Path(frames[0]["path"]).parent / "frames.txt" + concat_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return concat_path + + async def _convert_record(self, session: IosPhysicalRecordingSession, record: dict[str, Any]) -> None: + output = Path(record["output_path"]) + width = max(2, int(record["width"] or 1080)) // 2 * 2 + height = max(2, int(record["height"] or 1920)) // 2 * 2 + temporary = output.with_name(f"{output.stem}.part.mp4") + if temporary.exists(): + temporary.unlink() + try: + concat = self._write_concat(session, record) + returncode, stderr = await _run_ffmpeg( + [ + "-y", + "-f", + "concat", + "-safe", + "0", + "-i", + str(concat), + "-vf", + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:" + "color=black,setsar=1", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary), + ] + ) + metadata = await probe_video_segment(temporary) + valid = ( + returncode == 0 + and temporary.exists() + and metadata.get("duration", 0) > 0 + and metadata.get("width", 0) > 0 + and metadata.get("height", 0) > 0 + ) + if valid: + temporary.replace(output) + record["conversion_done"] = True + return + record["conversion_error"] = "finalization produced no valid MP4" + logger.error( + f"Physical iOS recording finalization failed (code {returncode}): " + f"{stderr.decode(errors='replace')[-2000:]}" + ) + except OSError as exc: + record["conversion_error"] = str(exc) + logger.error(f"Physical iOS recording finalization failed: {exc}") + if temporary.exists(): + temporary.unlink() + session.errors.append( + f"Segment {record['output_path']} failed finalization; frames kept" + ) + + async def _roll( + self, session: IosPhysicalRecordingSession, end_monotonic: float, reason: str + ) -> None: + """Seal the current frames and open the next segment directory.""" + record = self._seal_current_segment(session, end_monotonic) + leftover = session.frames + if record is not None: + logger.info(f"Rolling physical iOS recording segment after {reason}") + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.is_active: + await self._start_segment(session) + for frame in leftover: + target = session.frames_dir / Path(frame["path"]).name + Path(frame["path"]).rename(target) + frame["path"] = target + session.frames = leftover + if leftover: + with Image.open(leftover[0]["path"]) as image: + session.capture_width, session.capture_height = image.size + session.segment_started_monotonic = leftover[0]["at"] + + async def _watchdog(self, session: IosPhysicalRecordingSession) -> None: + try: + while True: + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS) + async with self._lock: + if not session.is_active: + return + if ( + session.anchor_monotonic is not None + and time.monotonic() - session.anchor_monotonic + >= self._max_duration_seconds + ): + session.errors.append("Recording reached its duration limit") + session.is_active = False + return + except asyncio.CancelledError: + return + except Exception as exc: + logger.error(f"Physical iOS recording supervisor failed: {exc}") + session.errors.append(f"recording supervisor failed: {exc}") + + async def start( + self, + output_dir: Path | None = None, + max_duration_seconds: int = 900, + ) -> IosPhysicalRecordingSession: + async with self._lock: + if self._session is not None and self._session.is_active: + raise RuntimeError( + f"iOS recording is already active on {self._device_id}" + ) + output = ( + Path(output_dir) + if output_dir is not None + else Path( + tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings")) + ) + ) + output.mkdir(parents=True, exist_ok=True) + self._output_dir = output + self._max_duration_seconds = max_duration_seconds + self._consecutive_failures = 0 + session = IosPhysicalRecordingSession( + video_id=uuid4(), + device_id=self._device_id, + start_time=time.time(), + is_active=True, + ) + self._session = session + await self._start_segment(session) + session.poll_task = asyncio.create_task(self._poll_loop(session)) + # The first-frame wait must run outside the lock: the poll loop takes + # the same lock to append captured frames. + try: + await self._poll_first_frame(session) + except BaseException: + session.is_active = False + if session.poll_task and not session.poll_task.done(): + session.poll_task.cancel() + raise + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + return session + + async def seal(self, through_time: float | None = None) -> None: + """Seal the current segment so its final MP4 can be read safely.""" + async with self._lock: + session = self._session + if session is None or not session.is_active: + return + if session.anchor_monotonic is None: + return + if through_time is not None and session.sealed_until >= through_time: + return + end_monotonic = ( + session.anchor_monotonic + through_time + if through_time is not None + else time.monotonic() + ) + end_monotonic = min(end_monotonic, time.monotonic()) + await self._roll(session, end_monotonic, "seal") + + async def _finalize(self, session: IosPhysicalRecordingSession) -> Path | None: + session.is_active = False + for task in (session.poll_task, session.watchdog_task): + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.debug(f"Physical iOS recording task ended with an error: {exc}") + async with self._lock: + record = self._seal_current_segment(session, time.monotonic()) + if record is not None: + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + mp4_paths = [ + Path(record["output_path"]) + for record in session.segments + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ] + if not mp4_paths: + session.errors.append("No finalized physical iOS recording segments") + return None + shift = 0.0 + if session.data_engine_start_time is not None: + shift = session.start_time - session.data_engine_start_time + offsets = { + Path(record["output_path"]): max(0.0, float(record["start"]) + shift) + for record in session.segments + if record.get("conversion_done") and Path(record["output_path"]).exists() + } + output_dir = mp4_paths[0].parent + manifest = await write_recording_manifest(output_dir, mp4_paths, offsets) + if manifest is None: + session.errors.append("Recording manifest has no valid segments") + return None + return mp4_paths[0] + + async def stop(self) -> Path | None: + """Finalize all owned segments and write the recording manifest.""" + session = self._session + if session is None: + return None + return await self._finalize(session) diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index 05b04e15..9429f98b 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -82,27 +82,29 @@ async def resolve_device(self) -> str: await self._resolve_device() return self._device_id - async def _resolve_device(self) -> dict[str, Any]: - if self._workspace_path is not None: - if ( - self._workspace_path.suffix.lower() - not in { - ".xcodeproj", - ".xcworkspace", - } - or not self._workspace_path.is_dir() - ): - raise ValueError( - "iOS workspace must be an existing .xcodeproj or .xcworkspace directory." - ) + async def _require_ios_host(self) -> None: + """Validate the macOS/Xcode host requirements shared by all iOS drivers.""" if sys.platform != "darwin": - raise RuntimeError("iOS Simulator support requires macOS and Xcode 27 or later.") + raise RuntimeError("iOS support requires macOS and Xcode 27 or later.") version = (await run_xcrun("xcodebuild", "-version")).decode() match = re.search(r"Xcode\s+(\d+)", version) if not match or int(match.group(1)) < 27: raise RuntimeError( "Native iOS interaction requires Xcode 27 or later. Set DEVELOPER_DIR to select it." ) + + def _validate_workspace(self) -> None: + if self._workspace_path is not None and ( + self._workspace_path.suffix.lower() not in {".xcodeproj", ".xcworkspace"} + or not self._workspace_path.is_dir() + ): + raise ValueError( + "iOS workspace must be an existing .xcodeproj or .xcworkspace directory." + ) + + async def _resolve_device(self) -> dict[str, Any]: + self._validate_workspace() + await self._require_ios_host() devices = json.loads(await run_xcrun("simctl", "list", "devices", "--json")) available = parse_simctl_devices(devices) if self._device_id == "booted": @@ -122,15 +124,53 @@ async def _resolve_device(self) -> dict[str, Any]: self._device_id = candidates[0]["udid"] return candidates[0] + async def _prepare_device(self, candidate: dict[str, Any]) -> None: + """Bring the resolved device to an interactable state before the session.""" + if candidate.get("state") == "Shutdown": + await run_xcrun("simctl", "boot", self._device_id) + clear_ios_simulator_cache() + await run_xcrun("simctl", "bootstatus", self._device_id, "-b", timeout=180.0) + + def _session_label(self) -> str: + return f"Artemis Simulator {uuid4().hex[:8]}" + + def _validate_session_device(self, session: dict[str, Any]) -> None: + if ( + not session.get("deviceIsSimulator") + or session.get("deviceUUID", "").lower() != self._device_id.lower() + ): + raise RuntimeError("Xcode selected a different device; refusing to interact.") + + async def _start_interaction_session(self) -> dict[str, Any]: + start_arguments = { + "deviceIdentifier": self._device_id, + "sessionIdentifier": self._session_label(), + } + try: + return await self._bridge.call("DeviceInteractionStartSession", start_arguments) + except XcodeApprovalRequiredError: + if self._workspace_path is None: + raise + if "XcodeOpenWorkspace" not in self._bridge.tools: + raise XcodeApprovalRequiredError( + "DeviceInteractionStartSession", + "Xcode's XcodeOpenWorkspace tool is unavailable; " + "approve Artemis's access in Xcode.", + ) + opened = await self._bridge.call( + "XcodeOpenWorkspace", {"path": str(self._workspace_path)} + ) + identifier = opened.get("workspaceIdentifier") + if not isinstance(identifier, str) or not identifier: + raise RuntimeError("XcodeOpenWorkspace returned no usable workspace identifier.") + return await self._bridge.call("DeviceInteractionStartSession", start_arguments) + async def connect(self) -> None: async with self._connect_lock: if self._session_key: return candidate = await self._resolve_device() - if candidate.get("state") == "Shutdown": - await run_xcrun("simctl", "boot", self._device_id) - clear_ios_simulator_cache() - await run_xcrun("simctl", "bootstatus", self._device_id, "-b", timeout=180.0) + await self._prepare_device(candidate) await self._bridge.start() required = { "DeviceInteractionStartSession", @@ -143,42 +183,11 @@ async def connect(self) -> None: "Xcode's native device interaction tools are unavailable. Select Xcode 27 and approve Artemis's access in Xcode." ) try: - start_arguments = { - "deviceIdentifier": self._device_id, - "sessionIdentifier": f"Artemis Simulator {uuid4().hex[:8]}", - } - try: - session = await self._bridge.call( - "DeviceInteractionStartSession", start_arguments - ) - except XcodeApprovalRequiredError: - if self._workspace_path is None: - raise - if "XcodeOpenWorkspace" not in self._bridge.tools: - raise XcodeApprovalRequiredError( - "DeviceInteractionStartSession", - "Xcode's XcodeOpenWorkspace tool is unavailable; " - "approve Artemis's access in Xcode.", - ) - opened = await self._bridge.call( - "XcodeOpenWorkspace", {"path": str(self._workspace_path)} - ) - identifier = opened.get("workspaceIdentifier") - if not isinstance(identifier, str) or not identifier: - raise RuntimeError( - "XcodeOpenWorkspace returned no usable workspace identifier." - ) - session = await self._bridge.call( - "DeviceInteractionStartSession", start_arguments - ) + session = await self._start_interaction_session() self._session_key = session.get("interactionSessionKey") if not self._session_key: raise RuntimeError("Xcode did not return a device interaction session key.") - if ( - not session.get("deviceIsSimulator") - or session.get("deviceUUID", "").lower() != self._device_id.lower() - ): - raise RuntimeError("Xcode selected a different device; refusing to interact.") + self._validate_session_device(session) await self.get_screen_data(skip_settling=True) except ( OSError, diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 925e82b7..7bfcd454 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -198,7 +198,7 @@ def run_command( "--platform", metavar="PLATFORM", rich_help_panel="Platform", - help="Target mobile platform: Android (default) or a local iOS simulator.", + help="Target mobile platform: Android (default) or a local iOS device.", ), ] = DevicePlatform.ANDROID, ios_workspace: Annotated[ @@ -316,7 +316,7 @@ def run_command( typer.Option( "--device-serial", "-s", - help="Android serial or iOS simulator UDID; iOS defaults to the single 'booted' simulator.", + help="Android serial or iOS device UDID (simulator or paired physical device); iOS defaults to the single 'booted' simulator.", ), ] = None, session_id: Annotated[ diff --git a/artemis/runtime/ios_device_pool.py b/artemis/runtime/ios_device_pool.py index eaeb3b3f..0614334f 100644 --- a/artemis/runtime/ios_device_pool.py +++ b/artemis/runtime/ios_device_pool.py @@ -22,7 +22,13 @@ from __future__ import annotations -from artemis.drivers.ios.discovery import list_ios_simulators, list_ios_simulators_sync +from artemis.drivers.ios.discovery import ( + is_physical_ios, + list_core_devices, + list_core_devices_sync, + list_ios_simulators, + list_ios_simulators_sync, +) from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.runtime.device_lock import DeviceExecutionLock from artemis.runtime.device_pool import DeviceStatus @@ -34,9 +40,12 @@ # ("Shutdown"). Anything else (e.g. "Creating") is rejected on explicit picks. ACCEPTABLE_STATES = frozenset({"device", "Shutdown"}) +# devicectl states that allow driving a physical device right now. +PHYSICAL_ACCEPTABLE_STATES = frozenset({"device"}) + class IosDevicePool: - """Discovers iOS simulators and reports their Artemis lock state.""" + """Discovers iOS simulators and physical devices and reports lock state.""" @staticmethod def _build_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: @@ -78,21 +87,62 @@ def _build_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: ) return devices + @staticmethod + def _build_physical_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: + """CoreDevice entries -> statuses; reachable devices read as "device".""" + active_owners = DeviceExecutionLock.get_active_owners() + ios_owners = { + DeviceExecutionLock._normalize_device_id(o.device_id): o + for o in active_owners.values() + if o and getattr(o, "lock_scope", None) == IOS_LOCK_SCOPE + } + devices: list[DeviceStatus] = [] + for device in raw_devices: + if not is_physical_ios(device): + continue + udid = str(device.get("udid") or "") + paired = device.get("pairing_state") == "paired" + connected = device.get("connection_state") == "connected" + state = "device" if connected and paired else "unpaired" if not paired else "offline" + owner = ios_owners.get(DeviceExecutionLock._normalize_device_id(udid)) + devices.append( + DeviceStatus( + serial=udid, + state=state, + model=device.get("name"), + product=f"iOS {device.get('os_version') or '?'} physical", + is_emulator=False, + is_busy=owner is not None, + active_pid=owner.pid if owner else None, + active_task_desc=owner.description if owner else None, + active_session_id=owner.session_id if owner else None, + acquired_at=owner.acquired_at if owner else None, + platform="ios", + ) + ) + return devices + async def list_devices_async(self) -> list[DeviceStatus]: - """All available simulators; an enumeration failure returns an empty list.""" - return self._build_statuses(await list_ios_simulators() or []) + """All simulators plus physical devices; enumeration failures degrade to [].""" + simulators = self._build_statuses(await list_ios_simulators() or []) + physical = self._build_physical_statuses(await list_core_devices() or []) + return simulators + physical async def try_list_devices_async(self) -> list[DeviceStatus] | None: - """Like list_devices_async, but ``None`` when simctl could not answer.""" + """Like list_devices_async, but ``None`` when enumeration cannot answer.""" raw = await list_ios_simulators() - return None if raw is None else self._build_statuses(raw) + core = await list_core_devices() + if raw is None and core is None: + return None + return self._build_statuses(raw or []) + self._build_physical_statuses(core or []) async def validate_explicit_serial_async(self, requested_serial: str) -> str | None: """Reject an explicitly requested UDID, or ``None`` when usable. Mirrors the Android validator: only a successful, non-empty enumeration may reject. ``Shutdown`` simulators are valid targets — - the driver boots them on connect. + the driver boots them on connect. Physical devices must be paired + and connected (``"device"``); offline devices fail here. """ devices = await self.try_list_devices_async() if not devices: @@ -101,12 +151,16 @@ async def validate_explicit_serial_async(self, requested_serial: str) -> str | N match = by_udid.get(str(requested_serial).lower()) if match is None: return ( - f"iOS simulator '{requested_serial}' is not available. " - f"Known simulators: {sorted(d.serial for d in devices)}." + f"iOS device '{requested_serial}' is not available. " + f"Known devices: {sorted(d.serial for d in devices)}." ) - if match.state not in ACCEPTABLE_STATES: + acceptable = ( + ACCEPTABLE_STATES if match.is_emulator else PHYSICAL_ACCEPTABLE_STATES + ) + if match.state not in acceptable: + kind = "simulator" if match.is_emulator else "physical device" return ( - f"iOS simulator '{requested_serial}' is in state '{match.state}' " + f"iOS {kind} '{requested_serial}' is in state '{match.state}' " "and cannot be used." ) return None @@ -114,17 +168,32 @@ async def validate_explicit_serial_async(self, requested_serial: str) -> str | N def validate_explicit_serial(self, requested_serial: str) -> str | None: """Synchronous validator for non-async admission paths (MCP tools).""" devices = list_ios_simulators_sync() - if not devices: - return None - statuses = {d["udid"].lower(): d.get("state") for d in devices} + statuses = {d["udid"].lower(): d.get("state") for d in devices or []} state = statuses.get(str(requested_serial).lower()) - if state is None: + if state is not None: + if state != "Booted" and state != "Shutdown": + return ( + f"iOS simulator '{requested_serial}' is in state '{state}' " + "and cannot be used." + ) + return None + physical = { + d["udid"].lower(): d + for d in list_core_devices_sync() or [] + if is_physical_ios(d) + } + match = physical.get(str(requested_serial).lower()) + if match is None: + if not statuses and not physical: + return None return ( - f"iOS simulator '{requested_serial}' is not available. " - f"Known simulators: {sorted(statuses)}." + f"iOS device '{requested_serial}' is not available. " + f"Known devices: {sorted([*statuses, *physical])}." ) - if state != "Booted" and state != "Shutdown": - return f"iOS simulator '{requested_serial}' is in state '{state}' and cannot be used." + if match.get("pairing_state") != "paired": + return f"iOS device '{requested_serial}' is not paired with this Mac." + if match.get("connection_state") != "connected": + return f"iOS device '{requested_serial}' is offline; attach it and retry." return None async def select_device_async(self, preferred_serial: str | None = None) -> str | None: diff --git a/docs/ios.md b/docs/ios.md index 6dd1b1a9..f6928782 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -1,16 +1,18 @@ -# iOS Simulator support - -Artemis can run Flash and Pro tasks on iOS simulators on macOS with Xcode 27 or -newer. Select iOS explicitly: the existing Android defaults still apply on a -Mac. iOS tasks work through the CLI, embedded Python SDK, Artemis Daemon task -queue, Admin Console, and the Artemis MCP tools — including native screen -recording, video analysis/replay, device discovery, live screen streaming, and -per-device locking that cannot collide with Android targets. - -The iOS driver uses tools included with Xcode and Artemis's existing Python MCP -dependency. It communicates with `xcrun mcpbridge` using an initialized MCP -session; it uses `xcrun simctl` for simulator and app lifecycle operations. -Appium, WebDriverAgent, and third-party simulator control utilities are not +# iOS support + +Artemis can run Flash and Pro tasks on iOS simulators and on paired physical +iPhones/iPads on macOS with Xcode 27 or newer. Select iOS explicitly: the +existing Android defaults still apply on a Mac. iOS tasks work through the CLI, +embedded Python SDK, Artemis Daemon task queue, Admin Console, and the Artemis +MCP tools — including native screen recording, video analysis/replay, device +discovery, live screen streaming, and per-device locking that cannot collide +with Android targets. + +The iOS drivers use tools included with Xcode and Artemis's existing Python MCP +dependency. They communicate with `xcrun mcpbridge` using an initialized MCP +session for screen observation and input. Simulators use `xcrun simctl` for +lifecycle operations; physical devices use `xcrun devicectl` (CoreDevice). +Appium, WebDriverAgent, and third-party device control utilities are not required. Apple added native agent device interactions in Xcode 27; see the [Xcode 27 release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). @@ -27,7 +29,11 @@ required. Apple added native agent device interactions in Xcode 27; see the server introduced in Xcode 27, follow Apple's enablement instructions in the [release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). Artemis does not enable the server or grant permissions automatically. -3. Check the toolchain and available simulators from the repository root: +3. For physical devices: connect the iPhone/iPad over USB (or enable network + pairing), tap **Trust** on the device pairing prompt, and enable **Developer + Mode** in Settings > Privacy & Security on iOS 16+. `xcrun devicectl list + devices` should report the device as `paired` and `connected`. +4. Check the toolchain and available simulators from the repository root: ```bash bash scripts/setup_ios_env.sh @@ -37,7 +43,7 @@ required. Apple added native agent device interactions in Xcode 27; see the `xcrun mcp-server status`. It does not install dependencies, boot a simulator, or change access settings. A passing check confirms prerequisites; the first driver connection checks native MCP tool availability and device access. -4. Install Artemis's Python dependencies and configure a model provider using +5. Install Artemis's Python dependencies and configure a model provider using the normal project configuration: ```bash @@ -78,6 +84,36 @@ To install a simulator build before the task, add `--app-path /absolute/path/MyApp.app`. The bundle's `Info.plist` must provide `CFBundleIdentifier`. +### Physical devices + +A paired physical device is selected the same way — by UDID: + +```bash +xcrun devicectl list devices # find the device UDID +uv run artemis run "Open Settings" \ + --platform ios --standalone --device-serial --profile flash +``` + +Physical UDIDs are routed to a separate driver: `devicectl` handles install, +launch, terminate, app listing, and URL opening, while the same Xcode +interaction session supplies screenshots, hierarchy, taps, swipes, and text. +`booted` remains a simulator-only selector — physical tasks always require an +explicit `--device-serial`. + +Differences from simulators: + +- The device must already be paired, trusted, and connected; Artemis never + boots or unlocks it. A locked device can be observed but most input fails — + unlock it first. +- `--app-path` expects a device-signed artifact: an `.app` built for an arm64 + device destination (signed with a valid provisioning profile) or a `.ipa`. + Simulator `.app` bundles are x86_64/arm64-simulator builds and cannot be + installed on hardware. +- Screen recording polls `devicectl device capture screenshot` and assembles + timestamped MP4 segments (~1–3 fps). There is no `recordVideo`-equivalent + stream on hardware, so motion fidelity is lower than simulator captures and + brief gaps between frames are expected. + ### First-run Xcode approval The very first time a new Python interpreter asks Xcode for device access, @@ -190,24 +226,25 @@ instead of acquiring a second one. ## Supported operations and limits -| Operation | iOS implementation | -| --- | --- | -| Device selection and boot | `simctl` inventory, explicit UDID, `boot`, and `bootstatus` | -| Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | -| Tap, long press, and swipe | Native synthesized touch events | -| Text entry | Native keyboard synthesis with `clear_exist=false` | -| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | -| App install, launch, and terminate | `simctl` using simulator `.app` bundles and bundle identifiers | +| Operation | Simulator | Physical device | +| --- | --- | --- | +| Device selection and readiness | `simctl` inventory, `boot`, `bootstatus` | `devicectl` inventory; must be paired and connected | +| Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | Same session (`deviceIsSimulator=false`) | +| Tap, long press, and swipe | Native synthesized touch events | Same | +| Text entry | Native keyboard synthesis with `clear_exist=false` | Same | +| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | Same | +| App install, launch, and terminate | `simctl` with simulator `.app` bundles | `devicectl` with signed `.app`/`.ipa`; terminate uses the launch PID or a live process scan | +| Screen recording | `simctl io recordVideo` (VFR H.264) | `devicectl` screenshot polling assembled to timestamped MP4 | The native hierarchy is normalized into the element tree used by Artemis's perception and action tools. Screenshots and touch coordinates are kept in the same coordinate space. Custom UI without accessible elements still relies on visual targeting. -This driver targets iOS simulators. Physical iPhones and iPads, watchOS, tvOS, -and visionOS are outside this implementation. Install a simulator build of an -app; a device `.ipa` or Android `.apk` is not interchangeable with a simulator -`.app` bundle. The driver does not build Xcode projects. +watchOS, tvOS, and visionOS are outside this implementation. Install a +simulator build on simulators and a device-signed build on hardware; neither +artifact is interchangeable with the other or with an Android `.apk`. The +drivers do not build Xcode projects. iOS has no system Back button. Use the app's visible navigation controls. The native driver supports `enter`, `home`, `power`, `volume_up`, `volume_down`, @@ -221,17 +258,20 @@ Android shell commands, Logcat, Android resource identifiers, Android package discovery, and the Android Accessibility Helper are unavailable on iOS. Platform-specific operations fail explicitly when unsupported. -Screen recording uses the native `xcrun simctl io recordVideo` capture with -no third-party device automation. Each segment is a variable-frame-rate H.264 -`.mov` anchored to its first captured frame; after recording stops (or when a -rotation or the duration limit rolls a segment), the bundled FFmpeg post- -processing finalizes a browser-safe 30 fps MP4 and a `recording.json` -manifest mapping every segment to its session-time offset. The video analyzer -can clip the sealed portion of an in-progress recording; request ranges that -reach past the sealed boundary are clipped with a warning. Limitations: iOS -capture is silent (no audio), pre-first-frame startup time is not captured, -and recorder restarts leave a brief gap in the timeline rather than stretching -recorded frames. +Screen recording on simulators uses the native `xcrun simctl io recordVideo` +capture with no third-party device automation. Each segment is a +variable-frame-rate H.264 `.mov` anchored to its first captured frame; after +recording stops (or when a rotation or the duration limit rolls a segment), +the bundled FFmpeg post-processing finalizes a browser-safe 30 fps MP4 and a +`recording.json` manifest mapping every segment to its session-time offset. +On physical devices, recording polls `devicectl device capture screenshot` +into timestamped PNG frames and assembles the same manifest/MP4 output shape +at the real capture cadence. The video analyzer can clip the sealed portion of +an in-progress recording; request ranges that reach past the sealed boundary +are clipped with a warning. Limitations: iOS capture is silent (no audio), +pre-first-frame startup time is not captured, physical capture runs at ~1–3 +fps rather than continuous video, and recorder restarts leave a brief gap in +the timeline rather than stretching recorded frames. ## Troubleshooting @@ -249,6 +289,15 @@ recorded frames. inventory is empty. - **Multiple booted simulators:** supply `--device-serial` with the intended simulator's UDID. +- **Physical device not found:** confirm `xcrun devicectl list devices` lists + it as `paired`; if pairing is absent, reconnect and approve the Trust prompt. + A device shown as `disconnected`/offline needs USB reattachment or reachable + network pairing. +- **Physical install fails:** the artifact must be signed for the device — + check the signing team and provisioning profile, or build an `.ipa` with + `xcodebuild -exportArchive` for the device destination. +- **Physical input fails on a locked device:** unlock the device; the + interaction session cannot inject touches while locked. - **A queued iOS task runs on the wrong surface:** confirm the submission carried `platform: "ios"` (CLI `--platform ios`, web request `platform`, or the MCP `platform` argument); tasks default to Android. diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py new file mode 100644 index 00000000..13ec00a0 --- /dev/null +++ b/tests/unit/drivers/test_ios_physical.py @@ -0,0 +1,565 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iOS device behavior tested without attached hardware.""" + +import json +from pathlib import Path +import plistlib +from types import SimpleNamespace + +import pytest + +from artemis.drivers.ios import discovery, physical_driver, physical_recording +from artemis.drivers.ios.physical_driver import PhysicalIosDriver +from artemis.drivers.ios.physical_recording import PhysicalIosRecorder + + +IPHONE_UDID = "00008130-000245193C60001C" +IPAD_UDID = "00008103-00197094013B001E" +SIM_UDID = "DE345DD3-5792-4DAD-B863-144682629565" +WATCH_UDID = "00008301-D8814D603680202E" + + +def _devicectl_device( + udid, + name="Device", + os_version="27.0", + platform="iOS", + reality="physical", + tunnel="connected", + pairing="paired", + visibility="default", +): + """One device entry in devicectl's deprecated split-properties shape.""" + return { + "deviceProperties": {"name": name, "osVersionNumber": os_version}, + "hardwareProperties": {"udid": udid, "platform": platform, "reality": reality}, + "connectionProperties": {"tunnelState": tunnel, "pairingState": pairing}, + "visibilityClass": visibility, + } + + +def _devicectl_payload(*devices): + return {"info": {}, "result": {"devices": list(devices)}} + + +def _parsed(*devices): + """Flattened entries as `parse_devicectl_devices` emits them.""" + return discovery.parse_devicectl_devices(_devicectl_payload(*devices)) + + +PHYSICAL_IPHONE = _devicectl_device(IPHONE_UDID, name="dongseok의 iPhone") +OFFLINE_IPHONE = _devicectl_device(IPHONE_UDID, name="iPhone", tunnel="disconnected") +UNPAIRED_IPHONE = _devicectl_device(IPHONE_UDID, name="iPhone", pairing="unpaired") +SIMULATOR = _devicectl_device( + SIM_UDID, name="iPhone 18 Pro", reality="simulated", visibility="simulators" +) +WATCH = _devicectl_device(WATCH_UDID, name="Watch", platform="watchOS") + + +def test_devicectl_devices_flatten_physical_and_simulator_entries(): + devices = discovery.parse_devicectl_devices( + _devicectl_payload(PHYSICAL_IPHONE, SIMULATOR, WATCH) + ) + by_udid = {device["udid"]: device for device in devices} + assert by_udid[IPHONE_UDID]["reality"] == "physical" + assert by_udid[IPHONE_UDID]["platform"] == "iOS" + assert by_udid[SIM_UDID]["reality"] == "simulated" + assert by_udid[WATCH_UDID]["platform"] == "watchOS" + + +def test_devicectl_devices_read_the_nested_properties_shape(): + """devicectl is migrating split property dicts into one `properties` map.""" + nested = { + "properties": { + "deviceProperties": {"name": "iPhone", "osVersionNumber": "27.0"}, + "hardwareProperties": {"udid": IPHONE_UDID, "platform": "iOS", "reality": "physical"}, + "connectionProperties": {"tunnelState": "connected", "pairingState": "paired"}, + } + } + devices = discovery.parse_devicectl_devices(_devicectl_payload(nested)) + assert devices == [ + { + "udid": IPHONE_UDID, + "name": "iPhone", + "os_version": "27.0", + "platform": "iOS", + "reality": "physical", + "product_type": None, + "connection_state": "connected", + "pairing_state": "paired", + "visibility": None, + } + ] + + +def test_devicectl_devices_skip_entries_without_udid(): + devices = discovery.parse_devicectl_devices( + _devicectl_payload({"hardwareProperties": {"platform": "iOS"}}, PHYSICAL_IPHONE) + ) + assert [device["udid"] for device in devices] == [IPHONE_UDID] + + +def test_physical_classification_requires_ios_and_hardware(): + assert discovery.is_physical_ios({"platform": "iOS", "reality": "physical"}) + assert not discovery.is_physical_ios({"platform": "iOS", "reality": "simulated"}) + assert not discovery.is_physical_ios({"platform": "watchOS", "reality": "physical"}) + assert not discovery.is_physical_ios({"platform": "iOS"}) + + +def test_find_physical_device_matches_udid_case_insensitively(monkeypatch): + monkeypatch.setattr( + discovery, "list_core_devices_sync", lambda force_refresh=False: _parsed(PHYSICAL_IPHONE, SIMULATOR) + ) + found = discovery.find_physical_ios_device_sync(IPHONE_UDID.lower()) + assert found is not None and found["udid"] == IPHONE_UDID + assert discovery.find_physical_ios_device_sync("dongseok의 iPhone") is not None + assert discovery.find_physical_ios_device_sync(SIM_UDID) is None + assert discovery.find_physical_ios_device_sync("missing") is None + + +def test_find_physical_device_survives_enumeration_failure(monkeypatch): + monkeypatch.setattr(discovery, "list_core_devices_sync", lambda force_refresh=False: None) + assert discovery.find_physical_ios_device_sync(IPHONE_UDID) is None + + +@pytest.fixture +def driver(): + return PhysicalIosDriver(device_id=IPHONE_UDID) + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_booted_and_empty_serials(driver): + driver._device_id = "booted" + with pytest.raises(ValueError, match="device-serial"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_physical_resolve_pins_the_udid(driver, monkeypatch): + monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(PHYSICAL_IPHONE, SIMULATOR))) + candidate = await driver._resolve_device() + assert candidate["udid"] == IPHONE_UDID + assert driver.device_id == IPHONE_UDID + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_simulator_udids(driver, monkeypatch): + driver._device_id = SIM_UDID + monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(SIMULATOR))) + with pytest.raises(ValueError, match="Simulator"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_non_ios_hardware(driver, monkeypatch): + driver._device_id = WATCH_UDID + monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(WATCH))) + with pytest.raises(ValueError, match="paired physical iOS"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_physical_resolve_reports_enumeration_failure(driver, monkeypatch): + monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(None)) + with pytest.raises(RuntimeError, match="devicectl"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_offline_device_fails_with_actionable_error(driver): + with pytest.raises(RuntimeError, match="not connected|Developer Mode"): + await driver._prepare_device(_parsed(OFFLINE_IPHONE)[0]) + + +@pytest.mark.asyncio +async def test_unpaired_device_fails_with_trust_guidance(driver): + with pytest.raises(RuntimeError, match="not paired|Trust"): + await driver._prepare_device(_parsed(UNPAIRED_IPHONE)[0]) + + +@pytest.mark.asyncio +async def test_connected_device_needs_no_boot(driver): + await driver._prepare_device(_parsed(PHYSICAL_IPHONE)[0]) + + +def test_physical_session_rejects_simulator_sessions(driver): + with pytest.raises(RuntimeError, match="simulator session"): + driver._validate_session_device( + {"deviceIsSimulator": True, "deviceUUID": IPHONE_UDID} + ) + + +def test_physical_session_rejects_mismatched_udids(driver): + with pytest.raises(RuntimeError, match="different device"): + driver._validate_session_device( + {"deviceIsSimulator": False, "deviceUUID": IPAD_UDID} + ) + driver._validate_session_device( + {"deviceIsSimulator": False, "deviceUUID": IPHONE_UDID.lower()} + ) + + +def test_physical_session_label_is_device_scoped(driver): + assert "Simulator" not in driver._session_label() + + +def _AsyncDevices(result): + async def _list(force_refresh=False): + return result + + return _list + + +@pytest.fixture +def connected_driver(): + """A driver that has passed device resolution with a stub connection.""" + driver = PhysicalIosDriver(device_id=IPHONE_UDID) + driver._session_key = "session" + return driver + + +@pytest.mark.asyncio +async def test_launch_records_the_pid_from_devicectl_json(connected_driver, monkeypatch): + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return json.dumps({"result": {"processIdentifier": 4242}}).encode() + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.launch_app("com.example.app") + assert calls[0][:5] == ("devicectl", "device", "process", "launch", "--device") + assert IPHONE_UDID in calls[0] + assert calls[0][-1] == "com.example.app" + assert connected_driver._launched_pids == {"com.example.app": 4242} + + +@pytest.mark.asyncio +async def test_stop_app_terminates_the_tracked_pid(connected_driver, monkeypatch): + connected_driver._launched_pids["com.example.app"] = 4242 + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.stop_app("com.example.app") + terminate = calls[0] + assert terminate[:5] == ("devicectl", "device", "process", "terminate", "--device") + assert terminate[terminate.index("--pid") + 1] == "4242" + assert connected_driver._launched_pids == {} + + +@pytest.mark.asyncio +async def test_stop_app_without_pid_scans_running_processes(connected_driver, monkeypatch): + processes = { + "result": { + "runningProcesses": [ + {"processIdentifier": 7, "executable": "com.example.other"}, + {"processIdentifier": 42, "executable": "com.example.app"}, + ] + } + } + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return json.dumps(processes).encode() + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.stop_app("com.example.app") + assert calls[0][:4] == ("devicectl", "device", "info", "processes") + terminate = calls[1] + assert terminate[terminate.index("--pid") + 1] == "42" + + +@pytest.mark.asyncio +async def test_stop_app_without_process_fails_clearly(connected_driver, monkeypatch): + async def fake_xcrun(*arguments, timeout=30.0): + return json.dumps({"result": {"runningProcesses": []}}).encode() + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + with pytest.raises(ValueError, match="No running process"): + await connected_driver.stop_app("com.example.app") + + +@pytest.mark.asyncio +async def test_install_accepts_signed_app_directories(connected_driver, monkeypatch, tmp_path): + app = tmp_path / "Writer.app" + app.mkdir() + (app / "Info.plist").write_bytes( + plistlib.dumps({"CFBundleIdentifier": "com.example.writer"}) + ) + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.install_app(app) == "com.example.writer" + assert calls[0][:4] == ("devicectl", "device", "install", "app") + assert str(app) in calls[0] + + +@pytest.mark.asyncio +async def test_install_accepts_ipa_files(connected_driver, monkeypatch, tmp_path): + ipa = tmp_path / "Writer.ipa" + ipa.write_bytes(b"PK") + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.install_app(ipa) == "Writer" + + +@pytest.mark.asyncio +async def test_install_rejects_simulator_built_artifacts(connected_driver, tmp_path): + with pytest.raises(ValueError, match="signed .app"): + await connected_driver.install_app(tmp_path / "Writer.zip") + + +@pytest.mark.asyncio +async def test_open_url_uses_devicectl(connected_driver, monkeypatch): + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.open_url("bookwriter://open") + assert calls[0][:5] == ("devicectl", "device", "process", "openURL", "--device") + assert calls[0][-1] == "bookwriter://open" + + +@pytest.mark.asyncio +async def test_list_apps_parses_devicectl_json(connected_driver, monkeypatch): + payload = { + "result": { + "apps": [ + {"bundleIdentifier": "com.example.writer", "name": "Writer"}, + {"bundleIdentifier": "com.example.other"}, + ] + } + } + + async def fake_xcrun(*arguments, timeout=30.0): + return json.dumps(payload).encode() + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + apps = await connected_driver.list_apps() + assert apps == {"com.example.writer": "Writer", "com.example.other": "com.example.other"} + + +@pytest.mark.asyncio +async def test_shell_commands_stay_rejected(connected_driver): + with pytest.raises(NotImplementedError): + await connected_driver.execute_shell("ls") + + +@pytest.mark.asyncio +async def test_recording_requires_connection(): + driver = PhysicalIosDriver(device_id=IPHONE_UDID) + with pytest.raises(RuntimeError, match="Connect"): + await driver.start_video_recording() + + +# --- Physical recorder ------------------------------------------------------ + + +def _png(path: Path, width: int = 100, height: int = 200) -> None: + from PIL import Image + + Image.new("RGB", (width, height), color="red").save(path) + + +@pytest.mark.asyncio +async def test_recorder_polls_frames_and_assembles_mp4(tmp_path, monkeypatch): + """Frames become one timestamped segment through the ffconcat demuxer.""" + ffmpeg_calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + destination = Path(arguments[arguments.index("--destination") + 1]) + _png(destination) + return b"" + + async def fake_ffmpeg(arguments): + ffmpeg_calls.append(arguments) + output = Path(arguments[-1]) + output.write_bytes(b"mp4") + return 0, b"" + + async def fake_probe(path): + return {"duration": 1.0, "width": 100, "height": 200} + + async def fake_manifest(output_dir, paths, offsets): + manifest = Path(output_dir) / "manifest.json" + manifest.write_text("{}") + return manifest + + monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_recording, "_run_ffmpeg", fake_ffmpeg) + monkeypatch.setattr(physical_recording, "probe_video_segment", fake_probe) + monkeypatch.setattr(physical_recording, "write_recording_manifest", fake_manifest) + recorder = PhysicalIosRecorder(IPHONE_UDID) + session = await recorder.start(tmp_path) + path = await recorder.stop() + assert path is not None and path.suffix == ".mp4" + assert any("frames.txt" in str(arg) for call in ffmpeg_calls for arg in call) + + +@pytest.mark.asyncio +async def test_recorder_partitions_frames_at_the_seam(tmp_path, monkeypatch): + async def fake_xcrun(*arguments, timeout=30.0): + destination = Path(arguments[arguments.index("--destination") + 1]) + _png(destination) + return b"" + + monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) + recorder = PhysicalIosRecorder(IPHONE_UDID) + session = await recorder.start(tmp_path) + anchor = session.anchor_monotonic + seam = anchor + 5.0 + session.frames = [ + {"path": tmp_path / "frames_0000" / "f1.png", "at": seam - 2}, + {"path": tmp_path / "frames_0000" / "f2.png", "at": seam - 1}, + {"path": tmp_path / "frames_0000" / "f3.png", "at": seam + 1}, + ] + record = recorder._seal_current_segment(session, seam) + session.poll_task.cancel() + assert len(record["frames"]) == 2 + assert len(session.frames) == 1 + + +@pytest.mark.asyncio +async def test_recorder_fails_closed_when_devicectl_never_delivers(tmp_path, monkeypatch): + async def fake_xcrun(*arguments, timeout=30.0): + raise RuntimeError("device disconnected") + + monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_recording, "MAX_CONSECUTIVE_FAILURES", 2) + recorder = PhysicalIosRecorder(IPHONE_UDID) + with pytest.raises(RuntimeError): + await recorder.start(tmp_path) + session = recorder.session + assert session is not None and not session.is_active + assert session.errors + + +# --- Factory routing -------------------------------------------------------- + + +def test_factory_routes_physical_udids_to_the_physical_driver(monkeypatch): + import artemis.drivers.factory as factory + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="ios", + device_id=IPHONE_UDID, + device_width=1179, + device_height=2556, + ), + agent_config=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.setattr( + discovery, "list_ios_simulators_sync", lambda force_refresh=False: _parsed(SIMULATOR) + ) + monkeypatch.setattr( + discovery, "find_physical_ios_device_sync", lambda identifier: _parsed(PHYSICAL_IPHONE)[0] + ) + driver = factory.create_driver(ctx) + assert isinstance(driver, PhysicalIosDriver) + assert driver.device_id == IPHONE_UDID + + +def test_factory_keeps_simulator_udids_on_the_simulator_driver(monkeypatch): + import artemis.drivers.factory as factory + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="ios", + device_id=SIM_UDID, + device_width=1179, + device_height=2556, + ), + agent_config=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.setattr( + discovery, "list_ios_simulators_sync", lambda force_refresh=False: _parsed(SIMULATOR) + ) + driver = factory.create_driver(ctx) + assert isinstance(driver, XcodeSimulatorDriver) + assert not isinstance(driver, PhysicalIosDriver) + + +def test_factory_defaults_unknown_serials_to_simulator_validation(monkeypatch): + """Unrecognized serials keep the simulator driver so its error path applies.""" + import artemis.drivers.factory as factory + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="ios", + device_id="unknown-serial", + device_width=1179, + device_height=2556, + ), + agent_config=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.setattr(discovery, "list_ios_simulators_sync", lambda force_refresh=False: []) + monkeypatch.setattr(discovery, "find_physical_ios_device_sync", lambda identifier: None) + driver = factory.create_driver(ctx) + assert type(driver) is XcodeSimulatorDriver + + +def test_factory_cloud_mode_still_rejects_ios(monkeypatch): + import artemis.drivers.factory as factory + + ctx = SimpleNamespace( + device=SimpleNamespace(mobile_platform="ios", device_id=IPHONE_UDID), + agent_config=None, + ) + monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1") + with pytest.raises(ValueError, match="local only"): + factory.create_driver(ctx) + + +def test_android_selection_is_unchanged(monkeypatch): + import artemis.drivers.factory as factory + from artemis.drivers.android.adb_driver import AndroidAdbDriver + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="android", + device_id="emulator-5554", + device_width=1080, + device_height=2400, + ), + agent_config=None, + adb_client=None, + ui_adb_client=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.delenv("ARTEMIS_MOCK_DRIVER", raising=False) + driver = factory.create_driver(ctx) + assert isinstance(driver, AndroidAdbDriver) diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index acb8d804..5a5f6807 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -286,6 +286,8 @@ def _pool_with_devices(monkeypatch, devices=None): module = importlib.import_module("artemis.runtime.ios_device_pool") monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(devices))) monkeypatch.setattr(module, "list_ios_simulators_sync", lambda: list(devices)) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=[])) + monkeypatch.setattr(module, "list_core_devices_sync", lambda: []) return pool @@ -350,6 +352,7 @@ async def test_ios_pool_validate_fails_open_on_enumeration_error(monkeypatch): module = importlib.import_module("artemis.runtime.ios_device_pool") monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=None)) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=None)) assert await pool.validate_explicit_serial_async("ANY") is None @@ -740,6 +743,10 @@ async def test_ios_probe_warns_on_old_xcode(monkeypatch): "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", AsyncMock(return_value=SIM_LIST), ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_core_devices", + AsyncMock(return_value=[]), + ) result = await IosSimulatorProbe().probe() assert result.status is ProbeStatus.WARN assert result.metadata["xcode_27_or_newer"] is False @@ -759,6 +766,10 @@ async def test_ios_probe_passes_with_xcode27_and_sims(monkeypatch): "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", AsyncMock(return_value=SIM_LIST), ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_core_devices", + AsyncMock(return_value=[]), + ) result = await IosSimulatorProbe().probe() assert result.status is ProbeStatus.PASS assert result.metadata["simulator_count"] == 2 @@ -780,9 +791,13 @@ async def test_ios_probe_warns_without_simulators(monkeypatch): "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", AsyncMock(return_value=[]), ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_core_devices", + AsyncMock(return_value=[]), + ) result = await IosSimulatorProbe().probe() assert result.status is ProbeStatus.WARN - assert "No Simulators" in result.summary + assert "No Devices" in result.summary def test_ios_probe_registered_in_engine(): From 2140f36a59357a22805647fc0730e892d97c81d3 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 01:41:20 +0900 Subject: [PATCH 17/32] feat: route physical iOS devices through SDK, console stream, and replay - Agent.init picks the driver via ios_driver_class so embedded-SDK iOS configs resolve physical UDIDs to PhysicalIosDriver - device stream service captures physical frames via devicectl screenshot (simctl io screenshot does not exist on hardware) - replay device picker lists paired physical devices for iOS retargeting - smoke-test repair hints cover pairing/trust/Developer Mode and offline physical devices Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/admin_console/replay_manager.py | 25 +++++++++- .../services/device_stream_service.py | 31 ++++++++++--- artemis/core/diagnostics/device_smoke.py | 14 +++++- artemis/drivers/factory.py | 46 ++++++++++--------- artemis/runtime/adb_endpoint.py | 4 +- artemis/sdk/agent.py | 8 ++-- tests/unit/test_ios_integration.py | 6 +-- 7 files changed, 95 insertions(+), 39 deletions(-) diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index 8686a104..cc6e741b 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -397,7 +397,7 @@ def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path return output_dir def list_devices(self) -> list[dict]: - """Connected Android devices plus available iOS simulators.""" + """Connected Android devices, iOS simulators, and physical iOS devices.""" devices: list[dict] = [] try: from adbutils import AdbClient @@ -423,6 +423,29 @@ def list_devices(self) -> list[dict]: ) except Exception as e: print(f"Warning: Failed to query iOS simulators: {e}") + try: + from artemis.drivers.ios.discovery import ( + is_physical_ios, + list_core_devices_sync, + ) + + for device in list_core_devices_sync() or []: + if not is_physical_ios(device): + continue + connected = ( + device.get("pairing_state") == "paired" + and device.get("connection_state") == "connected" + ) + devices.append( + { + "serial": device["udid"], + "status": "online" if connected else "offline", + "platform": "ios", + "model": device.get("name"), + } + ) + except Exception as e: + print(f"Warning: Failed to query iOS physical devices: {e}") return devices def load_session_goal( diff --git a/apps/admin_console/services/device_stream_service.py b/apps/admin_console/services/device_stream_service.py index 77ee9061..4c6157b8 100644 --- a/apps/admin_console/services/device_stream_service.py +++ b/apps/admin_console/services/device_stream_service.py @@ -114,16 +114,29 @@ async def _capture_android(self, serial: str | None) -> bytes | None: return stdout if proc.returncode == 0 and len(stdout) > 1000 else None async def _capture_ios(self, udid: str) -> bytes | None: - """One PNG frame from the simulator's framebuffer via simctl. + """One PNG frame via simctl (simulator) or devicectl (physical). - ``simctl io screenshot`` only writes files (``-`` is not a stdout - sink on Xcode 27), so frames stream through one reused temp path. + Both tools only write files, so frames stream through one reused temp + path per UDID. """ - # Reused path per UDID: the loop is serialized, and simctl writes the - # file itself, so no per-frame temp file churn is needed. + from artemis.drivers.ios.discovery import find_physical_ios_device_sync + frame_path = Path(get_temp_dir("streams")) / f"ios_stream_{udid}.png" - try: - proc = await asyncio.create_subprocess_exec( + physical = await asyncio.to_thread(find_physical_ios_device_sync, udid) + if physical is not None: + cmd = [ + "xcrun", + "devicectl", + "device", + "capture", + "screenshot", + "--device", + udid, + "--destination", + str(frame_path), + ] + else: + cmd = [ "xcrun", "simctl", "io", @@ -131,6 +144,10 @@ async def _capture_ios(self, udid: str) -> bytes | None: "screenshot", "--type=png", str(frame_path), + ] + try: + proc = await asyncio.create_subprocess_exec( + *cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index cb7e78ce..691467d8 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -97,7 +97,7 @@ def _uiautomator_fix(serial: str | None) -> list[str]: def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: - """iOS Simulator repair steps (simctl/Xcode instead of adb/UIAutomator).""" + """iOS repair steps (simctl/devicectl/Xcode instead of adb/UIAutomator).""" s = serial or "" text = (error or "").lower() if "busy" in text or "another task" in text: @@ -114,9 +114,21 @@ def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: return [ "Install Xcode 27 or later and select it: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", ] + if "pair" in text or "trust" in text or "developer mode" in text: + return [ + "Pair and trust the device: reconnect it and approve the Trust prompt on the iPhone/iPad.", + "On iOS 16+, enable Developer Mode in Settings > Privacy & Security and restart the device.", + "List devices: xcrun devicectl list devices", + ] + if "offline" in text or "not connected" in text or "disconnected" in text: + return [ + "Attach the iPhone/iPad over USB or ensure network pairing is reachable.", + f"Verify connectivity: xcrun devicectl list devices (expect 'connected' for {s})", + ] if "simulator" in text or "udid" in text or "boot" in text or "not available" in text: return [ f"List simulators: xcrun simctl list devices; boot one with: xcrun simctl boot {s}", + "For physical devices: xcrun devicectl list devices (must show paired + connected).", "Create a simulator in Xcode > Settings > Platforms if none exist.", ] return [ diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index 5c67d7ad..33ccf882 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -31,21 +31,15 @@ logger = get_logger(__name__) -def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: - """Select the simulator or physical-device driver for an iOS target. +def ios_driver_class(device_id: str | None) -> type[BaseDeviceDriver]: + """Pick the driver class for an iOS serial by device reality. - The selection keys off the requested UDID, not names: a serial that - resolves to a CoreDevice physical device gets ``PhysicalIosDriver``; - anything else keeps the simulator path so its existing validation and - error messages still apply. + A serial resolving to a CoreDevice physical entry maps to + ``PhysicalIosDriver``; simulators, ``booted``, and unknown serials keep + ``XcodeSimulatorDriver`` so its existing validation and error paths apply. """ - if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - raise ValueError("iOS support is local only; cloud mode targets Android.") - config = getattr(ctx, "agent_config", None) - workspace_path = ( - getattr(config, "ios_workspace_path", None) if config is not None else None - ) - device_id = ctx.device.device_id + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + if device_id and device_id != "booted": from artemis.drivers.ios.discovery import ( find_physical_ios_device_sync, @@ -59,15 +53,25 @@ def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: if find_physical_ios_device_sync(device_id) is not None: from artemis.drivers.ios.physical_driver import PhysicalIosDriver - return PhysicalIosDriver( - device_id=device_id, - width=ctx.device.device_width, - height=ctx.device.device_height, - workspace_path=workspace_path, - ) - from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + return PhysicalIosDriver + return XcodeSimulatorDriver + - return XcodeSimulatorDriver(device_id=device_id, workspace_path=workspace_path) +def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: + """Instantiate the simulator or physical-device driver for an iOS target.""" + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS support is local only; cloud mode targets Android.") + config = getattr(ctx, "agent_config", None) + workspace_path = ( + getattr(config, "ios_workspace_path", None) if config is not None else None + ) + driver_class = ios_driver_class(ctx.device.device_id) + return driver_class( + device_id=ctx.device.device_id, + width=ctx.device.device_width, + height=ctx.device.device_height, + workspace_path=workspace_path, + ) def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: diff --git a/artemis/runtime/adb_endpoint.py b/artemis/runtime/adb_endpoint.py index 986a3c11..f006efe1 100644 --- a/artemis/runtime/adb_endpoint.py +++ b/artemis/runtime/adb_endpoint.py @@ -154,7 +154,7 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class IosTarget: - """An iOS simulator UDID bound to the local CoreSimulator service.""" + """An iOS device UDID (simulator or paired physical device).""" serial: str | None = None @@ -176,7 +176,7 @@ def apply_to_environment( ) -> MutableMapping[str, str]: target = environment if environment is not None else os.environ # LOCK_SCOPE_ENV names the execution scope generically: the ADB - # endpoint identity for Android, the platform tag for iOS simulators. + # endpoint identity for Android, the platform tag for iOS devices. target[ADB_ENDPOINT_ID_ENV] = self.lock_scope # An iOS worker never touches ADB: a stale serial must not leak in. target.pop("ADB_DEVICE_SERIAL", None) diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index 091e4b4d..25cc1255 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -147,12 +147,12 @@ async def _init_internal( raise AgentError("iOS Simulator support is local only; cloud mode targets Android.") if self._initialized: return True - from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + from artemis.drivers.factory import ios_driver_class publish_startup_progress( - "device_check", "Checking the iOS simulator", session_id=self._session_id + "device_check", "Checking the iOS device", session_id=self._session_id ) - driver = XcodeSimulatorDriver( + driver = ios_driver_class(self._config.device_id or "booted")( device_id=self._config.device_id or "booted", workspace_path=getattr(self._config, "ios_workspace_path", None), ) @@ -177,7 +177,7 @@ async def _init_internal( # simctl recording is supported and honors the configured flag. self._config = self._config.model_copy(update={"disable_device_probes": True}) publish_startup_progress( - "device_ready", "iOS simulator selected", session_id=self._session_id + "device_ready", "iOS device selected", session_id=self._session_id ) asyncio.create_task(self._prewarm_llm_connections(api_key)) self._initialized = True diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py index c7d9a8ff..9eeb98bb 100644 --- a/tests/unit/test_ios_integration.py +++ b/tests/unit/test_ios_integration.py @@ -110,7 +110,7 @@ def test_factory_selects_ios_without_creating_adb(native_driver, monkeypatch): monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) context = ios_context() assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=None) + native_driver.constructor.assert_called_once_with(device_id="booted", width=1170, height=2532, workspace_path=None) assert context.adb_client is None @@ -125,13 +125,13 @@ def test_factory_forwards_configured_ios_workspace(native_driver, tmp_path): .build(validate_profiles=False) ) assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=project) + native_driver.constructor.assert_called_once_with(device_id="booted", width=1170, height=2532, workspace_path=project) def test_factory_tolerates_missing_agent_config(native_driver): context = ios_context() assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=None) + native_driver.constructor.assert_called_once_with(device_id="booted", width=1170, height=2532, workspace_path=None) def test_macos_host_keeps_android_default(monkeypatch): From 5280c6c8b2dac7804b9191e5eb10bda86407920d Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 03:24:26 +0900 Subject: [PATCH 18/32] feat: drive physical iOS UI through WebDriverAgent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Xcode's DeviceInteraction* MCP tools accept simulators only — verified live against a paired iPhone (the eligible-device list contains simulators exclusively), so physical observation and input move to WebDriverAgent: - wda.py: stdlib HTTP client (no new dependency) covering /status, /session, /source?format=json, /screenshot, /window/size, W3C pointer actions for tap/long-press/swipe, /wda/keys for text, /wda/homescreen and /wda/pressButton for keys, /wda/activeAppInfo for the foreground bundle. - Endpoint resolution probes ARTEMIS_IOS_WDA_URL, ARTEMIS_IOS_WDA_HOST, the CoreDevice tunnel address from devicectl info details, and 127.0.0.1:8100 for iproxy/pymobiledevice3 forwards. - connect() locates an installed *WebDriverAgent* runner, launches it via devicectl, and optionally hosts a build-for-testing .xctestrun through xcodebuild test-without-building (ARTEMIS_IOS_WDA_XCTESTRUN); disconnect tears both down. - Hierarchy flattens the WDA JSON tree into the same ui_elements shape (text/resource_id/class/parsed_bounds/hit_point) so controllers, element lookup, and tap_element are unchanged. - devicectl JSON for info subcommands now goes through a scratch file — '--json-output -' stdout is polluted by the human-readable table on hardware — and terminate resolves PIDs by matching the app's install URL prefix against running executables. - docs/ios.md and READMEs describe the WDA prerequisite, signing options, endpoint overrides, and the UI-Automation passcode gate. 47 physical unit tests pass; the simulator driver and Android paths are untouched. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.md | 7 +- README_CN.md | 4 +- artemis/drivers/ios/physical_driver.py | 573 ++++++++++++++++++++---- artemis/drivers/ios/wda.py | 380 ++++++++++++++++ docs/ios.md | 72 ++- tests/unit/drivers/test_ios_physical.py | 417 +++++++++++++++-- 6 files changed, 1290 insertions(+), 163 deletions(-) create mode 100644 artemis/drivers/ios/wda.py diff --git a/README.md b/README.md index 1a3b381d..2d6d4724 100644 --- a/README.md +++ b/README.md @@ -107,10 +107,11 @@ cd artemis ### iOS Devices (macOS, Xcode 27+) CLI, embedded Python SDK, web console, and daemon tasks can target iOS -simulators and paired physical iPhones/iPads through Xcode's native MCP -device-interaction tools plus `devicectl` lifecycle on hardware. See the +simulators through Xcode's native MCP device-interaction tools, and paired +physical iPhones/iPads through `devicectl` plus a WebDriverAgent runner +installed on the device. See the [iOS setup and usage guide](./docs/ios.md) for prerequisites, Xcode access -approval, and current limits: +approval, WDA setup, and current limits: ```bash bash scripts/setup_ios_env.sh diff --git a/README_CN.md b/README_CN.md index 3df106e7..2594d444 100644 --- a/README_CN.md +++ b/README_CN.md @@ -108,8 +108,8 @@ cd artemis ### iOS 设备（macOS，Xcode 27+） -CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器，真机另经 devicectl 管理生命周期。 -安装要求、Xcode 权限审批和当前限制见 [iOS 使用指南（英文）](./docs/ios.md)： +CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器；真机（iPhone/iPad）则通过 `devicectl` 管理生命周期、并在设备上安装 WebDriverAgent 负责界面操作。 +安装要求、Xcode 权限审批、WDA 配置和当前限制见 [iOS 使用指南（英文）](./docs/ios.md)： ```bash bash scripts/setup_ios_env.sh diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py index aad6b299..e63fec9e 100644 --- a/artemis/drivers/ios/physical_driver.py +++ b/artemis/drivers/ios/physical_driver.py @@ -10,27 +10,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Physical iPhone/iPad support via ``devicectl`` plus Xcode's native session. +"""Physical iPhone/iPad support via ``devicectl`` plus WebDriverAgent. -Lifecycle operations (install, launch, terminate, app list, URL open) run -through ``xcrun devicectl``; observation and input reuse the same Xcode 27 -``DeviceInteraction*`` session as the simulator driver — Apple's interaction -layer distinguishes ``deviceIsSimulator`` per session and drives paired -physical devices identically once the device is attached, trusted, and in -Developer Mode. Recording polls ``devicectl`` screenshots instead of -``simctl io recordVideo``, which does not exist for hardware. +Lifecycle operations (install, launch, terminate, app list, URL open, +screenshots) run through ``xcrun devicectl`` against a paired device. Xcode's +``DeviceInteraction*`` MCP tools accept simulators only, so UI observation and +input on hardware go through WebDriverAgent (WDA), the XCUITest bridge Appium +uses: the driver finds a signed WDA runner on the device, launches it through +``devicectl``, and talks to its HTTP endpoint over the CoreDevice tunnel, a +LAN address, or a forwarded port. Prerequisites surface as actionable errors: the device must appear in -``devicectl list devices`` as ``paired`` and ``connected`` (USB or network); -unpaired, offline, or non-iOS targets fail before any session starts. +``devicectl list devices`` as ``paired`` and ``connected`` (USB or network), +Developer Mode must be on, and a WebDriverAgent runner must be installed — +set ``ARTEMIS_IOS_WDA_URL`` to reach an existing server directly. """ +import asyncio +import base64 +from io import BytesIO import json +import os from pathlib import Path import plistlib import re -from typing import Any +import tempfile +from typing import Any, Literal +from PIL import Image + +from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection from artemis.drivers.ios.discovery import ( is_physical_ios, list_core_devices, @@ -40,7 +49,15 @@ IosPhysicalRecordingSession, PhysicalIosRecorder, ) +from artemis.drivers.ios.wda import ( + WdaClient, + WdaUnavailableError, + parse_wda_elements, + probe_wda, + wda_url_candidates, +) from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from third_party.mobile_use.controllers.types import ElementQuery from third_party.mobile_use.utils.logger import get_logger from third_party.mobile_use.utils.video import get_active_session, remove_active_session @@ -49,15 +66,29 @@ DEVICECTL_QUERY_TIMEOUT = 30.0 DEVICECTL_LAUNCH_TIMEOUT = 60.0 DEVICECTL_INSTALL_TIMEOUT = 300.0 +WDA_START_TIMEOUT = 45.0 +WDA_RUNNER_PATTERN = re.compile(r"webdriveragent", re.IGNORECASE) +WDA_BUNDLE_ENV = "ARTEMIS_IOS_WDA_BUNDLE_ID" +WDA_XCTESTRUN_ENV = "ARTEMIS_IOS_WDA_XCTESTRUN" + +_WDA_SETUP_HINT = ( + "Physical iOS UI automation needs WebDriverAgent on the device. Build it once with " + "'xcodebuild build-for-testing -project /WebDriverAgent.xcodeproj " + "-scheme WebDriverAgentRunner -destination id= -allowProvisioningUpdates " + "DEVELOPMENT_TEAM=', install the produced WebDriverAgentRunner-Runner.app via " + "'xcrun devicectl device install app', point ARTEMIS_IOS_WDA_XCTESTRUN at the " + "generated .xctestrun for Artemis to host it, or expose a running server through " + "ARTEMIS_IOS_WDA_URL (iproxy, pymobiledevice3, or the device LAN address)." +) class PhysicalIosDriver(XcodeSimulatorDriver): - """One paired physical iOS device and one native interaction session. + """One paired physical iOS device driven by devicectl and WebDriverAgent. - Inherits screenshots, hierarchy, taps, swipes, text, and keys from - ``XcodeSimulatorDriver`` — the Xcode device-interaction session is shared. - Only lifecycle plumbing differs: CoreDevice handles discovery, app - install/launch/terminate, URL opening, and frame capture. + Shares lifecycle conventions with ``XcodeSimulatorDriver`` but replaces the + interaction core entirely: CoreDevice handles discovery and app lifecycle, + WDA supplies hierarchy, screenshots, taps, swipes, text, and keys. The + simulator-only Xcode MCP bridge is never started for a physical target. """ def __init__( @@ -73,6 +104,11 @@ def __init__( ) self._physical_recorder: PhysicalIosRecorder | None = None self._launched_pids: dict[str, int] = {} + self._wda: WdaClient | None = None + self._wda_runner_pid: int | None = None + self._wda_test_process: asyncio.subprocess.Process | None = None + + # --- Resolution and connection --- async def _resolve_device(self) -> dict[str, Any]: self._validate_workspace() @@ -129,34 +165,397 @@ async def _prepare_device(self, candidate: dict[str, Any]) -> None: "on iOS 16+ also enable Developer Mode in Settings > Privacy & Security." ) - def _session_label(self) -> str: - return f"Artemis Device {self._device_id[:8]}" + async def connect(self) -> None: + async with self._connect_lock: + if self._session_key: + return + candidate = await self._resolve_device() + await self._prepare_device(candidate) + try: + self._wda = await self._ensure_wda() + self._session_key = await self._wda.open_session() + await self.get_screen_data(skip_settling=True) + except ( + OSError, + ValueError, + RuntimeError, + TimeoutError, + asyncio.CancelledError, + ): + try: + await self.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: + logger.warning( + "Could not release the WDA session after a connection failure: " + f"{cleanup_error}", + ) + raise - def _validate_session_device(self, session: dict[str, Any]) -> None: - if session.get("deviceIsSimulator"): + async def disconnect(self) -> None: + try: + if self._physical_recorder is not None: + session = self._physical_recorder.session + if session is not None: + if session.is_active: + try: + await self._physical_recorder.stop() + except Exception as exc: + logger.error( + f"Physical iOS recording finalization failed during disconnect: {exc}" + ) + if get_active_session(self._device_id) is session: + remove_active_session(self._device_id) + finally: + async with self._operation_lock: + self._session_key = None + client, self._wda = self._wda, None + runner_pid, self._wda_runner_pid = self._wda_runner_pid, None + test_process, self._wda_test_process = self._wda_test_process, None + if client is not None: + await client.close_session() + if test_process is not None and test_process.returncode is None: + test_process.terminate() + try: + await asyncio.wait_for(test_process.wait(), timeout=10.0) + except (TimeoutError, asyncio.TimeoutError): + test_process.kill() + if runner_pid is not None: + try: + await self._terminate_pid(runner_pid) + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + logger.debug(f"WDA runner termination failed: {exc}") + + def _require_connected(self) -> None: + if not self._session_key or self._device_id in ("", "booted"): + raise RuntimeError("Connect the physical iOS driver before interacting.") + + async def _ensure_wda(self) -> WdaClient: + """Attach to a reachable WDA server, starting one when possible. + + Order: probe known endpoints, then try launching an installed runner + app directly (newer WDA builds self-host), then — when + ``ARTEMIS_IOS_WDA_XCTESTRUN`` names a ``.xctestrun`` file from a + ``build-for-testing`` — spawn ``xcodebuild test-without-building``, + which is the canonical way to boot the XCTest session that hosts + WDA's HTTP server. + """ + candidates = wda_url_candidates(tunnel_ip=await self._tunnel_ip()) + client = await probe_wda(candidates) + if client is not None: + return client + runner = await self._wda_runner_bundle() + if runner is not None: + self._wda_runner_pid = await self._launch_bundle( + runner, terminate_existing=False + ) + logger.info( + f"Launched WebDriverAgent runner {runner} on {self._device_id}; " + "waiting for its HTTP server" + ) + xctestrun = os.environ.get(WDA_XCTESTRUN_ENV) + if runner is None and not xctestrun: raise RuntimeError( - "Xcode opened a simulator session for a physical UDID; refusing to interact." + f"No WebDriverAgent server answers on {self._device_id} and no WDA " + f"runner is installed. {_WDA_SETUP_HINT}" ) - if session.get("deviceUUID", "").lower() != self._device_id.lower(): - raise RuntimeError("Xcode selected a different device; refusing to interact.") + if xctestrun: + await self._start_xctest_session(xctestrun) + deadline = asyncio.get_running_loop().time() + WDA_START_TIMEOUT + while True: + client = await probe_wda(candidates) + if client is not None: + return client + if asyncio.get_running_loop().time() >= deadline: + raise RuntimeError( + f"WebDriverAgent did not answer within {WDA_START_TIMEOUT:.0f}s " + f"on {self._device_id} at {candidates}. If the device is showing " + "a passcode prompt to enable UI Automation, enter it on the " + f"device first. {_WDA_SETUP_HINT}" + ) + await asyncio.sleep(1.0) - async def launch_app(self, package_name: str) -> bool: - async with self._operation_lock: - self._require_connected() - raw = await run_xcrun( + async def _start_xctest_session(self, xctestrun: str) -> None: + """Hold WDA alive through ``xcodebuild test-without-building``.""" + path = Path(xctestrun).expanduser() + if path.suffix != ".xctestrun" or not path.is_file(): + raise RuntimeError( + f"{WDA_XCTESTRUN_ENV} must point at an existing .xctestrun file " + "produced by 'xcodebuild build-for-testing'." + ) + if self._wda_test_process is not None and self._wda_test_process.returncode is None: + return + self._wda_test_process = await asyncio.create_subprocess_exec( + "xcodebuild", + "test-without-building", + "-xctestrun", + str(path), + "-destination", + f"id={self._device_id}", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + + async def _tunnel_ip(self) -> str | None: + """CoreDevice's managed tunnel address for this device, when present.""" + try: + details = await self._devicectl_json("info", "details") + except (OSError, RuntimeError, TimeoutError, ValueError): + return None + connection = details.get("properties", {}).get("connection", {}) + address = connection.get("tunnelIPAddressString") + return address if isinstance(address, str) and address else None + + async def _wda_runner_bundle(self) -> str | None: + override = os.environ.get(WDA_BUNDLE_ENV) + try: + apps = await self._devicectl_json("info", "apps") + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + logger.debug(f"Could not list apps while looking for a WDA runner: {exc}") + return override or None + bundles = { + app.get("bundleIdentifier") or "" + for app in apps.get("apps", []) + if isinstance(app, dict) + } + if override: + return override if override in bundles else None + matches = sorted(b for b in bundles if WDA_RUNNER_PATTERN.search(b)) + return matches[0] if matches else None + + async def _launch_bundle( + self, bundle: str, terminate_existing: bool = True + ) -> int | None: + arguments = [ + "devicectl", + "device", + "process", + "launch", + "--device", + self._device_id, + ] + if terminate_existing: + arguments.append("--terminate-existing") + arguments += ["--json-output", "-", bundle] + raw = await run_xcrun(*arguments, timeout=DEVICECTL_LAUNCH_TIMEOUT) + return self._launched_pid(raw) + + async def _devicectl_json(self, *arguments: str) -> dict[str, Any]: + """Run a ``devicectl`` info subcommand and return its ``result`` JSON. + + ``--json-output -`` still writes a human table to stdout for ``info`` + subcommands, so the JSON must go to a scratch file. + """ + with tempfile.TemporaryDirectory(prefix="artemis-devicectl-") as tmp: + target = Path(tmp) / "out.json" + await run_xcrun( "devicectl", "device", - "process", - "launch", + *arguments, "--device", self._device_id, - "--terminate-existing", "--json-output", - "-", - package_name, - timeout=DEVICECTL_LAUNCH_TIMEOUT, + str(target), + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + payload = json.loads(target.read_text(encoding="utf-8")) + result = payload.get("result") + return result if isinstance(result, dict) else payload + + # --- Observation --- + + async def _screenshot_png(self) -> bytes: + """WDA screenshot first (same framebuffer as the hierarchy); devicectl fallback.""" + if self._wda is not None: + try: + return await self._wda.screenshot_png() + except (RuntimeError, WdaUnavailableError, OSError) as exc: + logger.debug(f"WDA screenshot failed, falling back to devicectl: {exc}") + with tempfile.TemporaryDirectory(prefix="artemis-shot-") as tmp: + target = Path(tmp) / "shot.png" + await run_xcrun( + "devicectl", + "device", + "capture", + "screenshot", + "--device", + self._device_id, + "--destination", + str(target), + timeout=DEVICECTL_QUERY_TIMEOUT, ) - pid = self._launched_pid(raw) + data = target.read_bytes() + if not data: + raise RuntimeError("devicectl produced an empty screenshot.") + return data + + async def _capture(self) -> ScreenData: + if self._wda is None: + raise RuntimeError("Connect the physical iOS driver before observing.") + screenshot = await self._screenshot_png() + with Image.open(BytesIO(screenshot)) as image: + self._width, self._height = image.size + win_w, win_h = await self._wda.window_size() + if win_w <= 0 or win_h <= 0: + raise RuntimeError("WebDriverAgent reported an unusable window size.") + if abs(self._width / win_w - self._height / win_h) > 0.05: + self._scale = None + raise RuntimeError( + "The device screenshot and hierarchy disagree on orientation. " + "Capture again after the rotation settles." + ) + self._scale = (self._width / win_w, self._height / win_h) + tree = await self._wda.source_json() + elements = parse_wda_elements( + tree, self._scale, self._width, self._height + ) + return ScreenData( + screenshot_bytes=screenshot, + screenshot_base64=base64.b64encode(screenshot).decode("ascii"), + ui_elements=elements, + width=self._width, + height=self._height, + platform="ios", + ) + + async def get_screen_data(self, skip_settling: bool = False) -> ScreenData: + async with self._operation_lock: + return await self._capture() + + def _wda_point(self, x: int, y: int) -> tuple[float, float]: + """Convert screenshot pixels to the WDA point coordinate space.""" + if self._scale is None: + raise RuntimeError("Capture an iOS screen before coordinate interaction.") + if not 0 <= x < self._width or not 0 <= y < self._height: + raise ValueError("iOS input coordinates are outside the current screenshot.") + return x / self._scale[0], y / self._scale[1] + + # --- Input --- + + async def tap( + self, x: int, y: int, duration_ms: int = 100, times: int = 1, delay_ms: int = 100 + ) -> bool: + if times < 1 or duration_ms < 0 or delay_ms < 0: + raise ValueError("Tap count must be positive and durations nonnegative.") + async with self._operation_lock: + previous_size = self.screen_size + await self._capture() + if self.screen_size != previous_size: + raise ValueError( + "The iOS screen changed orientation or size. Observe it again before tapping." + ) + point = self._wda_point(x, y) + for index in range(times): + await self._wda.tap(*point, hold_ms=duration_ms) + if index < times - 1: + await asyncio.sleep(delay_ms / 1000) + return True + + async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool: + return await self.tap(x, y, duration_ms=duration_ms) + + async def swipe( + self, start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 800 + ) -> bool: + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + previous_size = self.screen_size + await self._capture() + if self.screen_size != previous_size: + raise ValueError( + "The iOS screen changed orientation or size. Observe it again before swiping." + ) + start = self._wda_point(start_x, start_y) + end = self._wda_point(end_x, end_y) + await self._wda.swipe(*start, *end, duration_ms) + return True + + async def swipe_direction( + self, + direction: SwipeDirection | Literal["up", "down", "left", "right"], + duration_ms: int = 800, + ) -> bool: + direction = SwipeDirection(direction).value + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + await self._capture() + w, h = self.screen_size + points = { + "up": (w // 2, h * 3 // 4, w // 2, h // 4), + "down": (w // 2, h // 4, w // 2, h * 3 // 4), + "left": (w * 3 // 4, h // 2, w // 4, h // 2), + "right": (w // 4, h // 2, w * 3 // 4, h // 2), + } + start = self._wda_point(points[direction][0], points[direction][1]) + end = self._wda_point(points[direction][2], points[direction][3]) + await self._wda.swipe(*start, *end, duration_ms) + return True + + async def input_text(self, text: str, clear_existing: bool = True) -> bool: + if clear_existing: + raise NotImplementedError( + "Physical iOS typing appends to the focused field; clear it through the " + "UI or pass clear_existing=False. This matches the simulator behavior." + ) + async with self._operation_lock: + self._require_connected() + await self._wda.type_text(text) + return True + + async def press_key(self, key: KeyCode | str | int) -> bool: + key = key.value if isinstance(key, KeyCode) else str(key).lower() + async with self._operation_lock: + self._require_connected() + if key in ("home", "app_switch"): + presses = 2 if key == "app_switch" else 1 + for index in range(presses): + if not await self._wda.press_button("home"): + await self._wda.homescreen() + if index < presses - 1: + await asyncio.sleep(0.4) + elif key == "enter": + await self._wda.type_text("\n") + else: + buttons = { + "volume_up": "volumeUp", + "volume_down": "volumeDown", + "power": "power", + } + if key not in buttons or not await self._wda.press_button(buttons[key]): + raise NotImplementedError( + f"Key {key!r} is not supported by the physical iOS driver." + ) + return True + + async def tap_element( + self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 + ) -> bool: + async with self._operation_lock: + data = await self._capture() + element, center, error = await self.find_element(query, data) + if error or element is None or center is None: + return False + point = self._wda_point(*center) + await self._wda.tap( + *point, hold_ms=duration_ms if long_press else 0 + ) + return True + + async def find_element( + self, query: ElementQuery, screen_data: ScreenData | None = None + ) -> tuple[dict[str, Any] | None, list[int] | None, str | None]: + element, center, error = await super().find_element(query, screen_data) + if element is not None: + center = element.get("hit_point", center) + return element, center, error + + # --- App lifecycle via devicectl --- + + async def launch_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + pid = await self._launch_bundle(package_name) if pid is not None: self._launched_pids[package_name] = pid return True @@ -169,9 +568,12 @@ def _launched_pid(raw: bytes) -> int | None: except (ValueError, TypeError): payload = None if isinstance(payload, dict): - result = payload.get("result", {}) - for key in ("processIdentifier", "pid"): - pid = result.get(key) or payload.get(key) + result = payload.get("result") + result = result if isinstance(result, dict) else {} + process = result.get("process") + candidates = [c for c in (process, result, payload) if isinstance(c, dict)] + for candidate in candidates: + pid = candidate.get("processIdentifier") or candidate.get("pid") if isinstance(pid, int): return pid if isinstance(pid, str) and pid.isdigit(): @@ -185,25 +587,21 @@ async def _resolve_pid(self, package_name: str) -> int: if tracked is not None: return tracked try: - raw = await run_xcrun( - "devicectl", - "device", - "info", - "processes", - "--device", - self._device_id, - "--json-output", - "-", - timeout=DEVICECTL_QUERY_TIMEOUT, - ) - payload = json.loads(raw) - for process in payload.get("result", {}).get("runningProcesses", []): - executable = str( - process.get("executable") or process.get("bundleIdentifier") or "" + apps = await self._devicectl_json("info", "apps") + url_prefix = "" + for app in apps.get("apps", []): + if app.get("bundleIdentifier") == package_name: + url_prefix = str(app.get("url") or "").rstrip("/") + break + processes = await self._devicectl_json("info", "processes") + for process in processes.get("runningProcesses", []): + executable = str(process.get("executable") or "") + matched = ( + executable.startswith(url_prefix) + if url_prefix + else package_name in executable ) - if package_name in executable and isinstance( - process.get("processIdentifier"), int - ): + if matched and isinstance(process.get("processIdentifier"), int): return process["processIdentifier"] except (OSError, RuntimeError, TimeoutError, ValueError, TypeError) as exc: logger.debug(f"devicectl process enumeration failed: {exc}") @@ -212,21 +610,24 @@ async def _resolve_pid(self, package_name: str) -> int: "launch it with launch_app before stopping." ) + async def _terminate_pid(self, pid: int) -> None: + await run_xcrun( + "devicectl", + "device", + "process", + "terminate", + "--device", + self._device_id, + "--pid", + str(pid), + timeout=DEVICECTL_QUERY_TIMEOUT, + ) + async def stop_app(self, package_name: str) -> bool: async with self._operation_lock: self._require_connected() pid = await self._resolve_pid(package_name) - await run_xcrun( - "devicectl", - "device", - "process", - "terminate", - "--device", - self._device_id, - "--pid", - str(pid), - timeout=DEVICECTL_QUERY_TIMEOUT, - ) + await self._terminate_pid(pid) self._launched_pids.pop(package_name, None) return True @@ -266,21 +667,9 @@ async def install_app(self, app_path: Path) -> str: async def list_apps(self) -> dict[str, str]: async with self._operation_lock: self._require_connected() - raw = await run_xcrun( - "devicectl", - "device", - "info", - "apps", - "--device", - self._device_id, - "--json-output", - "-", - timeout=DEVICECTL_QUERY_TIMEOUT, - ) - payload = json.loads(raw) - apps = payload.get("result", {}).get("apps", []) + apps = await self._devicectl_json("info", "apps") result = {} - for app in apps: + for app in apps.get("apps", []): bundle = app.get("bundleIdentifier") or app.get("bundleID") if not bundle: continue @@ -302,6 +691,13 @@ async def open_url(self, url: str) -> bool: ) return True + async def get_current_package(self) -> str | None: + async with self._operation_lock: + self._require_connected() + if self._wda is not None: + return await self._wda.active_app() + return None + async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: raise NotImplementedError("Android shell commands are unavailable on iOS devices.") @@ -330,20 +726,3 @@ async def stop_video_recording(self) -> str | None: return None path = await self._physical_recorder.stop() return str(path) if path is not None else None - - async def disconnect(self) -> None: - try: - if self._physical_recorder is not None: - session = self._physical_recorder.session - if session is not None: - if session.is_active: - try: - await self._physical_recorder.stop() - except Exception as exc: - logger.error( - f"Physical iOS recording finalization failed during disconnect: {exc}" - ) - if get_active_session(self._device_id) is session: - remove_active_session(self._device_id) - finally: - await super().disconnect() diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py new file mode 100644 index 00000000..349ea56f --- /dev/null +++ b/artemis/drivers/ios/wda.py @@ -0,0 +1,380 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""WebDriverAgent HTTP client for physical iOS UI automation. + +Xcode's ``DeviceInteraction*`` MCP tools accept simulators only, so physical +devices are driven through WebDriverAgent — the same XCUITest bridge Appium +uses. The client speaks plain HTTP to the WDA server running on the device +and has no third-party dependencies; every call is offloaded to a thread so +the driver stays fully async. + +Reaching the device-side server needs one of: + +- ``ARTEMIS_IOS_WDA_URL``: an explicit endpoint such as + ``http://127.0.0.1:8100`` for ``iproxy``/``pymobiledevice3`` forwards or + ``http://:8100`` when the phone shares the LAN. +- The CoreDevice tunnel address from ``devicectl device info details`` — + paired devices already hold a managed IPv6 tunnel that routes TCP. +- ``ARTEMIS_IOS_WDA_HOST``: just a host/IP; port 8100 is assumed. +""" + +import asyncio +import base64 +import json +import os +from typing import Any +import urllib.error +import urllib.request + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +WDA_URL_ENV = "ARTEMIS_IOS_WDA_URL" +WDA_HOST_ENV = "ARTEMIS_IOS_WDA_HOST" +WDA_DEFAULT_PORT = 8100 +WDA_REQUEST_TIMEOUT = 30.0 + + +class WdaUnavailableError(RuntimeError): + """The WebDriverAgent server could not be reached or did not respond.""" + + +class WdaClient: + """Minimal WebDriverAgent client covering Artemis's interaction surface.""" + + def __init__(self, base_url: str, timeout: float = WDA_REQUEST_TIMEOUT): + self._base = base_url.rstrip("/") + self._timeout = timeout + self._session_id: str | None = None + + @property + def base_url(self) -> str: + return self._base + + @property + def session_id(self) -> str | None: + return self._session_id + + @property + def connected(self) -> bool: + return self._session_id is not None + + def _sync_request( + self, method: str, path: str, payload: dict[str, Any] | None, timeout: float + ) -> Any: + body = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request( + f"{self._base}{path}", + data=body, + method=method, + headers={"Content-Type": "application/json"} if body else {}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + raw = response.read() + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace")[:500] + raise RuntimeError(f"WebDriverAgent {method} {path} failed: HTTP {error.code} {detail}") + except (urllib.error.URLError, TimeoutError, OSError) as error: + raise WdaUnavailableError( + f"WebDriverAgent at {self._base} is unreachable: {error}. " + "Check the device connection, the WDA runner process, and any " + "port forwarding (iproxy or 'pymobiledevice3 remote')." + ) + try: + payload_out = json.loads(raw) + except ValueError: + return raw + if isinstance(payload_out, dict) and "value" in payload_out: + value = payload_out["value"] + if isinstance(value, dict) and value.get("error"): + message = value.get("message") or value["error"] + raise RuntimeError(f"WebDriverAgent {method} {path} failed: {message}") + return value + return payload_out + + async def _request( + self, + method: str, + path: str, + payload: dict[str, Any] | None = None, + timeout: float | None = None, + ) -> Any: + return await asyncio.to_thread( + self._sync_request, method, path, payload, timeout or self._timeout + ) + + # --- Session lifecycle --- + + async def status(self, timeout: float = 5.0) -> dict[str, Any] | None: + try: + value = await self._request("GET", "/status", timeout=timeout) + except (RuntimeError, WdaUnavailableError, OSError) as error: + logger.debug(f"WDA status probe failed for {self._base}: {error}") + return None + return value if isinstance(value, dict) else {"value": value} + + async def open_session(self) -> str: + payload = { + "capabilities": { + "alwaysMatch": {"platformName": "iOS"}, + "firstMatch": [{}], + } + } + value = await self._request("POST", "/session", payload, timeout=60.0) + session_id = None + if isinstance(value, dict): + session_id = value.get("sessionId") + if session_id is None and isinstance(value.get("capabilities"), dict): + session_id = value.get("capabilities", {}).get("sessionId") + if not isinstance(session_id, str) or not session_id: + raise RuntimeError("WebDriverAgent did not return a session id.") + self._session_id = session_id + return session_id + + async def close_session(self) -> None: + session_id, self._session_id = self._session_id, None + if session_id is None: + return + try: + await self._request("DELETE", f"/session/{session_id}", timeout=10.0) + except (RuntimeError, WdaUnavailableError, OSError) as error: + logger.debug(f"WDA session delete failed: {error}") + + # --- Observation --- + + async def screenshot_png(self) -> bytes: + value = await self._request("GET", "/screenshot") + if not isinstance(value, str) or not value: + raise RuntimeError("WebDriverAgent returned no screenshot data.") + return base64.b64decode(value) + + async def source_json(self) -> dict[str, Any]: + value = await self._request("GET", "/source?format=json") + if not isinstance(value, dict): + raise RuntimeError("WebDriverAgent returned a non-JSON hierarchy.") + return value + + async def window_size(self) -> tuple[float, float]: + path = f"/session/{self._session_id}/window/size" if self._session_id else "/window/size" + value = await self._request("GET", path) + if isinstance(value, dict) and "width" in value and "height" in value: + return float(value["width"]), float(value["height"]) + raise RuntimeError(f"WebDriverAgent returned an unexpected window size: {value!r}") + + async def active_app(self) -> str | None: + path = "/wda/activeAppInfo" + if self._session_id: + path = f"/session/{self._session_id}/wda/activeAppInfo" + try: + value = await self._request("GET", path) + except (RuntimeError, WdaUnavailableError): + return None + if isinstance(value, dict): + bundle = value.get("bundleId") or value.get("bundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + return None + + # --- Input --- + + async def _actions(self, pointer_actions: list[dict[str, Any]]) -> None: + session = self._require_session() + payload = { + "actions": [ + { + "type": "pointer", + "id": "artemis-finger", + "parameters": {"pointerType": "touch"}, + "actions": pointer_actions, + } + ] + } + await self._request("POST", f"/session/{session}/actions", payload) + + def _require_session(self) -> str: + if not self._session_id: + raise RuntimeError("WebDriverAgent session is not open.") + return self._session_id + + async def tap(self, x: float, y: float, hold_ms: int = 0) -> None: + actions: list[dict[str, Any]] = [ + {"type": "pointerMove", "duration": 0, "x": x, "y": y}, + {"type": "pointerDown", "button": 0}, + ] + if hold_ms > 0: + actions.append({"type": "pause", "duration": hold_ms}) + actions.append({"type": "pointerUp", "button": 0}) + await self._actions(actions) + + async def swipe(self, sx: float, sy: float, ex: float, ey: float, duration_ms: int) -> None: + await self._actions( + [ + {"type": "pointerMove", "duration": 0, "x": sx, "y": sy}, + {"type": "pointerDown", "button": 0}, + {"type": "pause", "duration": 50}, + {"type": "pointerMove", "duration": duration_ms, "x": ex, "y": ey}, + {"type": "pointerUp", "button": 0}, + ] + ) + + async def type_text(self, text: str) -> None: + session = self._require_session() + # ``value`` is a list of Unicode code points — newlines and non-ASCII + # text carry their literal values to the focused field. + payload = {"value": list(text)} + await self._request("POST", f"/session/{session}/wda/keys", payload) + + async def press_button(self, name: str) -> bool: + """WDA hardware buttons: home, volumeUp, volumeDown, power.""" + session = self._require_session() + try: + await self._request( + "POST", f"/session/{session}/wda/pressButton", {"name": name} + ) + return True + except (RuntimeError, WdaUnavailableError): + return False + + async def homescreen(self) -> None: + await self._request("POST", "/wda/homescreen") + + async def lock(self) -> None: + await self._request("POST", "/wda/lock") + + async def unlock(self) -> None: + await self._request("POST", "/wda/unlock") + + +def normalize_wda_url(raw: str) -> str: + """Accept bare hosts, IPv6 literals, or full URLs and return a base URL.""" + value = raw.strip() + if not value: + return value + if "://" not in value: + # Bare IPv6 literals need brackets once a port is attached. + if value.count(":") > 1 and not value.startswith("["): + value = f"[{value}]" + value = f"http://{value}" + value = value.rstrip("/") + authority = value.split("://", 1)[1] + host_port = authority.split("/", 1)[0] + # A port is present when ':' follows the host (or the IPv6 ']' bracket). + has_port = ( + host_port.rsplit("]", 1)[-1].startswith(":") + if "]" in host_port + else ":" in host_port + ) + if not has_port: + value = f"{value}:{WDA_DEFAULT_PORT}" + return value + + +def wda_url_candidates( + env_url: str | None = None, + env_host: str | None = None, + tunnel_ip: str | None = None, +) -> list[str]: + """Ordered endpoints to probe for a running WebDriverAgent server.""" + candidates: list[str] = [] + for raw in ( + env_url if env_url is not None else os.environ.get(WDA_URL_ENV), + env_host if env_host is not None else os.environ.get(WDA_HOST_ENV), + f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" if tunnel_ip else None, + f"127.0.0.1:{WDA_DEFAULT_PORT}", + ): + if not raw: + continue + url = normalize_wda_url(raw) + if url and url not in candidates: + candidates.append(url) + return candidates + + +async def probe_wda( + candidates: list[str], timeout: float = 5.0 +) -> WdaClient | None: + """Return a client bound to the first endpoint that answers ``/status``.""" + for url in candidates: + client = WdaClient(url) + status = await client.status(timeout=timeout) + if status is not None: + return client + return None + + +_ELEMENT_TYPE_PREFIX = "XCUIElementType" + + +def parse_wda_elements( + node: dict[str, Any], + scale: tuple[float, float], + width: int, + height: int, +) -> list[dict[str, Any]]: + """Flatten a WDA ``/source?format=json`` tree into Artemis ui_elements. + + Mirrors ``parse_hierarchy``'s output: ``text``, ``resource_id``, ``class``, + ``bounds``, ``parsed_bounds``, and ``hit_point`` in screenshot pixels. + WDA's ``isVisible`` is advisory; elements with usable geometry are kept so + downstream consumers see everything XCTest reports. + """ + elements: list[dict[str, Any]] = [] + + def visit(entry: dict[str, Any]) -> None: + rect = entry.get("rect") + if isinstance(rect, dict): + x = float(rect.get("x") or 0.0) + y = float(rect.get("y") or 0.0) + w = float(rect.get("width") or 0.0) + h = float(rect.get("height") or 0.0) + else: + x = y = w = h = 0.0 + if w > 0 and h > 0: + left, top = round(x * scale[0]), round(y * scale[1]) + right, bottom = round((x + w) * scale[0]), round((y + h) * scale[1]) + if right > 0 and bottom > 0 and left < width and top < height: + label = entry.get("label") + value = entry.get("value") + name = entry.get("name") + text = "" + if isinstance(label, str) and label: + text = label + elif value is not None and str(value): + text = str(value) + element_type = str(entry.get("type") or "") + element: dict[str, Any] = { + "text": text, + "resource_id": name if isinstance(name, str) else "", + "class": element_type.removeprefix(_ELEMENT_TYPE_PREFIX), + "bounds": f"[{left},{top}][{right},{bottom}]", + "parsed_bounds": { + "left": left, + "top": top, + "right": right, + "bottom": bottom, + }, + "hit_point": [(left + right) // 2, (top + bottom) // 2], + "visible": bool(entry.get("isVisible", True)), + } + if value is not None and str(value) != text: + element["value"] = str(value) + elements.append(element) + children = entry.get("children") + if isinstance(children, list): + for child in children: + if isinstance(child, dict): + visit(child) + + visit(node) + return elements diff --git a/docs/ios.md b/docs/ios.md index f6928782..150471b0 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -9,11 +9,14 @@ discovery, live screen streaming, and per-device locking that cannot collide with Android targets. The iOS drivers use tools included with Xcode and Artemis's existing Python MCP -dependency. They communicate with `xcrun mcpbridge` using an initialized MCP -session for screen observation and input. Simulators use `xcrun simctl` for -lifecycle operations; physical devices use `xcrun devicectl` (CoreDevice). -Appium, WebDriverAgent, and third-party device control utilities are not -required. Apple added native agent device interactions in Xcode 27; see the +dependency. Simulators communicate with `xcrun mcpbridge` using an initialized +MCP session for screen observation and input, and `xcrun simctl` for lifecycle +operations. Physical devices use `xcrun devicectl` (CoreDevice) for lifecycle +plus **WebDriverAgent (WDA)** for observation and input — Xcode's +`DeviceInteraction*` MCP tools accept simulators only. WDA is the same +XCUITest HTTP bridge Appium uses and must be built and installed on the device +once (see below). Apple added native agent device interactions in Xcode 27; +see the [Xcode 27 release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). ## Prepare Xcode @@ -95,10 +98,40 @@ uv run artemis run "Open Settings" \ ``` Physical UDIDs are routed to a separate driver: `devicectl` handles install, -launch, terminate, app listing, and URL opening, while the same Xcode -interaction session supplies screenshots, hierarchy, taps, swipes, and text. -`booted` remains a simulator-only selector — physical tasks always require an -explicit `--device-serial`. +launch, terminate, app listing, URL opening, and screenshot capture, while +the UI hierarchy and input (taps, swipes, text, buttons) go through +**WebDriverAgent**. `booted` remains a simulator-only selector — physical +tasks always require an explicit `--device-serial`. + +#### WebDriverAgent setup (one time per device) + +WDA is a signed XCTest runner app that exposes an HTTP automation endpoint on +port 8100. Build and install it once per device — Appium users can reuse an +existing WDA install: + +```bash +git clone https://github.com/appium/WebDriverAgent.git +cd WebDriverAgent +xcodebuild build-for-testing -project WebDriverAgent.xcodeproj \ + -scheme WebDriverAgentRunner -destination id= \ + -allowProvisioningUpdates DEVELOPMENT_TEAM= \ + PRODUCT_BUNDLE_IDENTIFIER=com.example.WebDriverAgentRunner +xcrun devicectl device install app --device \ + /Build/Products/Debug-iphoneos/WebDriverAgentRunner-Runner.app +``` + +Any Apple development team works; a free Personal Team profile must be +re-signed every 7 days while paid-program profiles last a year. + +When Artemis connects, it finds an installed `*WebDriverAgent*` runner on the +device, launches it via `devicectl process launch`, and probes its HTTP +endpoint on the CoreDevice tunnel address and `127.0.0.1:8100` (for +`iproxy`/`pymobiledevice3` port forwards). Endpoint overrides: + +- `ARTEMIS_IOS_WDA_URL` — full base URL, e.g. `http://127.0.0.1:8100` or + `http://:8100` +- `ARTEMIS_IOS_WDA_HOST` — host only; port 8100 assumed +- `ARTEMIS_IOS_WDA_BUNDLE_ID` — nonstandard runner bundle identifier Differences from simulators: @@ -229,10 +262,10 @@ instead of acquiring a second one. | Operation | Simulator | Physical device | | --- | --- | --- | | Device selection and readiness | `simctl` inventory, `boot`, `bootstatus` | `devicectl` inventory; must be paired and connected | -| Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | Same session (`deviceIsSimulator=false`) | -| Tap, long press, and swipe | Native synthesized touch events | Same | -| Text entry | Native keyboard synthesis with `clear_exist=false` | Same | -| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | Same | +| Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | WDA `/source` and `/screenshot` (devicectl screenshot fallback) | +| Tap, long press, and swipe | Native synthesized touch events | WDA W3C pointer actions | +| Text entry | Native keyboard synthesis with `clear_exist=false` | WDA `/wda/keys`, `clear_exist=false` | +| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | WDA `/wda/homescreen` and `/wda/pressButton` | | App install, launch, and terminate | `simctl` with simulator `.app` bundles | `devicectl` with signed `.app`/`.ipa`; terminate uses the launch PID or a live process scan | | Screen recording | `simctl io recordVideo` (VFR H.264) | `devicectl` screenshot polling assembled to timestamped MP4 | @@ -296,8 +329,17 @@ the timeline rather than stretching recorded frames. - **Physical install fails:** the artifact must be signed for the device — check the signing team and provisioning profile, or build an `.ipa` with `xcodebuild -exportArchive` for the device destination. -- **Physical input fails on a locked device:** unlock the device; the - interaction session cannot inject touches while locked. +- **Physical input fails on a locked device:** unlock the device; WDA cannot + inject touches while locked. +- **"No WebDriverAgent server answers":** no WDA endpoint responded on the + CoreDevice tunnel or `127.0.0.1:8100`, and no installed runner matching + `*WebDriverAgent*` was found to launch. Build and install the WDA runner + (one-time steps above), or point `ARTEMIS_IOS_WDA_URL` at a forwarded or + LAN-reachable server. +- **WDA runner launches but never answers:** the runner may be crashing on + launch — check provisioning (`get-task-allow`, matching certificate) and + whether a free-team profile expired; also try forwarding port 8100 + (`iproxy 8100 8100`) and setting `ARTEMIS_IOS_WDA_URL`. - **A queued iOS task runs on the wrong surface:** confirm the submission carried `platform: "ios"` (CLI `--platform ios`, web request `platform`, or the MCP `platform` argument); tasks default to Android. diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py index 13ec00a0..30302a0b 100644 --- a/tests/unit/drivers/test_ios_physical.py +++ b/tests/unit/drivers/test_ios_physical.py @@ -12,6 +12,9 @@ """Physical iOS device behavior tested without attached hardware.""" +import asyncio +import base64 +import io import json from pathlib import Path import plistlib @@ -19,9 +22,10 @@ import pytest -from artemis.drivers.ios import discovery, physical_driver, physical_recording +from artemis.drivers.ios import discovery, physical_driver, physical_recording, wda from artemis.drivers.ios.physical_driver import PhysicalIosDriver from artemis.drivers.ios.physical_recording import PhysicalIosRecorder +from artemis.drivers.ios.wda import WdaClient IPHONE_UDID = "00008130-000245193C60001C" @@ -193,39 +197,104 @@ async def test_connected_device_needs_no_boot(driver): await driver._prepare_device(_parsed(PHYSICAL_IPHONE)[0]) -def test_physical_session_rejects_simulator_sessions(driver): - with pytest.raises(RuntimeError, match="simulator session"): - driver._validate_session_device( - {"deviceIsSimulator": True, "deviceUUID": IPHONE_UDID} - ) +def _AsyncDevices(result): + async def _list(force_refresh=False): + return result + return _list -def test_physical_session_rejects_mismatched_udids(driver): - with pytest.raises(RuntimeError, match="different device"): - driver._validate_session_device( - {"deviceIsSimulator": False, "deviceUUID": IPAD_UDID} - ) - driver._validate_session_device( - {"deviceIsSimulator": False, "deviceUUID": IPHONE_UDID.lower()} - ) +def _xcrun_payload_writer(payloads): + """Fake run_xcrun that honors ``--json-output `` vs ``-``. + + ``payloads`` maps a devicectl subcommand ("apps", "processes", "details", + "launch", ...) to the dict written into the output file. Anything else + lands on stdout so ``--json-output -`` callers get JSON bytes. + """ + + async def fake(*arguments, timeout=30.0): + args = list(arguments) + if "--json-output" in args: + target = args[args.index("--json-output") + 1] + keys = [a for a in args if a in payloads] + payload = payloads.get(keys[0], {}) if keys else {} + if target == "-": + return json.dumps(payload).encode() + Path(target).write_text(json.dumps(payload), encoding="utf-8") + return b"" + return b"" -def test_physical_session_label_is_device_scoped(driver): - assert "Simulator" not in driver._session_label() + return fake -def _AsyncDevices(result): - async def _list(force_refresh=False): - return result +class _FakeWda: + """In-memory WebDriverAgent stand-in for driver interaction tests.""" - return _list + def __init__(self, window=(100.0, 200.0), tree=None): + self.session_id = None + self.window = window + self.tree = tree if tree is not None else {"type": "Application", "children": []} + self.tapped: list[tuple[float, float, int]] = [] + self.swiped: list[tuple[float, float, float, float, int]] = [] + self.typed: list[str] = [] + self.buttons: list[str] = [] + self.homescreen_calls = 0 + self._png = self._make_png() + self._closed = 0 + + @staticmethod + def _make_png(width: int = 300, height: int = 600) -> bytes: + from PIL import Image + + buffer = io.BytesIO() + Image.new("RGB", (width, height), color="red").save(buffer, format="PNG") + return buffer.getvalue() + + async def open_session(self): + self.session_id = "wda-session" + return self.session_id + + async def close_session(self): + self._closed += 1 + self.session_id = None + + async def screenshot_png(self): + return self._png + + async def window_size(self): + return self.window + + async def source_json(self): + return self.tree + + async def active_app(self): + return "com.example.foreground" + + async def tap(self, x, y, hold_ms=0): + self.tapped.append((x, y, hold_ms)) + + async def swipe(self, sx, sy, ex, ey, duration_ms): + self.swiped.append((sx, sy, ex, ey, duration_ms)) + + async def type_text(self, text): + self.typed.append(text) + + async def press_button(self, name): + self.buttons.append(name) + return True + + async def homescreen(self): + self.homescreen_calls += 1 @pytest.fixture def connected_driver(): - """A driver that has passed device resolution with a stub connection.""" + """A driver with a stub WDA session and a known 300x600 observation.""" driver = PhysicalIosDriver(device_id=IPHONE_UDID) - driver._session_key = "session" + driver._session_key = "wda-session" + driver._wda = _FakeWda() + # Pretend one capture already ran: the orientation guard compares sizes. + driver._width, driver._height = 300, 600 return driver @@ -235,7 +304,10 @@ async def test_launch_records_the_pid_from_devicectl_json(connected_driver, monk async def fake_xcrun(*arguments, timeout=30.0): calls.append(arguments) - return json.dumps({"result": {"processIdentifier": 4242}}).encode() + return json.dumps( + {"result": {"process": {"processIdentifier": 4242, + "executable": "file:///Apps/Example.app/Example"}}} + ).encode() monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) assert await connected_driver.launch_app("com.example.app") @@ -264,33 +336,58 @@ async def fake_xcrun(*arguments, timeout=30.0): @pytest.mark.asyncio async def test_stop_app_without_pid_scans_running_processes(connected_driver, monkeypatch): - processes = { - "result": { - "runningProcesses": [ - {"processIdentifier": 7, "executable": "com.example.other"}, - {"processIdentifier": 42, "executable": "com.example.app"}, - ] - } - } calls = [] async def fake_xcrun(*arguments, timeout=30.0): - calls.append(arguments) - return json.dumps(processes).encode() + args = list(arguments) + calls.append(args) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + if "apps" in args: + payload = { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + } + else: + payload = { + "result": { + "runningProcesses": [ + { + "processIdentifier": 7, + "executable": "file:///usr/libexec/other", + }, + { + "processIdentifier": 42, + "executable": "file:///var/containers/X/Example.app/Example", + }, + ] + } + } + Path(args[args.index("--json-output") + 1]).write_text(json.dumps(payload)) + return b"" monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) assert await connected_driver.stop_app("com.example.app") - assert calls[0][:4] == ("devicectl", "device", "info", "processes") - terminate = calls[1] + assert calls[0][2:4] == ["info", "apps"] + assert calls[1][2:4] == ["info", "processes"] + terminate = calls[2] + assert terminate[:5] == ["devicectl", "device", "process", "terminate", "--device"] assert terminate[terminate.index("--pid") + 1] == "42" @pytest.mark.asyncio async def test_stop_app_without_process_fails_clearly(connected_driver, monkeypatch): - async def fake_xcrun(*arguments, timeout=30.0): - return json.dumps({"result": {"runningProcesses": []}}).encode() - - monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + monkeypatch.setattr( + physical_driver, + "run_xcrun", + _xcrun_payload_writer({"apps": {"result": {"apps": []}}, + "processes": {"result": {"runningProcesses": []}}}), + ) with pytest.raises(ValueError, match="No running process"): await connected_driver.stop_app("com.example.app") @@ -351,16 +448,19 @@ async def fake_xcrun(*arguments, timeout=30.0): @pytest.mark.asyncio async def test_list_apps_parses_devicectl_json(connected_driver, monkeypatch): payload = { - "result": { - "apps": [ - {"bundleIdentifier": "com.example.writer", "name": "Writer"}, - {"bundleIdentifier": "com.example.other"}, - ] - } + "apps": [ + {"bundleIdentifier": "com.example.writer", "name": "Writer"}, + {"bundleIdentifier": "com.example.other"}, + ] } async def fake_xcrun(*arguments, timeout=30.0): - return json.dumps(payload).encode() + args = list(arguments) + if "--json-output" in args: + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps({"result": payload}) + ) + return b"" monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) apps = await connected_driver.list_apps() @@ -395,6 +495,9 @@ async def test_recorder_polls_frames_and_assembles_mp4(tmp_path, monkeypatch): ffmpeg_calls = [] async def fake_xcrun(*arguments, timeout=30.0): + # Pace like real devicectl round-trips; an instant fake lets the poll + # loop flood tmp_path with frames and stalls cleanup for minutes. + await asyncio.sleep(0.02) destination = Path(arguments[arguments.index("--destination") + 1]) _png(destination) return b"" @@ -427,6 +530,7 @@ async def fake_manifest(output_dir, paths, offsets): @pytest.mark.asyncio async def test_recorder_partitions_frames_at_the_seam(tmp_path, monkeypatch): async def fake_xcrun(*arguments, timeout=30.0): + await asyncio.sleep(0.02) destination = Path(arguments[arguments.index("--destination") + 1]) _png(destination) return b"" @@ -450,6 +554,7 @@ async def fake_xcrun(*arguments, timeout=30.0): @pytest.mark.asyncio async def test_recorder_fails_closed_when_devicectl_never_delivers(tmp_path, monkeypatch): async def fake_xcrun(*arguments, timeout=30.0): + await asyncio.sleep(0.01) raise RuntimeError("device disconnected") monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) @@ -462,6 +567,226 @@ async def fake_xcrun(*arguments, timeout=30.0): assert session.errors +# --- WebDriverAgent client and wiring -------------------------------------- + + +def test_normalize_wda_url_accepts_hosts_ips_and_urls(): + assert wda.normalize_wda_url("192.168.0.5") == "http://192.168.0.5:8100" + assert wda.normalize_wda_url("fd20::1") == "http://[fd20::1]:8100" + assert wda.normalize_wda_url("http://10.0.0.2:9000/") == "http://10.0.0.2:9000" + assert wda.normalize_wda_url("localhost:8100") == "http://localhost:8100" + + +def test_wda_url_candidates_order_env_host_tunnel_localhost(monkeypatch): + monkeypatch.delenv(wda.WDA_URL_ENV, raising=False) + monkeypatch.delenv(wda.WDA_HOST_ENV, raising=False) + candidates = wda.wda_url_candidates(tunnel_ip="fd20:85a4::1") + assert candidates == ["http://[fd20:85a4::1]:8100", "http://127.0.0.1:8100"] + monkeypatch.setenv(wda.WDA_URL_ENV, "http://10.1.1.1:8100") + monkeypatch.setenv(wda.WDA_HOST_ENV, "phone.lan") + assert wda.wda_url_candidates(tunnel_ip="fd20:85a4::1") == [ + "http://10.1.1.1:8100", + "http://phone.lan:8100", + "http://[fd20:85a4::1]:8100", + "http://127.0.0.1:8100", + ] + + +def test_parse_wda_elements_maps_tree_to_ui_elements(): + tree = { + "type": "XCUIElementTypeApplication", + "rect": {"x": 0, "y": 0, "width": 100, "height": 200}, + "children": [ + { + "type": "XCUIElementTypeButton", + "name": "saveButton", + "label": "Save", + "rect": {"x": 10, "y": 20, "width": 30, "height": 10}, + "isVisible": True, + "children": [], + }, + { + "type": "XCUIElementTypeStaticText", + "label": "Title", + "value": "hello", + "rect": {"x": 0, "y": 0, "width": 50, "height": 10}, + "children": [], + }, + {"type": "XCUIElementTypeOther", "rect": {"x": 0, "y": 0, "width": 0, "height": 0}}, + ], + } + elements = wda.parse_wda_elements(tree, (3.0, 3.0), 300, 600) + assert len(elements) == 3 # root window + two children with geometry + button = next(e for e in elements if e["resource_id"] == "saveButton") + assert button["text"] == "Save" + assert button["class"] == "Button" + assert button["parsed_bounds"] == {"left": 30, "top": 60, "right": 120, "bottom": 90} + assert button["hit_point"] == [75, 75] + label = next(e for e in elements if e["text"] == "Title") + assert label["value"] == "hello" + + +@pytest.mark.asyncio +async def test_wda_client_unwraps_values_and_raises_errors(monkeypatch): + client = WdaClient("http://wda.test:8100") + + class _Response: + def __init__(self, payload): + self._raw = json.dumps(payload).encode() + + def read(self): + return self._raw + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def fake_urlopen(request, timeout): + if request.full_url.endswith("/status"): + return _Response({"value": {"ready": True}}) + if request.full_url.endswith("/session"): + return _Response({"value": {"sessionId": "abc-123"}}) + return _Response({"value": {"error": "no such", "message": "nope"}}) + + monkeypatch.setattr(wda.urllib.request, "urlopen", fake_urlopen) + assert await client.status() == {"ready": True} + assert await client.open_session() == "abc-123" + with pytest.raises(RuntimeError, match="nope"): + await client.window_size() + + +@pytest.mark.asyncio +async def test_ensure_wda_returns_probed_client(driver, monkeypatch): + client = _FakeWda() + monkeypatch.setattr(physical_driver, "wda_url_candidates", lambda **kw: ["http://a:8100"]) + monkeypatch.setattr(physical_driver, "probe_wda", _async_return(client)) + monkeypatch.setattr(driver, "_tunnel_ip", _async_return("fd20::1")) + assert await driver._ensure_wda() is client + + +@pytest.mark.asyncio +async def test_ensure_wda_launches_runner_then_probes(driver, monkeypatch): + client = _FakeWda() + probes = [] + + async def probe(candidates, timeout=5.0): + probes.append(candidates) + return client if len(probes) > 1 else None + + async def apps_payload(*arguments): + return {"apps": [{"bundleIdentifier": "com.artemis.WebDriverAgentRunner.xctrunner"}]} + + monkeypatch.setattr(physical_driver, "wda_url_candidates", lambda **kw: ["http://a:8100"]) + monkeypatch.setattr(physical_driver, "probe_wda", probe) + monkeypatch.setattr(driver, "_tunnel_ip", _async_return(None)) + monkeypatch.setattr(driver, "_devicectl_json", apps_payload) + monkeypatch.setattr(driver, "_launch_bundle", _async_return(777)) + assert await driver._ensure_wda() is client + assert driver._wda_runner_pid == 777 + + +@pytest.mark.asyncio +async def test_ensure_wda_without_runner_or_server_fails(driver, monkeypatch): + monkeypatch.setattr(physical_driver, "wda_url_candidates", lambda **kw: ["http://a:8100"]) + monkeypatch.setattr(physical_driver, "probe_wda", _async_return(None)) + monkeypatch.setattr(driver, "_tunnel_ip", _async_return(None)) + monkeypatch.setattr(driver, "_devicectl_json", _async_return({"apps": []})) + with pytest.raises(RuntimeError, match="WebDriverAgent"): + await driver._ensure_wda() + + +@pytest.mark.asyncio +async def test_capture_maps_wda_tree_and_screenshot(connected_driver): + connected_driver._wda = _FakeWda( + window=(100.0, 200.0), + tree={ + "type": "XCUIElementTypeApplication", + "rect": {"x": 0, "y": 0, "width": 100, "height": 200}, + "children": [ + { + "type": "XCUIElementTypeButton", + "label": "OK", + "rect": {"x": 10, "y": 10, "width": 20, "height": 20}, + } + ], + }, + ) + data = await connected_driver.get_screen_data() + assert data.platform == "ios" + assert (data.width, data.height) == (300, 600) + assert connected_driver._scale == (3.0, 3.0) + button = next(e for e in data.ui_elements if e["text"] == "OK") + assert button["hit_point"] == [60, 60] + + +@pytest.mark.asyncio +async def test_tap_converts_pixels_to_wda_points(connected_driver): + await connected_driver.tap(150, 300) + assert connected_driver._wda.tapped == [(50.0, 100.0, 100)] + + +@pytest.mark.asyncio +async def test_swipe_converts_endpoints(connected_driver): + await connected_driver.swipe(30, 300, 150, 60, duration_ms=500) + assert connected_driver._wda.swiped == [(10.0, 100.0, 50.0, 20.0, 500)] + + +@pytest.mark.asyncio +async def test_input_text_and_keys_route_to_wda(connected_driver): + await connected_driver.input_text("hi", clear_existing=False) + assert connected_driver._wda.typed == ["hi"] + await connected_driver.press_key("volume_up") + assert connected_driver._wda.buttons == ["volumeUp"] + await connected_driver.press_key("home") + assert connected_driver._wda.buttons == ["volumeUp", "home"] + with pytest.raises(NotImplementedError, match="typing appends"): + await connected_driver.input_text("x", clear_existing=True) + + +@pytest.mark.asyncio +async def test_current_package_uses_wda_active_app(connected_driver): + assert await connected_driver.get_current_package() == "com.example.foreground" + + +@pytest.mark.asyncio +async def test_disconnect_closes_wda_and_terminates_runner(connected_driver, monkeypatch): + connected_driver._wda_runner_pid = 777 + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + await connected_driver.disconnect() + assert connected_driver._wda is None + assert connected_driver._session_key is None + assert connected_driver._wda_runner_pid is None + assert any( + "terminate" in args and str(777) in args for args in calls + ) + + +@pytest.mark.asyncio +async def test_connect_establishes_wda_session(driver, monkeypatch): + client = _FakeWda() + monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(PHYSICAL_IPHONE))) + monkeypatch.setattr(driver, "_ensure_wda", _async_return(client)) + monkeypatch.setattr(driver, "_require_ios_host", _async_return(None)) + await driver.connect() + assert driver._wda is client + assert driver._session_key == "wda-session" + + +def _async_return(value): + async def _inner(*args, **kwargs): + return value + + return _inner + + # --- Factory routing -------------------------------------------------------- From 9be6579efb2e54f3d38610457824412094cb3905 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 03:26:33 +0900 Subject: [PATCH 19/32] docs+setup: physical smoke-test wording and devicectl prerequisite check Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/core/diagnostics/device_smoke.py | 5 +++-- scripts/setup_ios_env.sh | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 691467d8..15c06a91 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -309,8 +309,9 @@ async def smoke_test_device( ) -> dict[str, Any]: """Observe the device exactly like ``mobile_get_device_state`` and report a verdict. - ``platform="ios"`` exercises the native Xcode 27 simulator path - (``simctl`` + mcpbridge session) instead of ADB/UIAutomator. + ``platform="ios"`` exercises the simulator path (``simctl`` + mcpbridge + session) when the target is a simulator UDID, or the physical path + (``devicectl`` + WebDriverAgent) when the target is paired hardware. Never raises. Returns:: diff --git a/scripts/setup_ios_env.sh b/scripts/setup_ios_env.sh index 7f82fd14..65ef41c5 100755 --- a/scripts/setup_ios_env.sh +++ b/scripts/setup_ios_env.sh @@ -61,7 +61,23 @@ if ! xcrun mcp-server status; then printf 'Could not read MCP server status. Check Xcode MCP access manually.\n' >&2 fi +if ! devicectl_path=$(xcrun --find devicectl); then + fail "The selected Xcode installation does not provide devicectl (needed for physical devices)." +fi +printf 'devicectl: %s\n\n' "$devicectl_path" + +printf 'Physical devices (devicectl inventory):\n' +if physical=$(xcrun devicectl list devices 2>/dev/null); then + printf '%s\n' "$physical" | awk 'NR > 2 && NF' + if ! printf '%s\n' "$physical" | grep -q 'paired'; then + printf 'No paired physical devices found. Pair over USB and tap Trust to use hardware.\n' + fi +else + printf 'Could not query CoreDevice. Physical-device runs need a paired, connected device.\n' >&2 +fi + printf '\nToolchain and simulator inventory checks passed.\n' printf 'Before running Artemis, review and grant Xcode MCP access for your agent.\n' printf 'This check does not verify that device-interaction permission is granted.\n' +printf 'Physical devices additionally need a WebDriverAgent runner installed; see docs/ios.md.\n' printf 'See docs/ios.md and https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode\n' From b96c45d738b9a42f48a91024fccebcb17c196aa3 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 04:10:25 +0900 Subject: [PATCH 20/32] Address product review: physical auto-select, async stall, WDA/recording hardening - ios_device_pool: never auto-select physical hardware; validators match by UDID or name and fail open when either enumeration is indeterminate - sdk/agent: pick the iOS driver class in a worker thread so simctl/ devicectl subprocesses never block the event loop during init - physical_driver: tunnel-IP reads every observed devicectl JSON shape; exclusive runner/xctestrun hosting; broad connect cleanup; --kill terminate with stale-PID fallback; normalize file:// executables; IPA bundle id read from Payload Info.plist; scale sanity guard - wda: map http.client.HTTPException, propagate transport errors from press_button, IPv6-safe URL normalization, bracket only v6 candidates - physical_recording: monotonic frame indices across segment rolls, shielded stop, broad poll-loop failure accounting, watchdog notices a dead poller, ffmpeg timeout, atomic .part cleanup - discovery: null-safe property reads, iPadOS classification - diagnose/probe: physical serials reachable via mobile_diagnose; pool auto-pick stays simulator-only; platform-aware smoke labels - Sweep stale 'simulator' wording across MCP tools, routers, schemas, daemon client, setup script, and docs; document XCTESTRUN + UI Automation consent; tests: generic fixtures, WDA env pinning, deterministic recorder seams Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/admin_console/replay_manager.py | 9 +- apps/admin_console/routers/system.py | 4 +- apps/admin_console/routers/tasks.py | 4 +- apps/admin_console/schemas/task_schema.py | 2 +- .../services/device_stream_service.py | 4 +- .../services/task_queue_service.py | 4 +- artemis/agents/platform_guidance.py | 4 +- artemis/core/diagnostics/device_smoke.py | 28 ++- artemis/core/diagnostics/probes/ios_probe.py | 11 +- artemis/drivers/ios/__init__.py | 2 +- artemis/drivers/ios/discovery.py | 29 ++- artemis/drivers/ios/physical_driver.py | 182 +++++++++++++----- artemis/drivers/ios/physical_recording.py | 49 ++++- artemis/drivers/ios/wda.py | 39 +++- artemis/interfaces/cli/commands/batch.py | 8 +- artemis/interfaces/cli/commands/run.py | 10 +- artemis/mcp/adb_server.py | 4 +- artemis/runtime/adb_endpoint.py | 1 + artemis/runtime/daemon_client.py | 6 +- artemis/runtime/ios_device_pool.py | 86 ++++++--- artemis/sdk/agent.py | 9 +- artemis/sdk/builders/agent_config_builder.py | 2 +- docs/ios.md | 47 +++-- mcp_server/tools/device_state.py | 4 +- mcp_server/tools/diagnose.py | 28 ++- mcp_server/tools/task_runner.py | 18 +- scripts/setup_ios_env.sh | 2 +- tests/unit/drivers/test_ios_physical.py | 29 ++- tests/unit/test_ios_integration.py | 2 +- tests/unit/test_ios_parity.py | 42 ++++ 30 files changed, 475 insertions(+), 194 deletions(-) diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index cc6e741b..fcccecba 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -1649,10 +1649,11 @@ def _replay_device_context(device_info: dict, override_device_id: str | None): """Build the simulated ``DeviceContext`` for a replayed session. The recorded session decides the platform: an iOS session replays - through the Xcode driver against a simulator, never through ADB. The - frontend's device pick only retargets iOS replays (the picker was - always decorative for Android and stays that way); the driver still - validates the UDID against simctl at connect time. + through the native iOS driver, never through ADB. The frontend's + device pick only retargets iOS replays (the picker was always + decorative for Android and stays that way); the driver still + validates the UDID — simctl for simulators, devicectl for physical + devices — at connect time. """ from artemis.context import DeviceContext, DevicePlatform diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index 70b1ace9..f4fef2a4 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -98,7 +98,7 @@ class SelectDeviceRequest(BaseModel): serial: str = Field(description="Serial number or identifier of the device to select") platform: str = Field( default="android", - description="Device platform: 'android' (default) or 'ios' (simulator UDID)", + description="Device platform: 'android' (default) or 'ios' (device UDID)", ) @@ -110,7 +110,7 @@ async def get_system_readiness(force: bool = False) -> SystemReadinessReport: @router.post("/devices/select") async def select_active_device(request: SelectDeviceRequest): - """Select the active device or iOS simulator for subsequent automated tasks.""" + """Select the active Android or iOS device for subsequent automated tasks.""" serial = request.serial.strip() if not serial: raise HTTPException(status_code=400, detail="Device serial cannot be empty.") diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index e6776453..05c16350 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -122,7 +122,7 @@ async def run_task(request: RunRequest): if platform == "ios" and request.locked_app_package: raise HTTPException( status_code=400, - detail="locked_app_package is unavailable for iOS simulator tasks.", + detail="locked_app_package is unavailable for iOS tasks.", ) # Reject an explicit unknown/offline target before running the more @@ -223,7 +223,7 @@ async def get_run_defaults(): @router.get("/api/devices") async def list_devices(): - """List all connected devices (Android + iOS simulators) with lock status.""" + """List all connected devices (Android + iOS) with lock status.""" android_devices, ios_devices = await asyncio.gather( device_pool.list_devices_async(), ios_device_pool.list_devices_async() ) diff --git a/apps/admin_console/schemas/task_schema.py b/apps/admin_console/schemas/task_schema.py index 6843dba4..e0ee9cbe 100644 --- a/apps/admin_console/schemas/task_schema.py +++ b/apps/admin_console/schemas/task_schema.py @@ -30,7 +30,7 @@ class RunRequest(BaseModel): app_path: str | None = None device_serial: str | None = None # Target platform: "android" (default) or "ios". For iOS, device_serial is a - # simulator UDID and ios_workspace optionally scopes the Xcode approval grant. + # simulator or physical UDID and ios_workspace optionally scopes the Xcode approval grant. platform: str | None = None ios_workspace: str | None = None ingress: str | None = "frontend" diff --git a/apps/admin_console/services/device_stream_service.py b/apps/admin_console/services/device_stream_service.py index 4c6157b8..08334df7 100644 --- a/apps/admin_console/services/device_stream_service.py +++ b/apps/admin_console/services/device_stream_service.py @@ -15,7 +15,7 @@ """Device Live Screen Streaming Service. Provides real-time, low-latency device screen frames over HTTP MJPEG and WebSocket. -Android frames come from ``adb exec-out screencap``; iOS Simulator frames come +Android frames come from ``adb exec-out screencap``; iOS frames come from the native ``xcrun simctl io screenshot`` capture. The target is picked per frame so the stream follows whichever platform is under automation. """ @@ -49,7 +49,7 @@ def __init__(self): async def get_stream_target(self) -> dict[str, str] | None: """Pick the device the stream should follow right now. - An actively locked iOS simulator wins (it is the device under + An actively locked iOS device wins (it is the device under automation), then a connected Android device, then the single unambiguous booted simulator. Returns ``{"platform", "serial"}``. """ diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index 0cefab5c..b27a97cc 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -1084,8 +1084,8 @@ async def enqueue_tasks( forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``; they are normalised here so the queue item and the CLI see one spelling. - ``platform="ios"`` enqueues a native iOS Simulator task: the serial is a - simulator UDID, the device lock is scoped under ``ios``, and the worker + ``platform="ios"`` enqueues a native iOS task: the serial is a simulator + or paired physical UDID, the device lock is scoped under ``ios``, and the worker runs with ``--platform ios`` instead of ADB bindings. """ platform = str(platform or "android").strip().lower() or "android" diff --git a/artemis/agents/platform_guidance.py b/artemis/agents/platform_guidance.py index e44dcb8c..8418cece 100644 --- a/artemis/agents/platform_guidance.py +++ b/artemis/agents/platform_guidance.py @@ -19,9 +19,9 @@ def device_action_guidance(ctx) -> str: if getattr(getattr(ctx, "device", None), "mobile_platform", None) != "ios": return "" return ( - "Target platform: iOS Simulator. For input_text, explicitly set clear_exist=false; " + "Target platform: iOS. For input_text, explicitly set clear_exist=false; " "type into an empty field or at its existing cursor. Whole-field clearing, Android " - "keycodes, Back/Delete keys, ADB commands, video analysis and app locking are " + "keycodes, Back/Delete keys, ADB commands, and app locking are " "unavailable. Navigate with visible iOS controls; press_key supports enter, home, " "power, volume_up, volume_down, and app_switch. manage_app accepts installed iOS " "display names or bundle identifiers. Action coordinates use the screenshot's " diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 15c06a91..ed338ba7 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -125,6 +125,12 @@ def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: "Attach the iPhone/iPad over USB or ensure network pairing is reachable.", f"Verify connectivity: xcrun devicectl list devices (expect 'connected' for {s})", ] + if "webdriveragent" in text or "wda" in text: + return [ + "Install a signed WebDriverAgent runner on the device (see docs/ios.md).", + "Approve the device's UI Automation passcode prompt, then retry.", + "Override the endpoint with ARTEMIS_IOS_WDA_URL when forwarding via iproxy/pymobiledevice3.", + ] if "simulator" in text or "udid" in text or "boot" in text or "not available" in text: return [ f"List simulators: xcrun simctl list devices; boot one with: xcrun simctl boot {s}", @@ -317,7 +323,7 @@ async def smoke_test_device( {"ok": bool, "serial": str | None, "elapsed_seconds": float, "screenshot_bytes": int | None, "element_count": int | None, - "hierarchy_backend": "helper" | "uiautomator" | "xcode" | None, + "hierarchy_backend": "helper" | "uiautomator" | "xcode" | "webdriveragent" | None, "error": str | None, "fix": list[str]} """ started = time.monotonic() @@ -369,14 +375,16 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any "controller-init", ) except TimeoutError: + label = "iOS driver" if is_ios else "UIAutomator/controller" return _finish( - f"UIAutomator/controller initialization did not respond within {timeout_seconds:g}s" + f"{label} initialization did not respond within {timeout_seconds:g}s" ) except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught cause = _describe_failure(exc) - return _finish(f"Failed to initialize Android device controller: {cause}", cause) + target = "iOS device" if is_ios else "Android device" + return _finish(f"Failed to initialize {target} controller: {cause}", cause) controller_ctx = getattr(controller, "ctx", None) device = getattr(controller_ctx, "device", None) @@ -399,7 +407,8 @@ async def _observe() -> Any: "screen-data", ) except TimeoutError: - return _finish(f"UIAutomator/screen capture did not respond within {timeout_seconds:g}s") + source = "native iOS" if is_ios else "UIAutomator/" + return _finish(f"{source}screen capture did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught @@ -409,7 +418,13 @@ async def _observe() -> Any: result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None)) result["element_count"] = _count_elements(getattr(device_data, "elements", None)) if is_ios: - result["hierarchy_backend"] = "xcode" + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + result["hierarchy_backend"] = ( + "webdriveragent" + if isinstance(getattr(controller, "_driver", None), PhysicalIosDriver) + else "xcode" + ) else: from artemis.clients.screen_client_factory import describe_backend @@ -424,8 +439,9 @@ async def _observe() -> Any: f"Screenshot capture failed (driver returned a {result['screenshot_bytes']}-byte placeholder image)" ) if not result["element_count"]: + source = "Hierarchy" if is_ios else "UIAutomator hierarchy" return _finish( - "UIAutomator hierarchy dump returned no UI elements (screenshot worked, hierarchy did not)" + f"{source} dump returned no UI elements (screenshot worked, hierarchy did not)" ) return _finish(None) diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py index 5eaaf057..b87b0272 100644 --- a/artemis/core/diagnostics/probes/ios_probe.py +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -10,12 +10,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""iOS Simulator Readiness Probe. +"""iOS Device Readiness Probe. Informational only (``is_blocker = False``): iOS support is opt-in per task, so a missing Xcode toolchain must never block the Android default path. The -probe reports whether this host could run an iOS Simulator task today — -macOS, Xcode 27+, and at least one available simulator. +probe reports whether this host could run an iOS task today — macOS, Xcode +27+, and at least one available simulator or paired+connected physical +device. """ import asyncio @@ -57,7 +58,7 @@ async def probe(self) -> ProbeResult: return ProbeResult( id=self.probe_id, category=self.category, - title="iOS Simulator", + title="iOS Devices", status=ProbeStatus.SKIPPED, is_blocker=self.is_blocker, summary="Unavailable", @@ -106,7 +107,7 @@ async def probe(self) -> ProbeResult: return ProbeResult( id=self.probe_id, category=self.category, - title="iOS Simulator", + title="iOS Devices", status=ProbeStatus.WARN, is_blocker=self.is_blocker, summary="Xcode 27+ Required", diff --git a/artemis/drivers/ios/__init__.py b/artemis/drivers/ios/__init__.py index 1c23238d..40da1ece 100644 --- a/artemis/drivers/ios/__init__.py +++ b/artemis/drivers/ios/__init__.py @@ -9,4 +9,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Native Xcode simulator drivers (loaded only when iOS is selected).""" +"""Native iOS drivers: Xcode simulators and paired physical devices (loaded only when iOS is selected).""" diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py index 105bbe99..c70f3247 100644 --- a/artemis/drivers/ios/discovery.py +++ b/artemis/drivers/ios/discovery.py @@ -10,13 +10,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared ``simctl`` device enumeration for iOS Simulator support. +"""Shared iOS device enumeration: ``simctl`` for simulators, ``devicectl`` for physical. Both the Xcode driver (target validation) and the runtime pools (discovery, explicit-serial validation, auto-selection) read the same ``xcrun simctl list devices --json`` output through this module so parsing -stays in one place. Every function fails closed to ``None``/``[]`` on -missing tooling so Android-only hosts never see iOS errors. +stays in one place; paired physical hardware is enumerated through +``xcrun devicectl list devices``. Every function fails closed to +``None``/``[]`` on missing tooling so Android-only hosts never see iOS errors. """ import asyncio @@ -197,17 +198,31 @@ def _device_property(device: dict[str, Any], section: str, key: str) -> Any: Xcode marks ``hardwareProperties``/``deviceProperties``/``connectionProperties`` deprecated in favor of a nested ``properties`` dictionary; accept both. + Sections may be present-but-null, so guard every hop. """ - value = device.get(section, {}).get(key) + parent = device.get(section) + value = parent.get(key) if isinstance(parent, dict) else None if value is not None: return value - return device.get("properties", {}).get(section, {}).get(key) + properties = device.get("properties") + if not isinstance(properties, dict): + return None + nested = properties.get(section) + return nested.get(key) if isinstance(nested, dict) else None def parse_devicectl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: """Flatten ``devicectl list devices --json-output -`` into device entries.""" devices = [] - for device in payload.get("result", {}).get("devices", []): + result = payload.get("result") + if not isinstance(result, dict): + return devices + entries = result.get("devices") + if not isinstance(entries, list): + return devices + for device in entries: + if not isinstance(device, dict): + continue udid = _device_property(device, "hardwareProperties", "udid") if not udid: continue @@ -292,7 +307,7 @@ def list_core_devices_sync( def is_physical_ios(device: dict[str, Any]) -> bool: """Whether a CoreDevice entry is a physical iPhone/iPad (not a simulator).""" return ( - device.get("platform") == "iOS" + device.get("platform") in ("iOS", "iPadOS") and device.get("reality") == "physical" ) diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py index e63fec9e..bef40e40 100644 --- a/artemis/drivers/ios/physical_driver.py +++ b/artemis/drivers/ios/physical_driver.py @@ -28,6 +28,7 @@ import asyncio import base64 +import contextlib from io import BytesIO import json import os @@ -158,7 +159,10 @@ async def _prepare_device(self, candidate: dict[str, Any]) -> None: raise RuntimeError( f"iOS device {self._device_id} is not paired. Connect it and tap Trust." ) - if candidate.get("connection_state") != "connected": + connection_state = candidate.get("connection_state") + # Older iOS versions over USB expose no CoreDevice tunnelState — an + # absent value on a paired device is acceptable; "disconnected" is not. + if connection_state is not None and connection_state != "connected": raise RuntimeError( f"iOS device {self._device_id} ({candidate.get('name') or 'unknown'}) is not " "connected. Attach it over USB or ensure network pairing is reachable; " @@ -175,13 +179,9 @@ async def connect(self) -> None: self._wda = await self._ensure_wda() self._session_key = await self._wda.open_session() await self.get_screen_data(skip_settling=True) - except ( - OSError, - ValueError, - RuntimeError, - TimeoutError, - asyncio.CancelledError, - ): + # Any failure here must release a half-started WDA runner, + # xcodebuild session, or server-side session. + except (Exception, asyncio.CancelledError): try: await self.disconnect() except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: @@ -196,29 +196,33 @@ async def disconnect(self) -> None: if self._physical_recorder is not None: session = self._physical_recorder.session if session is not None: - if session.is_active: - try: - await self._physical_recorder.stop() - except Exception as exc: - logger.error( - f"Physical iOS recording finalization failed during disconnect: {exc}" - ) + # A failed (non-active) session still owns frames and + # conversions — finalize it too so nothing is lost. + try: + await self._physical_recorder.stop() + except Exception as exc: + logger.error( + f"Physical iOS recording finalization failed during disconnect: {exc}" + ) if get_active_session(self._device_id) is session: remove_active_session(self._device_id) finally: async with self._operation_lock: self._session_key = None + self._scale = None + self._launched_pids.clear() client, self._wda = self._wda, None runner_pid, self._wda_runner_pid = self._wda_runner_pid, None test_process, self._wda_test_process = self._wda_test_process, None if client is not None: await client.close_session() if test_process is not None and test_process.returncode is None: - test_process.terminate() try: + test_process.terminate() await asyncio.wait_for(test_process.wait(), timeout=10.0) - except (TimeoutError, asyncio.TimeoutError): - test_process.kill() + except (TimeoutError, OSError): + with contextlib.suppress(OSError): + test_process.kill() if runner_pid is not None: try: await self._terminate_pid(runner_pid) @@ -243,36 +247,45 @@ async def _ensure_wda(self) -> WdaClient: client = await probe_wda(candidates) if client is not None: return client - runner = await self._wda_runner_bundle() - if runner is not None: - self._wda_runner_pid = await self._launch_bundle( - runner, terminate_existing=False - ) - logger.info( - f"Launched WebDriverAgent runner {runner} on {self._device_id}; " - "waiting for its HTTP server" - ) xctestrun = os.environ.get(WDA_XCTESTRUN_ENV) + runner = await self._wda_runner_bundle() + if xctestrun: + await self._start_xctest_session(xctestrun) + elif runner is not None: + try: + self._wda_runner_pid = await self._launch_bundle( + runner, terminate_existing=False + ) + logger.info( + f"Launched WebDriverAgent runner {runner} on {self._device_id}; " + "waiting for its HTTP server" + ) + except (OSError, RuntimeError, TimeoutError) as exc: + # A stale runner may already be hosting the server — the + # probe loop below still gets a chance to attach. + logger.debug(f"WDA runner launch failed ({runner}): {exc}") if runner is None and not xctestrun: raise RuntimeError( f"No WebDriverAgent server answers on {self._device_id} and no WDA " f"runner is installed. {_WDA_SETUP_HINT}" ) - if xctestrun: - await self._start_xctest_session(xctestrun) deadline = asyncio.get_running_loop().time() + WDA_START_TIMEOUT while True: - client = await probe_wda(candidates) - if client is not None: - return client - if asyncio.get_running_loop().time() >= deadline: + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: raise RuntimeError( f"WebDriverAgent did not answer within {WDA_START_TIMEOUT:.0f}s " f"on {self._device_id} at {candidates}. If the device is showing " "a passcode prompt to enable UI Automation, enter it on the " f"device first. {_WDA_SETUP_HINT}" ) - await asyncio.sleep(1.0) + client = await probe_wda(candidates, timeout=min(5.0, remaining)) + if client is not None: + logger.info( + f"WebDriverAgent attached at {client.base_url} for {self._device_id}" + ) + return client + await asyncio.sleep(min(1.0, remaining)) async def _start_xctest_session(self, xctestrun: str) -> None: """Hold WDA alive through ``xcodebuild test-without-building``.""" @@ -301,9 +314,21 @@ async def _tunnel_ip(self) -> str | None: details = await self._devicectl_json("info", "details") except (OSError, RuntimeError, TimeoutError, ValueError): return None - connection = details.get("properties", {}).get("connection", {}) - address = connection.get("tunnelIPAddressString") - return address if isinstance(address, str) and address else None + # The tunnel address has moved across Xcode releases: probe every + # observed shape, canonical first. + paths = ( + ((details.get("connectionProperties") or {}), "tunnelIPAddress"), + ( + ((details.get("properties") or {}).get("connection") or {}), + "tunnelIPAddressString", + ), + ((details.get("tunnel") or {}), "ipAddress"), + ) + for section, key in paths: + address = section.get(key) + if isinstance(address, str) and ":" in address: + return address + return None async def _wda_runner_bundle(self) -> str | None: override = os.environ.get(WDA_BUNDLE_ENV) @@ -403,7 +428,18 @@ async def _capture(self) -> ScreenData: "The device screenshot and hierarchy disagree on orientation. " "Capture again after the rotation settles." ) - self._scale = (self._width / win_w, self._height / win_h) + scale = (self._width / win_w, self._height / win_h) + if not (0.9 <= scale[0] <= 4.5 and 0.9 <= scale[1] <= 4.5): + # iOS displays render at 1x-3x; a wildly off scale means the WDA + # window is not full-screen (e.g. iPad multitasking) and every + # element bound would be wrong. + self._scale = None + raise RuntimeError( + f"WebDriverAgent window {win_w}x{win_h} does not match the " + f"{self._width}x{self._height} framebuffer (scale {scale}); " + "bring the session app full-screen before interacting." + ) + self._scale = scale tree = await self._wda.source_json() elements = parse_wda_elements( tree, self._scale, self._width, self._height @@ -581,6 +617,15 @@ def _launched_pid(raw: bytes) -> int | None: match = re.search(r"pid[:= ]+(\d+)", raw.decode(errors="replace")) return int(match.group(1)) if match else None + @staticmethod + def _normalize_executable(raw: Any) -> str: + """devicectl reports executables as file:// URLs or paths; normalize.""" + value = raw if isinstance(raw, str) else str(raw or "") + if isinstance(raw, dict): + value = str(raw.get("url") or raw.get("path") or "") + value = value.removeprefix("file://") + return value.removeprefix("/private") or value + async def _resolve_pid(self, package_name: str) -> int: """Find a running process id for the bundle, tracking launches first.""" tracked = self._launched_pids.get(package_name) @@ -590,19 +635,26 @@ async def _resolve_pid(self, package_name: str) -> int: apps = await self._devicectl_json("info", "apps") url_prefix = "" for app in apps.get("apps", []): - if app.get("bundleIdentifier") == package_name: - url_prefix = str(app.get("url") or "").rstrip("/") + if isinstance(app, dict) and app.get("bundleIdentifier") == package_name: + url_prefix = self._normalize_executable(app.get("url")).rstrip("/") break processes = await self._devicectl_json("info", "processes") for process in processes.get("runningProcesses", []): - executable = str(process.get("executable") or "") + if not isinstance(process, dict): + continue + executable = self._normalize_executable(process.get("executable")) matched = ( executable.startswith(url_prefix) if url_prefix else package_name in executable ) - if matched and isinstance(process.get("processIdentifier"), int): - return process["processIdentifier"] + pid = ( + process.get("processIdentifier") + or process.get("pid") + or process.get("processID") + ) + if matched and isinstance(pid, int): + return pid except (OSError, RuntimeError, TimeoutError, ValueError, TypeError) as exc: logger.debug(f"devicectl process enumeration failed: {exc}") raise ValueError( @@ -610,8 +662,8 @@ async def _resolve_pid(self, package_name: str) -> int: "launch it with launch_app before stopping." ) - async def _terminate_pid(self, pid: int) -> None: - await run_xcrun( + async def _terminate_pid(self, pid: int, kill: bool = False) -> None: + arguments = [ "devicectl", "device", "process", @@ -620,26 +672,34 @@ async def _terminate_pid(self, pid: int) -> None: self._device_id, "--pid", str(pid), - timeout=DEVICECTL_QUERY_TIMEOUT, - ) + ] + if kill: + arguments.append("--kill") + await run_xcrun(*arguments, timeout=DEVICECTL_QUERY_TIMEOUT) async def stop_app(self, package_name: str) -> bool: async with self._operation_lock: self._require_connected() + tracked = self._launched_pids.pop(package_name, None) + if tracked is not None: + try: + await self._terminate_pid(tracked, kill=True) + return True + except (OSError, RuntimeError, TimeoutError) as exc: + # A stale or recycled PID must not poison stop_app — + # fall through to the live process scan. + logger.debug(f"Tracked-PID terminate failed for {package_name}: {exc}") pid = await self._resolve_pid(package_name) - await self._terminate_pid(pid) - self._launched_pids.pop(package_name, None) + await self._terminate_pid(pid, kill=True) return True async def install_app(self, app_path: Path) -> str: self._require_connected() path = app_path.expanduser().resolve() - if path.suffix == ".ipa": + if path.suffix.lower() == ".ipa": if not path.is_file(): raise ValueError("iOS installation requires an existing .ipa file.") - # The bundle id lives inside the archive; the filename stem is the - # best available label until the app is installed and listed. - bundle = path.stem + bundle = self._ipa_bundle_id(path) or path.stem elif path.suffix == ".app" and path.is_dir(): with (path / "Info.plist").open("rb") as stream: bundle = plistlib.load(stream).get("CFBundleIdentifier") @@ -664,6 +724,22 @@ async def install_app(self, app_path: Path) -> str: ) return bundle + @staticmethod + def _ipa_bundle_id(path: Path) -> str | None: + """Read CFBundleIdentifier from an IPA's embedded app Info.plist.""" + import zipfile + + try: + with zipfile.ZipFile(path) as archive: + for name in archive.namelist(): + if name.startswith("Payload/") and name.endswith(".app/Info.plist"): + with archive.open(name) as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + except (OSError, zipfile.BadZipFile, KeyError, plistlib.InvalidFileException): + return None + return None + async def list_apps(self) -> dict[str, str]: async with self._operation_lock: self._require_connected() diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py index 8d92cf45..8e0b87bb 100644 --- a/artemis/drivers/ios/physical_recording.py +++ b/artemis/drivers/ios/physical_recording.py @@ -44,6 +44,9 @@ MAX_CONSECUTIVE_FAILURES = 5 +FFMPEG_TIMEOUT_SECONDS = 300.0 + + async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: """Seam for segment assembly so tests can intercept the ffmpeg process.""" process = await asyncio.create_subprocess_exec( @@ -53,7 +56,14 @@ async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - _stdout, stderr = await process.communicate() + try: + _stdout, stderr = await asyncio.wait_for( + process.communicate(), timeout=FFMPEG_TIMEOUT_SECONDS + ) + except TimeoutError: + process.kill() + await process.wait() + return -1, b"ffmpeg timed out" return process.returncode, stderr @@ -131,7 +141,10 @@ async def _poll_loop(self, session: IosPhysicalRecordingSession) -> None: self._consecutive_failures = 0 except asyncio.CancelledError: raise - except (OSError, RuntimeError, TimeoutError) as exc: + except Exception as exc: + # Count any unexpected failure (PIL errors, devicectl + # surprises) toward the limit so the loop can never + # die silently with the session still active. failed = True self._consecutive_failures += 1 session.errors.append(f"Frame capture failed: {exc}") @@ -149,7 +162,9 @@ async def _poll_loop(self, session: IosPhysicalRecordingSession) -> None: async def _start_segment(self, session: IosPhysicalRecordingSession) -> None: assert self._output_dir is not None session.frames = [] - session.frame_index = 0 + # frame_index stays monotonic for the whole session: rolled leftover + # frames keep their original names in the new segment directory, so a + # fresh index range can never collide with them. session.capture_width = session.capture_height = None session.segment_started_monotonic = None session.frames_dir = self._output_dir / f"frames_{session.segment_index:04d}" @@ -216,9 +231,9 @@ async def _convert_record(self, session: IosPhysicalRecordingSession, record: di width = max(2, int(record["width"] or 1080)) // 2 * 2 height = max(2, int(record["height"] or 1920)) // 2 * 2 temporary = output.with_name(f"{output.stem}.part.mp4") - if temporary.exists(): - temporary.unlink() try: + if temporary.exists(): + temporary.unlink() concat = self._write_concat(session, record) returncode, stderr = await _run_ffmpeg( [ @@ -267,8 +282,11 @@ async def _convert_record(self, session: IosPhysicalRecordingSession, record: di except OSError as exc: record["conversion_error"] = str(exc) logger.error(f"Physical iOS recording finalization failed: {exc}") - if temporary.exists(): - temporary.unlink() + try: + if temporary.exists(): + temporary.unlink() + except OSError: + pass session.errors.append( f"Segment {record['output_path']} failed finalization; frames kept" ) @@ -303,6 +321,13 @@ async def _watchdog(self, session: IosPhysicalRecordingSession) -> None: async with self._lock: if not session.is_active: return + poll = session.poll_task + if poll is not None and poll.done() and not poll.cancelled(): + exc = poll.exception() + if exc is not None: + session.errors.append(f"Frame capture task died: {exc}") + session.is_active = False + return if ( session.anchor_monotonic is not None and time.monotonic() - session.anchor_monotonic @@ -426,4 +451,12 @@ async def stop(self) -> Path | None: session = self._session if session is None: return None - return await self._finalize(session) + # _finalize takes _lock itself for the seal — do not hold it here. + finalize = asyncio.ensure_future(self._finalize(session)) + try: + # Shield so a cancelled caller cannot abort mid-finalize and + # lose already-captured frames (mirrors IosScreenRecorder). + return await asyncio.shield(finalize) + except asyncio.CancelledError: + await finalize + raise diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py index 349ea56f..61a6da82 100644 --- a/artemis/drivers/ios/wda.py +++ b/artemis/drivers/ios/wda.py @@ -30,6 +30,8 @@ import asyncio import base64 +import http.client +import ipaddress import json import os from typing import Any @@ -86,7 +88,9 @@ def _sync_request( except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace")[:500] raise RuntimeError(f"WebDriverAgent {method} {path} failed: HTTP {error.code} {detail}") - except (urllib.error.URLError, TimeoutError, OSError) as error: + except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException) as error: + # HTTPException covers garbage services answering probed ports — + # BadStatusLine is not an OSError, so it needs mapping here. raise WdaUnavailableError( f"WebDriverAgent at {self._base} is unreachable: {error}. " "Check the device connection, the WDA runner process, and any " @@ -139,7 +143,9 @@ async def open_session(self) -> str: if session_id is None and isinstance(value.get("capabilities"), dict): session_id = value.get("capabilities", {}).get("sessionId") if not isinstance(session_id, str) or not session_id: - raise RuntimeError("WebDriverAgent did not return a session id.") + raise RuntimeError( + f"WebDriverAgent did not return a session id (response: {value!r:.300})." + ) self._session_id = session_id return session_id @@ -243,7 +249,11 @@ async def press_button(self, name: str) -> bool: "POST", f"/session/{session}/wda/pressButton", {"name": name} ) return True - except (RuntimeError, WdaUnavailableError): + except WdaUnavailableError: + # The transport died — propagate so callers don't misreport an + # outage as an unsupported button. + raise + except RuntimeError: return False async def homescreen(self) -> None: @@ -262,9 +272,20 @@ def normalize_wda_url(raw: str) -> str: if not value: return value if "://" not in value: - # Bare IPv6 literals need brackets once a port is attached. + # Bare IPv6 literals need brackets once a port is attached. A bare + # "v6:port" string is ambiguous — treat the last group as a port only + # when the address part parses as a real IPv6 literal. if value.count(":") > 1 and not value.startswith("["): - value = f"[{value}]" + try: + ipaddress.IPv6Address(value) + value = f"[{value}]" + except ValueError: + address, _, port = value.rpartition(":") + try: + ipaddress.IPv6Address(address) + value = f"[{address}]:{port}" + except ValueError: + value = f"[{value}]" value = f"http://{value}" value = value.rstrip("/") authority = value.split("://", 1)[1] @@ -290,7 +311,9 @@ def wda_url_candidates( for raw in ( env_url if env_url is not None else os.environ.get(WDA_URL_ENV), env_host if env_host is not None else os.environ.get(WDA_HOST_ENV), - f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" if tunnel_ip else None, + (f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" if ":" in tunnel_ip else f"{tunnel_ip}:{WDA_DEFAULT_PORT}") + if tunnel_ip + else None, f"127.0.0.1:{WDA_DEFAULT_PORT}", ): if not raw: @@ -333,12 +356,12 @@ def parse_wda_elements( def visit(entry: dict[str, Any]) -> None: rect = entry.get("rect") - if isinstance(rect, dict): + try: x = float(rect.get("x") or 0.0) y = float(rect.get("y") or 0.0) w = float(rect.get("width") or 0.0) h = float(rect.get("height") or 0.0) - else: + except (TypeError, ValueError, AttributeError): x = y = w = h = 0.0 if w > 0 and h > 0: left, top = round(x * scale[0]), round(y * scale[1]) diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 4ba4d436..eb4d65d5 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -52,8 +52,8 @@ async def run_batch_tasks( 'strict') for the Pro profile; ignored by Flash. explorer_pro_mode: Explorer tier ('flash', 'pro', 'ultra') behind ``ask_explorer`` under the Pro profile; ignored by Flash. - platform: 'android' (default) or 'ios' (local iOS Simulator). - device_serial: Android serial or iOS simulator UDID for all tasks. + platform: 'android' (default) or 'ios' (local simulator or paired device). + device_serial: Android serial or iOS device UDID for all tasks. ios_workspace: Xcode project/workspace for first-run iOS approval. """ if not os.environ.get("ARTEMIS_TASK_INGRESS"): @@ -181,14 +181,14 @@ def batch_command( str, typer.Option( "--platform", - help="Target mobile platform: 'android' (default) or 'ios' (local iOS simulator).", + help="Target mobile platform: 'android' (default) or 'ios' (local simulator or paired device).", ), ] = "android", device_serial: Annotated[ str | None, typer.Option( "--device-serial", - help="Android serial or iOS simulator UDID for every task in the batch.", + help="Android serial or iOS device UDID (simulator or paired physical device) for every task in the batch.", ), ] = None, ios_workspace: Annotated[ diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 7bfcd454..c350cf56 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -94,7 +94,7 @@ async def execute_task( if ios_workspace_path is not None and platform != DevicePlatform.IOS: raise ValueError("--ios-workspace requires --platform ios.") if platform == DevicePlatform.IOS and os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - raise ValueError("iOS Simulator support is local only; cloud mode targets Android.") + raise ValueError("iOS support is local only; cloud mode targets Android.") effective_sid = ( session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") ) @@ -230,7 +230,7 @@ def run_command( str | None, typer.Option( "--app-path", - help="Local Android APK or iOS simulator .app directory to install before the task.", + help="Local Android APK or signed iOS .app/.ipa to install before the task.", ), ] = None, enable_planner_validation: Annotated[ @@ -334,7 +334,7 @@ def run_command( ), ] = False, ) -> None: - """Run an autonomous UI automation task on an Android device or iOS simulator.""" + """Run an autonomous UI automation task on an Android or iOS device.""" console = Console() @@ -350,10 +350,10 @@ def run_command( if platform == DevicePlatform.IOS: if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": raise typer.BadParameter( - "iOS Simulator support is local only; cloud mode targets Android." + "iOS support is local only; cloud mode targets Android." ) if locked_app_package: - raise typer.BadParameter("--locked-app is unavailable for iOS simulator tasks.") + raise typer.BadParameter("--locked-app is unavailable for iOS tasks.") ensure_video_recording_available(with_video_recording_tools) # All platforms route through unified Artemis Daemon unless specifically configured as standalone diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 5a3e962a..00211176 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -118,8 +118,8 @@ def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: def _get_controller(device_serial: str | None = None, target_platform: str | None = None): """Lazy-load device controller on-demand, caching per device serial. - ``target_platform="ios"`` builds a native iOS Simulator controller: the - serial is a simulator UDID (or "booted"), the context carries + ``target_platform="ios"`` builds a native iOS controller: the serial is a + simulator UDID (or "booted") or a paired physical device UDID, the context carries ``DevicePlatform.IOS`` so the factory selects the Xcode driver, and the cache key is namespaced so a UDID can never collide with an Android serial. Callers must ``await driver.connect()`` before interacting -- diff --git a/artemis/runtime/adb_endpoint.py b/artemis/runtime/adb_endpoint.py index f006efe1..bf4f65b4 100644 --- a/artemis/runtime/adb_endpoint.py +++ b/artemis/runtime/adb_endpoint.py @@ -180,6 +180,7 @@ def apply_to_environment( target[ADB_ENDPOINT_ID_ENV] = self.lock_scope # An iOS worker never touches ADB: a stale serial must not leak in. target.pop("ADB_DEVICE_SERIAL", None) + target.pop("ARTEMIS_DEVICE_ID", None) return target def to_dict(self) -> dict[str, Any]: diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 45a17029..fae28ad6 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -232,8 +232,8 @@ def submit_task_to_daemon( ``explorer_mode`` ('flash' | 'pro' | 'ultra') are the Pro-profile tuning knobs of ``/api/run``; they are forwarded verbatim and ignored by Flash. - ``platform="ios"`` submits an iOS Simulator task: ``device_serial`` is a - simulator UDID and ``ios_workspace`` optionally scopes Xcode approval. + ``platform="ios"`` submits an iOS task: ``device_serial`` is a simulator + or physical UDID and ``ios_workspace`` optionally scopes Xcode approval. Returns the response JSON dict if successfully enqueued, or None on error. """ @@ -361,7 +361,7 @@ def submit_batch_to_daemon( ``verification_level`` / ``explorer_mode`` apply to every goal of the batch (see :func:`submit_task_to_daemon`). ``platform="ios"`` targets an iOS - Simulator; ``device_serial`` is then the simulator UDID. + device; ``device_serial`` is then the device UDID. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" payload = { diff --git a/artemis/runtime/ios_device_pool.py b/artemis/runtime/ios_device_pool.py index 0614334f..a71cabbc 100644 --- a/artemis/runtime/ios_device_pool.py +++ b/artemis/runtime/ios_device_pool.py @@ -10,14 +10,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""iOS Simulator device pool: discovery, lock status, and target selection. +"""iOS device pool: simulator and physical discovery, locks, selection. The iOS counterpart of :class:`~artemis.runtime.device_pool.DevicePool`. -Enumeration rides ``xcrun simctl list devices``; lock ownership shares the -same :class:`DeviceExecutionLock` registry under the ``ios`` scope so a -simulator and an Android serial can never share a lock identity. All methods -fail open on missing Xcode tooling or an enumeration error, mirroring the -Android pool's admission contract. +Enumeration rides ``xcrun simctl list devices`` for simulators and +``xcrun devicectl list devices`` for paired physical hardware; lock ownership +shares the same :class:`DeviceExecutionLock` registry under the ``ios`` scope +so a simulator, a physical device, and an Android serial can never share a +lock identity. Auto-selection only ever picks simulators — physical hardware +requires an explicit serial. All methods fail open on missing Xcode tooling +or an enumeration error, mirroring the Android pool's admission contract. """ from __future__ import annotations @@ -137,19 +139,35 @@ async def try_list_devices_async(self) -> list[DeviceStatus] | None: return self._build_statuses(raw or []) + self._build_physical_statuses(core or []) async def validate_explicit_serial_async(self, requested_serial: str) -> str | None: - """Reject an explicitly requested UDID, or ``None`` when usable. + """Reject an explicitly requested UDID or device name, ``None`` when usable. Mirrors the Android validator: only a successful, non-empty - enumeration may reject. ``Shutdown`` simulators are valid targets — - the driver boots them on connect. Physical devices must be paired - and connected (``"device"``); offline devices fail here. + enumeration may reject, and an enumeration that cannot answer (None) + must not reject. ``Shutdown`` simulators are valid targets — the + driver boots them on connect. Physical devices must be paired and + connected (``"device"``); offline devices fail here. """ - devices = await self.try_list_devices_async() - if not devices: + raw = await list_ios_simulators() + core = await list_core_devices() + if raw is None and core is None: return None - by_udid = {d.serial.lower(): d for d in devices} - match = by_udid.get(str(requested_serial).lower()) + devices = self._build_statuses(raw or []) + self._build_physical_statuses( + core or [] + ) + needle = str(requested_serial).lower() + match = next( + ( + d + for d in devices + if d.serial.lower() == needle or (d.model or "") == requested_serial + ), + None, + ) if match is None: + # A failed enumeration cannot prove the serial is absent — + # defer to the driver's own resolution rather than reject. + if raw is None or core is None: + return None return ( f"iOS device '{requested_serial}' is not available. " f"Known devices: {sorted(d.serial for d in devices)}." @@ -169,7 +187,8 @@ def validate_explicit_serial(self, requested_serial: str) -> str | None: """Synchronous validator for non-async admission paths (MCP tools).""" devices = list_ios_simulators_sync() statuses = {d["udid"].lower(): d.get("state") for d in devices or []} - state = statuses.get(str(requested_serial).lower()) + needle = str(requested_serial).lower() + state = statuses.get(needle) if state is not None: if state != "Booted" and state != "Shutdown": return ( @@ -177,18 +196,26 @@ def validate_explicit_serial(self, requested_serial: str) -> str | None: "and cannot be used." ) return None - physical = { - d["udid"].lower(): d - for d in list_core_devices_sync() or [] - if is_physical_ios(d) - } - match = physical.get(str(requested_serial).lower()) + core = list_core_devices_sync() + physical = [d for d in core or [] if is_physical_ios(d)] + match = next( + ( + d + for d in physical + if str(d.get("udid") or "").lower() == needle + or d.get("name") == requested_serial + ), + None, + ) if match is None: + # An enumeration that could not answer must not reject the serial. + if devices is None or core is None: + return None if not statuses and not physical: return None return ( f"iOS device '{requested_serial}' is not available. " - f"Known devices: {sorted([*statuses, *physical])}." + f"Known devices: {sorted([*statuses, *[d['udid'] for d in physical]])}." ) if match.get("pairing_state") != "paired": return f"iOS device '{requested_serial}' is not paired with this Mac." @@ -201,18 +228,21 @@ async def select_device_async(self, preferred_serial: str | None = None) -> str An explicit serial wins as-is. Otherwise prefer an idle booted simulator, then any booted one, then the only available simulator - when none are booted (the driver boots it on connect). ``None`` - when no simulator exists; strict ``booted`` ambiguity rejection is - the driver's job for users who literally target "booted". + when none are booted (the driver boots it on connect). Physical + devices are never auto-selected — hardware always needs an explicit + serial. ``None`` when no simulator exists; strict ``booted`` + ambiguity rejection is the driver's job for users who literally + target "booted". """ if preferred_serial: return preferred_serial devices = await self.try_list_devices_async() or [] - booted = [d for d in devices if d.state == "device"] + simulators = [d for d in devices if d.is_emulator] + booted = [d for d in simulators if d.state == "device"] if len(booted) == 1: return booted[0].serial - if not booted and len(devices) == 1: - return devices[0].serial + if not booted and len(simulators) == 1: + return simulators[0].serial idle_booted = [d for d in booted if not d.is_busy] return idle_booted[0].serial if idle_booted else (booted[0].serial if booted else None) diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index 25cc1255..ae09ae5c 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -144,7 +144,7 @@ async def _init_internal( if self._config.device_platform != DevicePlatform.IOS: return await super()._init_internal(api_key, retry_count, retry_wait_seconds) if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - raise AgentError("iOS Simulator support is local only; cloud mode targets Android.") + raise AgentError("iOS support is local only; cloud mode targets Android.") if self._initialized: return True from artemis.drivers.factory import ios_driver_class @@ -152,7 +152,12 @@ async def _init_internal( publish_startup_progress( "device_check", "Checking the iOS device", session_id=self._session_id ) - driver = ios_driver_class(self._config.device_id or "booted")( + # The picker runs simctl/devicectl subprocesses — keep them off the + # event loop so init timeouts and progress stays responsive. + driver_class = await asyncio.to_thread( + ios_driver_class, self._config.device_id or "booted" + ) + driver = driver_class( device_id=self._config.device_id or "booted", workspace_path=getattr(self._config, "ios_workspace_path", None), ) diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index a717e171..97008748 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -81,7 +81,7 @@ def for_device_serial(self, device_serial: str) -> "AgentConfigBuilder": def for_ios_simulator( self, device_id: str = "booted", *, workspace_path: str | Path | None = None ) -> "AgentConfigBuilder": - """Target an iOS simulator UDID, or the single booted simulator.""" + """Target an iOS device UDID (simulator or paired physical), or the booted simulator.""" if workspace_path is not None: self.with_ios_workspace(workspace_path) return self.for_device(DevicePlatform.IOS, device_id) diff --git a/docs/ios.md b/docs/ios.md index 150471b0..d4ddd41b 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -35,16 +35,17 @@ see the 3. For physical devices: connect the iPhone/iPad over USB (or enable network pairing), tap **Trust** on the device pairing prompt, and enable **Developer Mode** in Settings > Privacy & Security on iOS 16+. `xcrun devicectl list - devices` should report the device as `paired` and `connected`. + devices` should show the device `connected` (or `available (paired)`); the + JSON fields are `pairingState: paired` and `tunnelState: connected`. 4. Check the toolchain and available simulators from the repository root: ```bash bash scripts/setup_ios_env.sh ``` - This check reads the Xcode version, tool locations, simulator inventory, and - `xcrun mcp-server status`. It does not install dependencies, boot a simulator, - or change access settings. A passing check confirms prerequisites; the first + This check reads the Xcode version, tool locations, simulator and + physical-device (`devicectl`) inventories, and `xcrun mcp-server status`. It + does not install dependencies, boot a simulator, or change access settings. A passing check confirms prerequisites; the first driver connection checks native MCP tool availability and device access. 5. Install Artemis's Python dependencies and configure a model provider using the normal project configuration: @@ -124,7 +125,7 @@ Any Apple development team works; a free Personal Team profile must be re-signed every 7 days while paid-program profiles last a year. When Artemis connects, it finds an installed `*WebDriverAgent*` runner on the -device, launches it via `devicectl process launch`, and probes its HTTP +device, launches it via `devicectl device process launch`, and probes its HTTP endpoint on the CoreDevice tunnel address and `127.0.0.1:8100` (for `iproxy`/`pymobiledevice3` port forwards). Endpoint overrides: @@ -132,12 +133,17 @@ endpoint on the CoreDevice tunnel address and `127.0.0.1:8100` (for `http://:8100` - `ARTEMIS_IOS_WDA_HOST` — host only; port 8100 assumed - `ARTEMIS_IOS_WDA_BUNDLE_ID` — nonstandard runner bundle identifier +- `ARTEMIS_IOS_WDA_XCTESTRUN` — path to a `WebDriverAgentRunner_*.xctestrun` + bundle from `xcodebuild build-for-testing`; Artemis hosts it via + `xcodebuild test-without-building` (the canonical WDA session — a bare + runner app launch does not start the HTTP server) Differences from simulators: - The device must already be paired, trusted, and connected; Artemis never - boots or unlocks it. A locked device can be observed but most input fails — - unlock it first. + boots or unlocks it — unlock the device before connecting, since WDA cannot + inject touches while it is locked. The first XCTest attach may also show an + on-device "Enable UI Automation" passcode prompt; approve it once. - `--app-path` expects a device-signed artifact: an `.app` built for an arm64 device destination (signed with a valid provisioning profile) or a `.ipa`. Simulator `.app` bundles are x86_64/arm64-simulator builds and cannot be @@ -222,19 +228,22 @@ probe is involved. `artemis batch` accepts the same `--platform`, The Admin Console `/api/run` accepts `platform: "ios"`, a simulator UDID in `device_serial`, and an optional `ios_workspace`; `/api/devices` lists Android -devices and iOS simulators together, each tagged with its `platform`. The live -screen view (`/api/stream/device-live`) streams simulator frames captured with -`simctl io screenshot` when an iOS task holds the lock or a simulator is the -only viable target; `/api/stream/device-state` reports the `platform` of the -streamed device. Replay preserves the recorded session's `mobile_platform`, so -an iOS trace replays through the Xcode driver on a simulator (the device -picker retargets iOS replays to a chosen UDID). +devices and iOS devices (simulators and paired physical hardware) together, +each tagged with its `platform`. The live screen view +(`/api/stream/device-live`) streams frames captured with `simctl io +screenshot` for simulators or `devicectl capture screenshot` for physical +devices when an iOS task holds the lock; `/api/stream/device-state` reports +the `platform` of the streamed device. Replay preserves the recorded session's +`mobile_platform`, so an iOS trace replays through the native iOS driver (the +device picker retargets iOS replays to a chosen UDID — simulator or paired +physical). `mobile_run_task` accepts `platform="ios"`, `device_serial=`, and -`ios_workspace=`; it validates the UDID against `simctl` rather than ADB -and queues the runner under the `ios` lock scope. `mobile_get_device_state` -and `mobile_diagnose` accept the same `platform` switch — the latter runs a -native screenshot/hierarchy smoke test on the simulator. The legacy +`ios_workspace=`; it validates the UDID against the `simctl` and +`devicectl` inventories rather than ADB and queues the runner under the `ios` +lock scope. `mobile_get_device_state` and `mobile_diagnose` accept the same +`platform` switch — the latter runs a native screenshot/hierarchy smoke test +on the selected iOS device. The legacy `Android_ADB_Controller` actuator server (tap/swipe/type tools) remains Android-only. @@ -265,7 +274,7 @@ instead of acquiring a second one. | Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | WDA `/source` and `/screenshot` (devicectl screenshot fallback) | | Tap, long press, and swipe | Native synthesized touch events | WDA W3C pointer actions | | Text entry | Native keyboard synthesis with `clear_exist=false` | WDA `/wda/keys`, `clear_exist=false` | -| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | WDA `/wda/homescreen` and `/wda/pressButton` | +| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | WDA `/wda/keys` (Enter), `/wda/homescreen`, and `/wda/pressButton` | | App install, launch, and terminate | `simctl` with simulator `.app` bundles | `devicectl` with signed `.app`/`.ipa`; terminate uses the launch PID or a live process scan | | Screen recording | `simctl io recordVideo` (VFR H.264) | `devicectl` screenshot polling assembled to timestamped MP4 | diff --git a/mcp_server/tools/device_state.py b/mcp_server/tools/device_state.py index 0df3c64f..df8860dd 100644 --- a/mcp_server/tools/device_state.py +++ b/mcp_server/tools/device_state.py @@ -49,10 +49,10 @@ async def mobile_get_device_state( - "hierarchy": returns the simplified text-labeled element list — exactly what the automation subagent sees when making decisions. device_serial: Optional device serial (e.g. "emulator-5554", or an iOS - simulator UDID with platform="ios") to inspect a specific device; + device UDID with platform="ios") to inspect a specific device; omitted → the default connected device. platform: "android" (default) or "ios". For iOS, device_serial is a - simulator UDID; omit it to use the single booted simulator. + device UDID (simulator or paired physical device); omit it to use the single booted simulator. """ try: controller = _get_controller( diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index cd7197ba..c49a8f44 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -482,15 +482,23 @@ async def _run_device_probe( ) -> dict[str, Any]: if platform == "ios": simulators = (ios_result.metadata.get("simulators") if ios_result else None) or [] - ready_udids = [str(d.get("udid")) for d in simulators if d.get("udid")] + physical = ( + ios_result.metadata.get("connected_physical_devices") if ios_result else None + ) or [] + ready_udids = [str(d.get("udid")) for d in simulators if d.get("udid")] + [ + str(d.get("udid")) for d in physical if d.get("udid") + ] if requested_device and requested_device not in ready_udids: return _probe_unavailable( requested_device, - f"requested iOS simulator '{requested_device}' is not available; nothing to probe", + f"requested iOS device '{requested_device}' is not available " + "(simulators plus paired+connected physical devices); nothing to probe", ) if not ready_udids: - return _probe_unavailable(None, "no available iOS simulator found; nothing to probe") - serial = requested_device or (ready_udids[0] if len(ready_udids) == 1 else None) + return _probe_unavailable(None, "no available iOS device found; nothing to probe") + # Auto-pick only a lone simulator — never silently choose hardware. + sim_udids = [str(d.get("udid")) for d in simulators if d.get("udid")] + serial = requested_device or (sim_udids[0] if len(sim_udids) == 1 else None) try: return await _device_smoke_test(serial, platform="ios") except Exception as exc: @@ -1046,10 +1054,13 @@ def _requested_device_ready( if ios_result is None: return False simulators = ios_result.metadata.get("simulators") or [] - return any( + if any( d.get("udid") == requested_device and d.get("state") in ("Booted", "Shutdown") for d in simulators - ) + ): + return True + physical = ios_result.metadata.get("connected_physical_devices") or [] + return any(d.get("udid") == requested_device for d in physical) adb_result = _find(results, "android_adb") if adb_result is None: return False @@ -1235,8 +1246,9 @@ async def mobile_diagnose( the device, or the screen stays black. A failed probe makes the verdict "blocked" and lists the fix. platform: 'android' (default) or 'ios'. With 'ios', `device_serial` - is a simulator UDID and `probe_device` exercises the native Xcode - path; ADB-only extras (AVD launch, accessibility helper) are skipped. + is a simulator or paired physical UDID and `probe_device` exercises + the native path (Xcode MCP for simulators, WebDriverAgent for + hardware); ADB-only extras (AVD launch, accessibility helper) are skipped. """ fixes_applied: list[dict[str, Any]] = [] platform_name = (platform or "android").strip().lower() diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index 23e21b5a..5686ddec 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -157,7 +157,7 @@ def _validate_device_serial(device_serial: str, platform: str = "android") -> di # the task proceeds and fails downstream with a clear no-device error. if platform == "ios": detail = ios_device_pool.validate_explicit_serial(device_serial) - inspect_hint = "Run `xcrun simctl list devices` to inspect simulators" + inspect_hint = "Run `xcrun simctl list devices` (simulators) or `xcrun devicectl list devices` (physical devices)" else: detail = device_pool.validate_explicit_serial(device_serial) inspect_hint = "Run `adb devices -l` to inspect attached hardware" @@ -267,14 +267,20 @@ def mobile_run_task( model: `"Flash"` or `"Pro"` — see model selection above. locked_app_package: Optional package name to lock execution to; the agent auto-launches it and restricts actions to that app. - app_path: Optional local APK path to install before running. + app_path: Optional local APK (Android) or signed .app/.ipa (iOS) to install before running. expected_output_desc: Optional, Pro only. If set, a summarization agent writes a report to `output.md` in `notes_dir`. Ignored for Flash. - device_serial: Optional device serial (e.g. "emulator-5554") to bind - execution to a specific device; distinct devices run concurrently. - If omitted, an available device is selected automatically. When + device_serial: Optional device serial (e.g. "emulator-5554" or an iOS + device UDID) to bind execution to a specific device; distinct devices + run concurrently. If omitted, an available device is selected + automatically (physical iOS hardware is never auto-selected). When several devices are attached, confirm the target with the user first - (`adb devices -l` lists serials and authorization states). + (`adb devices -l` lists Android serials; `xcrun devicectl list + devices` lists physical iOS UDIDs). + platform: `"android"` (default) or `"ios"` — iOS runs locally against + a simulator or paired physical device. + ios_workspace: Optional Xcode workspace path for iOS; scopes the + agent-access approval grant (simulator interaction only). verification_level: Optional, Pro only. Coarse Checker preset: `"off"` (no audit; the Operator self-reports), `"final"` (one exit review against the goal, the default), `"checkpoints"` (every plan diff --git a/scripts/setup_ios_env.sh b/scripts/setup_ios_env.sh index 65ef41c5..028077f4 100755 --- a/scripts/setup_ios_env.sh +++ b/scripts/setup_ios_env.sh @@ -22,7 +22,7 @@ fail() { exit 1 } -[[ "$(uname -s)" == "Darwin" ]] || fail "iOS simulator support requires macOS." +[[ "$(uname -s)" == "Darwin" ]] || fail "iOS support requires macOS." command -v xcodebuild >/dev/null 2>&1 || fail "Install Xcode 27 or newer." command -v xcrun >/dev/null 2>&1 || fail "Xcode's xcrun is unavailable." diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py index 30302a0b..03c1a648 100644 --- a/tests/unit/drivers/test_ios_physical.py +++ b/tests/unit/drivers/test_ios_physical.py @@ -13,7 +13,6 @@ """Physical iOS device behavior tested without attached hardware.""" import asyncio -import base64 import io import json from pathlib import Path @@ -62,7 +61,7 @@ def _parsed(*devices): return discovery.parse_devicectl_devices(_devicectl_payload(*devices)) -PHYSICAL_IPHONE = _devicectl_device(IPHONE_UDID, name="dongseok의 iPhone") +PHYSICAL_IPHONE = _devicectl_device(IPHONE_UDID, name="Jane's iPhone") OFFLINE_IPHONE = _devicectl_device(IPHONE_UDID, name="iPhone", tunnel="disconnected") UNPAIRED_IPHONE = _devicectl_device(IPHONE_UDID, name="iPhone", pairing="unpaired") SIMULATOR = _devicectl_device( @@ -127,7 +126,7 @@ def test_find_physical_device_matches_udid_case_insensitively(monkeypatch): ) found = discovery.find_physical_ios_device_sync(IPHONE_UDID.lower()) assert found is not None and found["udid"] == IPHONE_UDID - assert discovery.find_physical_ios_device_sync("dongseok의 iPhone") is not None + assert discovery.find_physical_ios_device_sync("Jane's iPhone") is not None assert discovery.find_physical_ios_device_sync(SIM_UDID) is None assert discovery.find_physical_ios_device_sync("missing") is None @@ -142,6 +141,17 @@ def driver(): return PhysicalIosDriver(device_id=IPHONE_UDID) +@pytest.fixture(autouse=True) +def _clean_wda_env(monkeypatch): + for key in ( + 'ARTEMIS_IOS_WDA_URL', + 'ARTEMIS_IOS_WDA_HOST', + 'ARTEMIS_IOS_WDA_XCTESTRUN', + 'ARTEMIS_IOS_WDA_BUNDLE_ID', + ): + monkeypatch.delenv(key, raising=False) + + @pytest.mark.asyncio async def test_physical_resolve_rejects_booted_and_empty_serials(driver): driver._device_id = "booted" @@ -151,7 +161,7 @@ async def test_physical_resolve_rejects_booted_and_empty_serials(driver): @pytest.mark.asyncio async def test_physical_resolve_pins_the_udid(driver, monkeypatch): - monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(PHYSICAL_IPHONE, SIMULATOR))) + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE, SIMULATOR))) candidate = await driver._resolve_device() assert candidate["udid"] == IPHONE_UDID assert driver.device_id == IPHONE_UDID @@ -160,7 +170,7 @@ async def test_physical_resolve_pins_the_udid(driver, monkeypatch): @pytest.mark.asyncio async def test_physical_resolve_rejects_simulator_udids(driver, monkeypatch): driver._device_id = SIM_UDID - monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(SIMULATOR))) + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(SIMULATOR))) with pytest.raises(ValueError, match="Simulator"): await driver._resolve_device() @@ -168,14 +178,14 @@ async def test_physical_resolve_rejects_simulator_udids(driver, monkeypatch): @pytest.mark.asyncio async def test_physical_resolve_rejects_non_ios_hardware(driver, monkeypatch): driver._device_id = WATCH_UDID - monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(WATCH))) + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(WATCH))) with pytest.raises(ValueError, match="paired physical iOS"): await driver._resolve_device() @pytest.mark.asyncio async def test_physical_resolve_reports_enumeration_failure(driver, monkeypatch): - monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(None)) + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(None)) with pytest.raises(RuntimeError, match="devicectl"): await driver._resolve_device() @@ -197,7 +207,7 @@ async def test_connected_device_needs_no_boot(driver): await driver._prepare_device(_parsed(PHYSICAL_IPHONE)[0]) -def _AsyncDevices(result): +def _async_devices(result): async def _list(force_refresh=False): return result @@ -232,6 +242,7 @@ class _FakeWda: def __init__(self, window=(100.0, 200.0), tree=None): self.session_id = None + self.base_url = "http://fake-wda:8100" self.window = window self.tree = tree if tree is not None else {"type": "Application", "children": []} self.tapped: list[tuple[float, float, int]] = [] @@ -772,7 +783,7 @@ async def fake_xcrun(*arguments, timeout=30.0): @pytest.mark.asyncio async def test_connect_establishes_wda_session(driver, monkeypatch): client = _FakeWda() - monkeypatch.setattr(physical_driver, "list_core_devices", _AsyncDevices(_parsed(PHYSICAL_IPHONE))) + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE))) monkeypatch.setattr(driver, "_ensure_wda", _async_return(client)) monkeypatch.setattr(driver, "_require_ios_host", _async_return(None)) await driver.connect() diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py index 9eeb98bb..25afe13c 100644 --- a/tests/unit/test_ios_integration.py +++ b/tests/unit/test_ios_integration.py @@ -502,7 +502,7 @@ def test_flash_and_pro_prompts_teach_ios_constraints(native_driver): prompts = {"main_template": PLAN_HISTORY_TEMPLATE_SECTION + "\n# CURRENT OBSERVATION"} pro = render_transcript_static_system(prompts, context, State.initial("Open Settings")) for prompt in (flash, pro): - assert "iOS Simulator" in prompt + assert "Target platform: iOS." in prompt assert "clear_exist=false" in prompt assert ( "press_key supports enter, home, " diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index 5a5f6807..8a5dcb13 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -371,6 +371,48 @@ async def test_ios_pool_select_device_prefers_booted_and_idle(monkeypatch): assert await pool.select_device_async(preferred_serial="BBBB-2222") == "BBBB-2222" +@pytest.mark.asyncio +async def test_ios_pool_select_device_never_picks_physical(monkeypatch): + """Auto-selection must never target paired hardware without an explicit serial.""" + import importlib + + physical = { + "udid": "00008130-0000ABCD1234FFFF", + "name": "Test iPhone", + "reality": "physical", + "platform": "iOS", + "pairing_state": "paired", + "connection_state": "connected", + "os_version": "26.0", + } + module = importlib.import_module("artemis.runtime.ios_device_pool") + pool = IosDevicePool() + + # Physical alone: nothing to auto-pick. + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=[])) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=[physical])) + assert await pool.select_device_async() is None + + # Physical alongside one booted sim: only the sim is eligible. + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(SIM_LIST))) + assert await pool.select_device_async() == "AAAA-1111" + + # Explicit serial still passes through untouched. + assert await pool.select_device_async(preferred_serial=physical["udid"]) == physical["udid"] + + +@pytest.mark.asyncio +async def test_ios_pool_validate_fails_open_on_partial_enumeration(monkeypatch): + """A dead devicectl enumeration cannot disprove a physical serial.""" + import importlib + + module = importlib.import_module("artemis.runtime.ios_device_pool") + pool = IosDevicePool() + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(SIM_LIST))) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=None)) + assert await pool.validate_explicit_serial_async("00008130-0000ABCD1234FFFF") is None + + @pytest.mark.asyncio async def test_ios_pool_select_device_boots_single_shutdown_sim(monkeypatch): pool = _pool_with_devices(monkeypatch, [d for d in SIM_LIST if d["udid"] == "BBBB-2222"]) From 99c5c16e586527d1f09a18f3930dd1b55f9b153c Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 04:16:49 +0900 Subject: [PATCH 21/32] Test hygiene: hermetic resolve tests, synthetic fixtures, awaited cancels - Mock _require_ios_host on the driver fixture so _resolve_device tests never spawn real xcodebuild or fail on non-macOS CI - Synthetic UDID/name/URL fixtures (no real-looking hardware identifiers) - Await the cancelled poll task on recording start failure - Base the session poll/watchdog fields as instance defaults Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/drivers/test_ios_physical.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py index 03c1a648..fc2a1bad 100644 --- a/tests/unit/drivers/test_ios_physical.py +++ b/tests/unit/drivers/test_ios_physical.py @@ -27,10 +27,10 @@ from artemis.drivers.ios.wda import WdaClient -IPHONE_UDID = "00008130-000245193C60001C" -IPAD_UDID = "00008103-00197094013B001E" +IPHONE_UDID = "00008130-0000ABCD1234AAAA" +IPAD_UDID = "00008103-0000ABCD1234BBBB" SIM_UDID = "DE345DD3-5792-4DAD-B863-144682629565" -WATCH_UDID = "00008301-D8814D603680202E" +WATCH_UDID = "00008301-0000ABCD1234CCCC" def _devicectl_device( @@ -137,8 +137,10 @@ def test_find_physical_device_survives_enumeration_failure(monkeypatch): @pytest.fixture -def driver(): - return PhysicalIosDriver(device_id=IPHONE_UDID) +def driver(monkeypatch): + instance = PhysicalIosDriver(device_id=IPHONE_UDID) + monkeypatch.setattr(instance, "_require_ios_host", _async_return(None)) + return instance @pytest.fixture(autouse=True) @@ -451,9 +453,9 @@ async def fake_xcrun(*arguments, timeout=30.0): return b"{}" monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) - assert await connected_driver.open_url("bookwriter://open") + assert await connected_driver.open_url("example://open") assert calls[0][:5] == ("devicectl", "device", "process", "openURL", "--device") - assert calls[0][-1] == "bookwriter://open" + assert calls[0][-1] == "example://open" @pytest.mark.asyncio From f77adb36030d00bb9ba6d93882fd6397b7ecbefe Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 04:17:44 +0900 Subject: [PATCH 22/32] Await the cancelled poll task on recording start failure Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/drivers/ios/physical_recording.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py index 8e0b87bb..34eed8f0 100644 --- a/artemis/drivers/ios/physical_recording.py +++ b/artemis/drivers/ios/physical_recording.py @@ -22,6 +22,7 @@ """ import asyncio +import contextlib from io import BytesIO from pathlib import Path import tempfile @@ -380,6 +381,9 @@ async def start( session.is_active = False if session.poll_task and not session.poll_task.done(): session.poll_task.cancel() + if session.poll_task is not None: + with contextlib.suppress(asyncio.CancelledError, Exception): + await session.poll_task raise session.watchdog_task = asyncio.create_task(self._watchdog(session)) return session From ee76c7d2ad90c11befad84b9194f603270462f3d Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 15:46:45 +0900 Subject: [PATCH 23/32] Prune stale code: unreachable erase path and unused test imports Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/controllers/unified_controller.py | 7 ++----- tests/unit/test_ios_parity.py | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 0a6311e9..3202b639 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -67,11 +67,8 @@ async def open_url(self, url: str) -> bool: async def erase_text(self, nb_chars: int | None = None) -> bool: if self.ctx.device.mobile_platform == "ios": - if nb_chars is not None and nb_chars > 0: - for _ in range(nb_chars): - if not await self._driver.press_key("delete"): - return False - return True + # No native clear/Backspace key exists; the driver raises + # NotImplementedError for unsupported clear operations. return await self._driver.input_text("", clear_existing=True) return await super().erase_text(nb_chars) diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index acb8d804..59345753 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -18,9 +18,8 @@ enumeration is stubbed at the discovery boundary. """ -import asyncio import os -from unittest.mock import AsyncMock, MagicMock, Mock, patch +from unittest.mock import AsyncMock, MagicMock, Mock import pytest From f506e250d54fdc1df7a12e7897170d34df3316d5 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 16:14:08 +0900 Subject: [PATCH 24/32] Mark iOS Platform Expansion as delivered in the roadmap Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8620cbf7..59aa1111 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [ ] **iOS Platform Expansion**: [iOS Simulator support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming; physical devices remain planned. +- [x] **iOS Platform Expansion**: [iOS Simulator support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming; physical devices remain planned. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index b2e198b9..df0b9d04 100644 --- a/README_CN.md +++ b/README_CN.md @@ -309,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 路线图 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [ ] **iOS 跨平台支持**：[iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏；真机仍在规划中。 +- [x] **iOS 跨平台支持**：[iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏；真机仍在规划中。 - [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。 From 42604adc67ea4c96709f938ba935583f90c2d585 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 16:35:07 +0900 Subject: [PATCH 25/32] Reword roadmap: physical iOS support is delivered, not planned Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 59aa1111..b83e7e18 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [x] **iOS Platform Expansion**: [iOS Simulator support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming; physical devices remain planned. +- [x] **iOS Platform Expansion**: [iOS Simulator support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming; physical iPhone/iPad support is delivered in a stacked follow-up change. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index df0b9d04..c35ec00d 100644 --- a/README_CN.md +++ b/README_CN.md @@ -309,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 路线图 - [ ] **Android Studio 深度集成**：推出官方 IDE 插件与协同工作流，支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [x] **iOS 跨平台支持**：[iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏；真机仍在规划中。 +- [x] **iOS 跨平台支持**：[iOS 模拟器支持](./docs/ios.md)已支持 Xcode 27+，覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏；真机（iPhone/iPad）支持由后续叠加变更提供。 - [ ] **端侧轻量化模型**：支持离线运行的轻量级 Edge VLM，实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**：支持自然语音下发任务与实时打断（Barge-in）控制。 From f50f89838e18fd1ffa2d019a71f81e9a0afa3dd0 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 17:31:14 +0900 Subject: [PATCH 26/32] Fix deep-review findings: listapps parse, lock scope, probe leaks - list_apps: simctl emits OpenStep plists that plistlib cannot read; route through plutil -convert json (plistlib fast-path kept) - connect(): reconnect when the MCP bridge retired but a stale session key remained; null-safe deviceUUID; widen connect-cleanup catch - device_lock: annotate_active_owner preserves lock_scope and resolves scoped lock files; cleanup_stale_locks glob matches scoped names - readiness: SKIPPED probes no longer force a degraded verdict on non-macOS hosts - recording: kill+reap display-probe on timeout/cancel; bound crash-loop respawns (spawn success no longer resets the counter; only a segment surviving a healthy interval does); ffmpeg/probe timeouts; bound concurrent conversions; guard stale seal through_time - bridge: scoped child env (no API keys into mcpbridge), start() lock, close() never masks caller errors - adb_server: iOS controller no longer aliases into the Android global - cli: batch --ios-workspace path validation; run no longer misclassifies iOS prerequisite errors as missing API keys - stream: reap cancelled simctl child; drop stale frames on target switch; annotate active_tasks with platform; queue items surface platform in the UI mapper - env scrub pops ARTEMIS_DEVICE_ID for iOS workers (daemon + MCP) - device_smoke: platform-aware error wording for iOS - docs: fix broken SDK builder example; mobile_run_task documents platform/ios_workspace; platform_guidance no longer claims video analysis is unavailable - diagnose validates platform values; visualization resolves 0-1000 coords on small screenshots - pyright-core covers the new iOS runtime/actuator modules Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/admin_console/routers/tasks.py | 2 + .../services/device_stream_service.py | 16 ++++- .../src/app/services/agent.service.ts | 3 +- artemis/agents/platform_guidance.py | 2 +- artemis/core/diagnostics/device_smoke.py | 12 ++-- artemis/core/diagnostics/readiness.py | 2 +- artemis/drivers/ios/bridge.py | 44 ++++++++---- artemis/drivers/ios/discovery.py | 40 ++++++++++- artemis/drivers/ios/recording.py | 72 ++++++++++++++++--- artemis/drivers/ios/xcode_driver.py | 29 ++++++-- artemis/interfaces/cli/commands/batch.py | 4 ++ artemis/interfaces/cli/commands/run.py | 2 +- artemis/mcp/adb_server.py | 2 - artemis/runtime/adb_endpoint.py | 2 + artemis/runtime/device_lock.py | 20 ++++-- artemis/utils/visualization.py | 7 +- docs/ios.md | 4 +- mcp_server/tools/diagnose.py | 5 ++ mcp_server/tools/task_runner.py | 19 +++-- pyright-core.json | 2 + tests/unit/drivers/test_ios_driver.py | 11 ++- tests/unit/runtime/test_device_lock.py | 22 ++++++ 22 files changed, 267 insertions(+), 55 deletions(-) diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index e6776453..b44e6562 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -20,6 +20,7 @@ from fastapi.responses import StreamingResponse from artemis.core.diagnostics import readiness_engine from artemis.runtime import DeviceExecutionLock, device_pool, ios_device_pool +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE try: from admin_console.core.state import state @@ -302,6 +303,7 @@ async def get_status(): "pid": owner.pid, "ingress": owner.ingress, "acquired_at": owner.acquired_at, + "platform": "ios" if getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE else "android", } for owner in active_owners.values() ] diff --git a/apps/admin_console/services/device_stream_service.py b/apps/admin_console/services/device_stream_service.py index 77ee9061..686854a4 100644 --- a/apps/admin_console/services/device_stream_service.py +++ b/apps/admin_console/services/device_stream_service.py @@ -45,6 +45,7 @@ def __init__(self): self._last_frame_time: float = 0.0 self._is_capturing = False self._capture_task: asyncio.Task | None = None + self._last_target: dict[str, str] | None = None async def get_stream_target(self) -> dict[str, str] | None: """Pick the device the stream should follow right now. @@ -134,7 +135,16 @@ async def _capture_ios(self, udid: str) -> bytes | None: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - await proc.communicate() + try: + await proc.communicate() + except asyncio.CancelledError: + if proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + await proc.communicate() + raise if proc.returncode == 0 and frame_path.exists(): data = frame_path.read_bytes() return data if len(data) > 1000 else None @@ -149,6 +159,10 @@ async def _capture_loop(self): try: start_t = time.time() target = await self.get_stream_target() + if target != self._last_target: + # Never carry frames across a device/platform switch. + self._latest_frame = None + self._last_target = target frame = None if target is not None: if target["platform"] == "ios": diff --git a/apps/showcase_ui/src/app/services/agent.service.ts b/apps/showcase_ui/src/app/services/agent.service.ts index 878876f8..2ee14c60 100644 --- a/apps/showcase_ui/src/app/services/agent.service.ts +++ b/apps/showcase_ui/src/app/services/agent.service.ts @@ -1556,7 +1556,8 @@ export class AgentService { initial_goal: item.goal || '', start_time: item.start_time || item.created_at || (Date.now() / 1000 + index), status: item.status || 'pending', - device_serial: item.device_serial || item.device_id || null + device_serial: item.device_serial || item.device_id || null, + platform: item.platform || null }; } return { diff --git a/artemis/agents/platform_guidance.py b/artemis/agents/platform_guidance.py index e44dcb8c..86f98b03 100644 --- a/artemis/agents/platform_guidance.py +++ b/artemis/agents/platform_guidance.py @@ -21,7 +21,7 @@ def device_action_guidance(ctx) -> str: return ( "Target platform: iOS Simulator. For input_text, explicitly set clear_exist=false; " "type into an empty field or at its existing cursor. Whole-field clearing, Android " - "keycodes, Back/Delete keys, ADB commands, video analysis and app locking are " + "keycodes, Back/Delete keys, ADB commands, and app locking are " "unavailable. Navigate with visible iOS controls; press_key supports enter, home, " "power, volume_up, volume_down, and app_switch. manage_app accepts installed iOS " "display names or bundle identifiers. Action coordinates use the screenshot's " diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index cb7e78ce..4eff9c22 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -356,14 +356,16 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any "controller-init", ) except TimeoutError: + label = "iOS driver" if is_ios else "UIAutomator/controller" return _finish( - f"UIAutomator/controller initialization did not respond within {timeout_seconds:g}s" + f"{label} initialization did not respond within {timeout_seconds:g}s" ) except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught cause = _describe_failure(exc) - return _finish(f"Failed to initialize Android device controller: {cause}", cause) + target = "iOS device" if is_ios else "Android device" + return _finish(f"Failed to initialize {target} controller: {cause}", cause) controller_ctx = getattr(controller, "ctx", None) device = getattr(controller_ctx, "device", None) @@ -386,7 +388,8 @@ async def _observe() -> Any: "screen-data", ) except TimeoutError: - return _finish(f"UIAutomator/screen capture did not respond within {timeout_seconds:g}s") + label = "iOS session capture" if is_ios else "UIAutomator/screen capture" + return _finish(f"{label} did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught @@ -411,8 +414,9 @@ async def _observe() -> Any: f"Screenshot capture failed (driver returned a {result['screenshot_bytes']}-byte placeholder image)" ) if not result["element_count"]: + backend = "Xcode accessibility tree" if is_ios else "UIAutomator hierarchy dump" return _finish( - "UIAutomator hierarchy dump returned no UI elements (screenshot worked, hierarchy did not)" + f"{backend} returned no UI elements (screenshot worked, hierarchy did not)" ) return _finish(None) diff --git a/artemis/core/diagnostics/readiness.py b/artemis/core/diagnostics/readiness.py index c197c374..03c2e8fb 100644 --- a/artemis/core/diagnostics/readiness.py +++ b/artemis/core/diagnostics/readiness.py @@ -71,7 +71,7 @@ def base_verdict(results: Sequence[ProbeResult]) -> Verdict: blockers = [r for r in results if r.is_blocker] if not blockers or any(r.status is not ProbeStatus.PASS for r in blockers): return "blocked" - if any(r.status is not ProbeStatus.PASS for r in results): + if any(r.status not in (ProbeStatus.PASS, ProbeStatus.SKIPPED) for r in results): return "degraded" return "ready" diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py index 91aedf26..d488fb56 100644 --- a/artemis/drivers/ios/bridge.py +++ b/artemis/drivers/ios/bridge.py @@ -25,9 +25,13 @@ import anyio from mcp import ClientSession, StdioServerParameters -from mcp.client.stdio import stdio_client +from mcp.client.stdio import get_default_environment, stdio_client from mcp.shared.exceptions import McpError +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + _APPROVAL_MESSAGES = ( "agent isn't approved", @@ -102,6 +106,7 @@ def __init__(self, timeout_seconds: float = 120.0): self._ready: asyncio.Future[set[str]] | None = None self._requests: asyncio.Queue[Any] = asyncio.Queue() self._active: asyncio.Future[Any] | None = None + self._start_lock = asyncio.Lock() self.tools: set[str] = set() @property @@ -109,22 +114,29 @@ def connected(self) -> bool: return self._owner is not None and not self._owner.done() async def start(self) -> None: - if self._owner is not None and not self._owner.done(): - return - self._ready = asyncio.get_running_loop().create_future() - self._owner = asyncio.create_task(self._run(), name="artemis-xcode-bridge") - try: - self.tools = await asyncio.wait_for(asyncio.shield(self._ready), self.timeout_seconds) - except (*_ERRORS, TimeoutError, asyncio.CancelledError): - await self.close() - raise + async with self._start_lock: + if self._owner is not None and not self._owner.done(): + return + self._ready = asyncio.get_running_loop().create_future() + self._owner = asyncio.create_task(self._run(), name="artemis-xcode-bridge") + try: + self.tools = await asyncio.wait_for( + asyncio.shield(self._ready), self.timeout_seconds + ) + except (*_ERRORS, TimeoutError, asyncio.CancelledError): + await self.close() + raise async def _run(self) -> None: try: - # Preserve DEVELOPER_DIR/MCP_XCODE_PID and the user's approved configuration. - params = StdioServerParameters( - command="xcrun", args=["mcpbridge"], env=dict(os.environ) - ) + # mcpbridge only needs a clean base env plus the Xcode pinning + # vars — do not forward the full environment (API keys) to a + # long-lived child process. + env = get_default_environment() + for key in ("DEVELOPER_DIR", "MCP_XCODE_PID"): + if value := os.environ.get(key): + env[key] = value + params = StdioServerParameters(command="xcrun", args=["mcpbridge"], env=env) async with stdio_client(params) as (reader, writer): async with ClientSession(reader, writer) as session: await session.initialize() @@ -207,6 +219,10 @@ async def close(self) -> None: await owner except asyncio.CancelledError: pass + except Exception as exc: + # Owner failures already propagate to requesters via futures; a + # close() must never mask the error its caller is handling. + logger.debug("Xcode bridge owner exited with an error: %s", exc) if self._ready is not None and self._ready.done() and not self._ready.cancelled(): self._ready.exception() # consume an initialization error on cancellation self.tools.clear() diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py index 7baa2605..61047c26 100644 --- a/artemis/drivers/ios/discovery.py +++ b/artemis/drivers/ios/discovery.py @@ -76,7 +76,10 @@ async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) except (TimeoutError, asyncio.CancelledError): if process.returncode is None: - process.kill() + try: + process.kill() + except ProcessLookupError: + pass await process.communicate() raise if process.returncode: @@ -86,6 +89,41 @@ async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: return stdout +async def plist_to_json(payload: bytes, timeout: float = 30.0) -> bytes: + """Convert an OpenStep/XML/binary plist payload to JSON via ``plutil``. + + ``simctl listapps`` emits OpenStep (ASCII) plists that ``plistlib`` + cannot read; ``plutil -convert json`` accepts every plist flavor. + """ + process = await asyncio.create_subprocess_exec( + "plutil", + "-convert", + "json", + "-o", + "-", + "--", + "-", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(input=payload), timeout) + except (TimeoutError, asyncio.CancelledError): + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.communicate() + raise + if process.returncode != 0: + raise RuntimeError( + f"plutil conversion failed: {stderr.decode(errors='replace').strip()}" + ) + return stdout + + def simctl_available() -> bool: """Whether this host can enumerate iOS simulators at all.""" return sys.platform == "darwin" and shutil.which("xcrun") is not None diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py index bbdb717c..0bcaaa2c 100644 --- a/artemis/drivers/ios/recording.py +++ b/artemis/drivers/ios/recording.py @@ -45,8 +45,11 @@ SIGINT_FLUSH_TIMEOUT_SECONDS = 10.0 TERMINATE_TIMEOUT_SECONDS = 3.0 PROBES_TIMEOUT_SECONDS = 3.0 +FFMPEG_TIMEOUT_SECONDS = 120.0 WATCHDOG_INTERVAL_SECONDS = 0.5 MAX_CONSECUTIVE_FAILURES = 3 +MIN_HEALTHY_SEGMENT_SECONDS = 10.0 +MAX_CONCURRENT_CONVERSIONS = 2 STDERR_BUFFER_LINES = 200 @@ -95,14 +98,30 @@ async def probe_display_dimensions(device_id: str) -> tuple[int, int] | None: stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, ) + except OSError: + return None + try: stdout, _stderr = await asyncio.wait_for(process.communicate(), PROBES_TIMEOUT_SECONDS) - except (OSError, TimeoutError): + except TimeoutError: + await _reap_probe(process) return None + except asyncio.CancelledError: + await _reap_probe(process) + raise if process.returncode != 0: return None return _parse_display_dimensions(stdout.decode(errors="replace")) +async def _reap_probe(process: asyncio.subprocess.Process) -> None: + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.communicate() + + async def finalize_mov_to_mp4( source_path: Path, output_path: Path, @@ -160,8 +179,21 @@ async def finalize_mov_to_mp4( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - _stdout, stderr = await process.communicate() - metadata = await probe_video_segment(temporary_path) + try: + _stdout, stderr = await asyncio.wait_for( + process.communicate(), FFMPEG_TIMEOUT_SECONDS + ) + metadata = await asyncio.wait_for( + probe_video_segment(temporary_path), PROBES_TIMEOUT_SECONDS * 10 + ) + except TimeoutError: + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.communicate() + raise RuntimeError("iOS recording finalization timed out") valid = ( process.returncode == 0 and temporary_path.exists() @@ -178,6 +210,8 @@ async def finalize_mov_to_mp4( ) except OSError as exc: logger.error(f"iOS recording finalization failed: {exc}") + except RuntimeError as exc: + logger.error(f"iOS recording finalization failed: {exc}") if temporary_path.exists(): temporary_path.unlink() return False @@ -193,6 +227,7 @@ def __init__(self, device_id: str): self._lock = asyncio.Lock() self._max_duration_seconds = 900 self._consecutive_failures = 0 + self._conversion_semaphore = asyncio.Semaphore(MAX_CONCURRENT_CONVERSIONS) @property def session(self) -> IosRecordingSession | None: @@ -260,12 +295,18 @@ async def _reap(self, process: asyncio.subprocess.Process) -> float: await asyncio.wait_for(process.wait(), SIGINT_FLUSH_TIMEOUT_SECONDS) except (ProcessLookupError, TimeoutError): if process.returncode is None: - process.terminate() + try: + process.terminate() + except ProcessLookupError: + return end_monotonic try: await asyncio.wait_for(process.wait(), TERMINATE_TIMEOUT_SECONDS) except (ProcessLookupError, TimeoutError): if process.returncode is None: - process.kill() + try: + process.kill() + except ProcessLookupError: + pass await process.wait() return end_monotonic @@ -344,9 +385,10 @@ async def _convert_record(self, session: IosRecordingSession, record: dict[str, source = Path(record["path"]) output = Path(record["output_path"]) span = max(0.001, float(record["end"]) - float(record["start"])) - ok = await finalize_mov_to_mp4( - source, output, int(record["width"] or 1080), int(record["height"] or 1920), span - ) + async with self._conversion_semaphore: + ok = await finalize_mov_to_mp4( + source, output, int(record["width"] or 1080), int(record["height"] or 1920), span + ) if ok: record["conversion_done"] = True else: @@ -370,7 +412,6 @@ async def _roll(self, session: IosRecordingSession, end_monotonic: float, reason if session.is_active: try: await self._spawn_recorder(session) - self._consecutive_failures = 0 restarted = True except asyncio.CancelledError: raise @@ -405,8 +446,14 @@ async def _watchdog(self, session: IosRecordingSession) -> None: ) age = time.monotonic() - (session.segment_started_monotonic or time.monotonic()) overdue = age >= self._max_duration_seconds - if not crashed and not rotated and not overdue: + if crashed and age < MIN_HEALTHY_SEGMENT_SECONDS: + # A respawn that dies before surviving a healthy interval + # is a crash loop, not recovery — count it even though a + # successful respawn would otherwise reset the counter. + self._consecutive_failures += 1 + elif not crashed and age >= MIN_HEALTHY_SEGMENT_SECONDS: self._consecutive_failures = 0 + if not crashed and not rotated and not overdue: continue reason = ( "recorder exit" if crashed else "rotation" if rotated else "duration limit" @@ -467,6 +514,11 @@ async def seal(self, through_time: float | None = None) -> None: else time.monotonic() ) end_monotonic = min(end_monotonic, time.monotonic()) + if ( + session.segment_started_monotonic is not None + and end_monotonic <= session.segment_started_monotonic + ): + return await self._roll(session, end_monotonic, "seal") async def _finalize(self, session: IosRecordingSession) -> Path | None: diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index 05b04e15..794ce646 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -29,6 +29,7 @@ from artemis.drivers.ios.discovery import ( clear_ios_simulator_cache, parse_simctl_devices, + plist_to_json, run_xcrun, ) from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy @@ -40,6 +41,20 @@ logger = get_logger(__name__) +async def _parse_listapps_output(raw: bytes) -> dict[str, Any]: + """Parse ``simctl listapps`` output into a bundle-id keyed dict. + + simctl emits OpenStep (ASCII) plists on older Xcode, XML/binary on newer + ones; ``plistlib`` only reads the latter, so OpenStep payloads are routed + through ``plutil -convert json``. + """ + try: + return plistlib.loads(raw) + except (plistlib.InvalidFileException, ValueError): + pass + return json.loads(await plist_to_json(raw)) + + class XcodeSimulatorDriver(BaseDeviceDriver): """One explicitly selected iOS simulator and one native interaction session. @@ -125,7 +140,11 @@ async def _resolve_device(self) -> dict[str, Any]: async def connect(self) -> None: async with self._connect_lock: if self._session_key: - return + if self._bridge.connected: + return + # The bridge retires itself on tool-call failures; release the + # orphaned native session before starting over. + await self.disconnect() candidate = await self._resolve_device() if candidate.get("state") == "Shutdown": await run_xcrun("simctl", "boot", self._device_id) @@ -176,7 +195,7 @@ async def connect(self) -> None: raise RuntimeError("Xcode did not return a device interaction session key.") if ( not session.get("deviceIsSimulator") - or session.get("deviceUUID", "").lower() != self._device_id.lower() + or str(session.get("deviceUUID") or "").lower() != self._device_id.lower() ): raise RuntimeError("Xcode selected a different device; refusing to interact.") await self.get_screen_data(skip_settling=True) @@ -191,7 +210,7 @@ async def connect(self) -> None: error.workspace_path = self._workspace_path try: await self.disconnect() - except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: + except (Exception, asyncio.CancelledError) as cleanup_error: logger.warning( f"Could not release the Xcode session after a connection failure: {cleanup_error}", ) @@ -421,7 +440,9 @@ async def install_app(self, app_path: Path) -> str: async def list_apps(self) -> dict[str, str]: async with self._operation_lock: self._require_connected() - data = plistlib.loads(await run_xcrun("simctl", "listapps", self._device_id)) + data = await _parse_listapps_output( + await run_xcrun("simctl", "listapps", self._device_id) + ) return { bundle: info.get("CFBundleDisplayName") or info.get("CFBundleName") or bundle for bundle, info in data.items() diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 4ba4d436..fe358cd7 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -195,6 +195,10 @@ def batch_command( Path | None, typer.Option( "--ios-workspace", + exists=True, + dir_okay=True, + file_okay=False, + resolve_path=True, help="Existing Xcode project/workspace to request iOS first-run agent approval.", ), ] = None, diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 925e82b7..464ac1b2 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -503,7 +503,7 @@ def on_status(sess_info): raise SystemExit(2) except Exception as e: err_msg = str(e) - if "API_KEY" in err_msg or "requires" in err_msg: + if "API_KEY" in err_msg or "api key" in err_msg.lower(): console.print() console.print( Panel( diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 5a3e962a..293f34e5 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -146,8 +146,6 @@ def _get_controller(device_serial: str | None = None, target_platform: str | Non ) controller = UnifiedMobileController(ctx=ctx) _CONTROLLERS[cache_key] = controller - if _GLOBAL_CONTROLLER is None: - _GLOBAL_CONTROLLER = controller return controller if target_serial and target_serial in _CONTROLLERS: diff --git a/artemis/runtime/adb_endpoint.py b/artemis/runtime/adb_endpoint.py index 986a3c11..6c65d3b3 100644 --- a/artemis/runtime/adb_endpoint.py +++ b/artemis/runtime/adb_endpoint.py @@ -180,6 +180,8 @@ def apply_to_environment( target[ADB_ENDPOINT_ID_ENV] = self.lock_scope # An iOS worker never touches ADB: a stale serial must not leak in. target.pop("ADB_DEVICE_SERIAL", None) + # The worker's device id comes from the queue target, not a parent env. + target.pop("ARTEMIS_DEVICE_ID", None) return target def to_dict(self) -> dict[str, Any]: diff --git a/artemis/runtime/device_lock.py b/artemis/runtime/device_lock.py index 9579d234..1b8e92f6 100644 --- a/artemis/runtime/device_lock.py +++ b/artemis/runtime/device_lock.py @@ -939,9 +939,15 @@ def annotate_active_owner( lock_dir = get_temp_dir("device-locks") if device_id is not None: clean_id = cls._normalize_device_id(device_id) - path = lock_dir / f"artemis-device-{clean_id}.lock" - if not path.exists(): - path = lock_dir / "artemis-global-device.lock" + candidates = [ + lock_dir / f"artemis-device-{clean_id}.lock", + *sorted(lock_dir.glob(f"artemis-device-*__{clean_id}.lock")), + lock_dir / "artemis-global-device.lock", + ] + path = next( + (candidate for candidate in candidates if candidate.exists()), + candidates[-1], + ) else: path = cls._find_lock_by_pid(os.getpid()) if path is None: @@ -959,6 +965,7 @@ def annotate_active_owner( "acquired_at": owner.acquired_at, "session_id": str(session_id), "ingress": ingress or owner.ingress or "sdk", + "lock_scope": owner.lock_scope, } return cls._atomic_write_json(path, payload) @@ -971,7 +978,12 @@ def cleanup_stale_locks(cls, device_id: str | None = None) -> int: return removed patterns = ( - [f"artemis-device-{cls._normalize_device_id(device_id)}.lock"] + [ + # Both the bare key and every scoped variant + # (e.g. artemis-device-ios__.lock) target this device. + f"artemis-device-{cls._normalize_device_id(device_id)}.lock", + f"artemis-device-*__{cls._normalize_device_id(device_id)}.lock", + ] if device_id else ["artemis-device-*.lock", "artemis-global-device.lock", "*.lock"] ) diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py index 68126481..6a90d00c 100644 --- a/artemis/utils/visualization.py +++ b/artemis/utils/visualization.py @@ -556,8 +556,11 @@ def _resolve_coordinates(coord: Any, width: int, height: int) -> tuple[int, int] if 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0: px = int(round(x * width)) py = int(round(y * height)) - elif 0 <= x <= 1000 and 0 <= y <= 1000 and (width > 1000 or height > 1000): - # Android normalized 1000 coordinate space + elif 0 <= x <= 1000 and 0 <= y <= 1000 and ( + width > 1000 or height > 1000 or x > width or y > height + ): + # Normalized 0-1000 space — also on small (logical-size) iOS + # screenshots where the values exceed the pixel bounds anyway. px = int(round(x * width / 1000.0)) py = int(round(y * height / 1000.0)) else: diff --git a/docs/ios.md b/docs/ios.md index 6dd1b1a9..e79e48a9 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -113,9 +113,7 @@ Configure iOS through the embedded SDK's builder: from artemis.sdk import Agent from artemis.sdk.builders import AgentConfigBuilder -config = ( - AgentConfigBuilder().for_ios_simulator("").with_default_profile("flash").build() -) +config = AgentConfigBuilder().for_ios_simulator("").build() agent = Agent(config=config) ``` diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index cd7197ba..35edd3c6 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -1240,6 +1240,11 @@ async def mobile_diagnose( """ fixes_applied: list[dict[str, Any]] = [] platform_name = (platform or "android").strip().lower() + if platform_name not in ("android", "ios"): + return { + "verdict": "blocked", + "error": f"Invalid platform '{platform}'. Expected 'android' or 'ios'.", + } requested_device = device_serial.strip() if device_serial and device_serial.strip() else None avd_name = ( launch_avd.strip() if launch_avd and launch_avd.strip() and platform_name != "ios" else None diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index 23e21b5a..cafa1928 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -270,11 +270,17 @@ def mobile_run_task( app_path: Optional local APK path to install before running. expected_output_desc: Optional, Pro only. If set, a summarization agent writes a report to `output.md` in `notes_dir`. Ignored for Flash. - device_serial: Optional device serial (e.g. "emulator-5554") to bind - execution to a specific device; distinct devices run concurrently. - If omitted, an available device is selected automatically. When - several devices are attached, confirm the target with the user first - (`adb devices -l` lists serials and authorization states). + device_serial: Optional device serial to bind execution to a specific + device; distinct devices run concurrently. For `platform="android"`, + an ADB serial (e.g. "emulator-5554"). For `platform="ios"`, a + simulator UDID (`xcrun simctl list devices` lists them); omit to + auto-select a booted simulator. + platform: Optional target platform: `"android"` (default, ADB) or + `"ios"` (local Xcode 27 simulator). iOS requires macOS; locked-app + mode is not supported on iOS. + ios_workspace: Optional Xcode project/workspace path used to request + first-run device-interaction approval for iOS runs. Ignored for + Android. verification_level: Optional, Pro only. Coarse Checker preset: `"off"` (no audit; the Operator self-reports), `"final"` (one exit review against the goal, the default), `"checkpoints"` (every plan @@ -504,6 +510,9 @@ def mobile_run_task( env = os.environ.copy() env["ARTEMIS_SESSION_ID"] = trace_id env["ARTEMIS_TASK_INGRESS"] = "mcp" + if platform == "ios": + # An iOS worker never touches ADB: a stale serial must not leak in. + env.pop("ADB_DEVICE_SERIAL", None) if device_serial: if platform == "ios": env["ARTEMIS_DEVICE_ID"] = device_serial diff --git a/pyright-core.json b/pyright-core.json index 4252abfe..2d9ede52 100644 --- a/pyright-core.json +++ b/pyright-core.json @@ -8,6 +8,8 @@ "artemis/runtime/adb_endpoint.py", "artemis/runtime/process_probe.py", "artemis/drivers/ios", + "artemis/runtime/ios_device_pool.py", + "artemis/mcp/actuators/ios.py", "artemis/mcp/action_types.py", "artemis/data_engine/models.py", "artemis/agents/explorer/native_runner.py", diff --git a/tests/unit/drivers/test_ios_driver.py b/tests/unit/drivers/test_ios_driver.py index 117aedc2..faf4491b 100644 --- a/tests/unit/drivers/test_ios_driver.py +++ b/tests/unit/drivers/test_ios_driver.py @@ -113,12 +113,21 @@ async def command(*arguments, timeout=30): if arguments == ("simctl", "list", "devices", "--json"): return json.dumps(inventory).encode() if arguments[:2] == ("simctl", "listapps"): - return plistlib.dumps({"com.example.app": {"CFBundleDisplayName": "Example"}}) + # Real simctl emits an OpenStep/ASCII plist that plistlib cannot + # parse — the driver must route it through plutil. + return ( + b'{ "com.example.app" = { CFBundleDisplayName = Example; ' + b'CFBundleIdentifier = "com.example.app"; }; }' + ) return b"" + async def fake_plutil(payload: bytes, timeout: float = 30.0) -> bytes: + return json.dumps({"com.example.app": {"CFBundleDisplayName": "Example"}}).encode() + commands = AsyncMock(side_effect=command) monkeypatch.setattr(xcode_driver.sys, "platform", "darwin") monkeypatch.setattr(xcode_driver, "run_xcrun", commands) + monkeypatch.setattr(xcode_driver, "plist_to_json", fake_plutil) monkeypatch.setattr(xcode_driver, "XcodeBridge", lambda: native) return SimpleNamespace( driver=XcodeSimulatorDriver(), diff --git a/tests/unit/runtime/test_device_lock.py b/tests/unit/runtime/test_device_lock.py index 008f4256..51cae530 100644 --- a/tests/unit/runtime/test_device_lock.py +++ b/tests/unit/runtime/test_device_lock.py @@ -326,6 +326,28 @@ def test_active_owner_is_discoverable_and_can_be_annotated(monkeypatch): owner_lock.release() +def test_annotation_preserves_lock_scope(monkeypatch): + """Annotating a scoped (e.g. iOS) owner must not strip its lock_scope.""" + monkeypatch.setenv("ARTEMIS_TASK_INGRESS", "daemon") + owner_lock = DeviceExecutionLock( + "00008100-000000000000001E", "iOS task", lock_scope="ios" + ) + owner_lock.acquire() + try: + assert DeviceExecutionLock.annotate_active_owner( + session_id="ios-session", + device_id="00008100-000000000000001E", + ) + annotated = DeviceExecutionLock.get_active_owner( + "00008100-000000000000001E", lock_scope="ios" + ) + assert annotated is not None + assert annotated.lock_scope == "ios" + assert annotated.session_id == "ios-session" + finally: + owner_lock.release() + + def test_multi_device_locks_can_run_concurrently(): lock_a = DeviceExecutionLock("emulator-5554", "task on device A") lock_b = DeviceExecutionLock("pixel-9-test", "task on device B") From 1442a6fcd3e218659ea26c0a79690625d6df9997 Mon Sep 17 00:00:00 2001 From: James Lee Date: Tue, 6 Oct 2026 18:47:38 +0900 Subject: [PATCH 27/32] Align press_key vocabulary and filter logcat-only tools on iOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Operator/schema key enum gains POWER, VOLUME_UP, VOLUME_DOWN — the iOS driver and platform guidance already support them; both platforms map them natively (regenerated action-surface fixtures) - action_names translation now keeps power/volume keys bare so iOS drivers receive supported names - Diagnoser drops the logcat-only analyze_logs tool on iOS - mobile_run_task app_path doc covers iOS .app directories - Regression test pins the iOS key set to the operator gate Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/agents/diagnoser/diagnoser.py | 5 +- artemis/agents/operator/operator.py | 10 +- artemis/mcp/action_names.py | 14 +- artemis/mcp/action_specs.py | 6 +- mcp_server/tools/task_runner.py | 3 +- .../action_surfaces/operator_shells.json | 305 +++++++-------- .../action_surfaces/tool_declarations.json | 363 +++++++++--------- tests/unit/agents/test_operator.py | 10 +- tests/unit/test_ios_parity.py | 8 + 9 files changed, 384 insertions(+), 340 deletions(-) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 133893ad..3518a7f7 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -114,7 +114,10 @@ def _build_traced_tools(self) -> list: get_submit_answer_tool(self.ctx), ] if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": - all_tools = [t for t in all_tools if t.name != "run_adb_command"] + # logcat-backed analysis can never succeed on iOS. + all_tools = [ + t for t in all_tools if t.name not in ("run_adb_command", "analyze_logs") + ] if not self.is_device_online: logger.info( "Diagnoser running in OFFLINE environment: stripping adb short" diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index 49bd3abf..4514a2b2 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -72,7 +72,15 @@ } # Bare key names press_key accepts (case-insensitive, optional KEYCODE_ prefix). -SUPPORTED_PRESS_KEYS = ("ENTER", "BACK", "HOME", "APP_SWITCH") +SUPPORTED_PRESS_KEYS = ( + "ENTER", + "BACK", + "HOME", + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN", +) from artemis.agents.operator.prompts import ( OPERATOR_MAX_TOOL_ITERATIONS, diff --git a/artemis/mcp/action_names.py b/artemis/mcp/action_names.py index 78cef0a5..88ca45ad 100644 --- a/artemis/mcp/action_names.py +++ b/artemis/mcp/action_names.py @@ -151,7 +151,19 @@ def to_canonical_call( # bare key word for the common keys. Anything else keeps its original spelling # so the driver can forward it verbatim (arbitrary KEYCODE_* / numeric codes). bare = keycode.removeprefix("KEYCODE_").lower() if keycode else "" - known = {"home", "back", "enter", "delete", "tab", "search", "menu", "app_switch"} + known = { + "home", + "back", + "enter", + "delete", + "tab", + "search", + "menu", + "app_switch", + "power", + "volume_up", + "volume_down", + } return "press_key", {"key": bare if bare in known else keycode} if verb == "back": diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 1ea69a11..60a63779 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -565,13 +565,13 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action operator=OperatorDialect( description=( "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK," - " HOME, APP_SWITCH)." + " HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN)." ), params=( ParamSpec( "key", - Literal["ENTER", "BACK", "HOME", "APP_SWITCH"], - "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", + Literal["ENTER", "BACK", "HOME", "APP_SWITCH", "POWER", "VOLUME_UP", "VOLUME_DOWN"], + "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", ), ), ), diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index cafa1928..4b6d94f2 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -267,7 +267,8 @@ def mobile_run_task( model: `"Flash"` or `"Pro"` — see model selection above. locked_app_package: Optional package name to lock execution to; the agent auto-launches it and restricts actions to that app. - app_path: Optional local APK path to install before running. + app_path: Optional local app path to install before running (APK for + Android, a simulator-built `.app` directory for iOS). expected_output_desc: Optional, Pro only. If set, a summarization agent writes a report to `output.md` in `notes_dir`. Ignored for Flash. device_serial: Optional device serial to bind execution to a specific diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index 6f3e6c81..19ebb364 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -1,11 +1,15 @@ { "click": { - "type": "function", "function": { - "name": "click", "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "click", "parameters": { "properties": { + "delay_ms": { + "default": 100, + "description": "Delay in milliseconds between consecutive clicks. Default is 100.", + "type": "integer" + }, "target": { "anyOf": [ { @@ -36,11 +40,6 @@ "default": 1, "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", "type": "integer" - }, - "delay_ms": { - "default": 100, - "description": "Delay in milliseconds between consecutive clicks. Default is 100.", - "type": "integer" } }, "required": [ @@ -48,18 +47,19 @@ ], "type": "object" } - } + }, + "type": "function" }, "input_text": { - "type": "function", "function": { - "name": "input_text", "description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').", + "name": "input_text", "parameters": { "properties": { - "text": { - "description": "The text content to input. Supports multi-line content with '\\n'.", - "type": "string" + "clear_exist": { + "default": true, + "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", + "type": "boolean" }, "target": { "anyOf": [ @@ -87,10 +87,9 @@ "default": null, "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." }, - "clear_exist": { - "default": true, - "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", - "type": "boolean" + "text": { + "description": "The text content to input. Supports multi-line content with '\\n'.", + "type": "string" } }, "required": [ @@ -99,13 +98,115 @@ ], "type": "object" } - } + }, + "type": "function" + }, + "long_press": { + "function": { + "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "long_press", + "parameters": { + "properties": { + "duration": { + "default": 1000, + "description": "Long press duration in milliseconds (default 1000).", + "type": "integer" + }, + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + }, + "target_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + } + }, + "required": [ + "target" + ], + "type": "object" + } + }, + "type": "function" + }, + "manage_app": { + "function": { + "description": "[ACTION] Launch or force stop a specified application.", + "name": "manage_app", + "parameters": { + "properties": { + "action": { + "description": "The action type.", + "enum": [ + "launch", + "stop" + ], + "type": "string" + }, + "app_name": { + "description": "Display name or package name of the application.", + "type": "string" + } + }, + "required": [ + "action", + "app_name" + ], + "type": "object" + } + }, + "type": "function" + }, + "press_key": { + "function": { + "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", + "name": "press_key", + "parameters": { + "properties": { + "key": { + "description": "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", + "enum": [ + "ENTER", + "BACK", + "HOME", + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN" + ], + "type": "string" + } + }, + "required": [ + "key" + ], + "type": "object" + } + }, + "type": "function" }, "swipe": { - "type": "function", "function": { + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n\u2022 Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n\u2022 Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "parameters": { "properties": { "direction": { @@ -126,20 +227,17 @@ "default": null, "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." }, - "start": { + "duration": { "anyOf": [ { - "items": { - "type": "integer" - }, - "type": "array" + "type": "integer" }, { "type": "null" } ], "default": null, - "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." + "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." }, "end": { "anyOf": [ @@ -156,10 +254,16 @@ "default": null, "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." }, - "target": { + "gesture": { "anyOf": [ { - "type": "integer" + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" }, { "items": { @@ -167,27 +271,15 @@ }, "type": "array" }, - { - "type": "string" - }, { "type": "null" } ], "default": null, - "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." }, - "gesture": { + "start": { "anyOf": [ - { - "enum": [ - "up", - "down", - "left", - "right" - ], - "type": "string" - }, { "items": { "type": "integer" @@ -199,10 +291,19 @@ } ], "default": null, - "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." }, - "target_description": { + "target": { "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, { "type": "string" }, @@ -211,83 +312,30 @@ } ], "default": null, - "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." }, - "duration": { + "target_description": { "anyOf": [ { - "type": "integer" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." - } - }, - "type": "object" - } - } - }, - "press_key": { - "type": "function", - "function": { - "name": "press_key", - "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).", - "parameters": { - "properties": { - "key": { - "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", - "enum": [ - "ENTER", - "BACK", - "HOME", - "APP_SWITCH" - ], - "type": "string" - } - }, - "required": [ - "key" - ], - "type": "object" - } - } - }, - "manage_app": { - "type": "function", - "function": { - "name": "manage_app", - "description": "[ACTION] Launch or force stop a specified application.", - "parameters": { - "properties": { - "action": { - "description": "The action type.", - "enum": [ - "launch", - "stop" - ], - "type": "string" - }, - "app_name": { - "description": "Display name or package name of the application.", - "type": "string" + "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." } }, - "required": [ - "action", - "app_name" - ], "type": "object" } - } + }, + "type": "function" }, "wait_for_delay": { - "type": "function", "function": { + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse\u2014whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "name": "wait_for_delay", - "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "parameters": { "properties": { "time_in_ms": { @@ -300,52 +348,7 @@ ], "type": "object" } - } - }, - "long_press": { - "type": "function", - "function": { - "name": "long_press", - "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", - "parameters": { - "properties": { - "target": { - "anyOf": [ - { - "type": "integer" - }, - { - "items": { - "type": "integer" - }, - "type": "array" - } - ], - "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." - }, - "target_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." - }, - "duration": { - "default": 1000, - "description": "Long press duration in milliseconds (default 1000).", - "type": "integer" - } - }, - "required": [ - "target" - ], - "type": "object" - } - } + }, + "type": "function" } } diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 33f5ac85..d1527c33 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -1,131 +1,92 @@ { - "CLICK_TOOL": { - "type": "function", - "function": { - "name": "click", - "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", - "parameters": { - "type": "object", - "properties": { - "target": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "array", - "items": { - "type": "integer" - } - } - ], - "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." - }, - "target_description": { - "type": "string", - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." - }, - "times": { - "type": "integer", - "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1." - }, - "delay_ms": { - "type": "integer", - "description": "Delay in milliseconds between consecutive clicks. Default is 100." - } - }, - "required": [ - "target" - ] - } - } - }, "CLICK_SEQUENCE_TOOL": { - "type": "function", "function": { - "name": "click_sequence", "description": "[ACTION] Executes a sequence of taps one by one in order on the specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks in the sequence have completed.", + "name": "click_sequence", "parameters": { - "type": "object", "properties": { + "delay_ms": { + "description": "Delay between consecutive taps in milliseconds (default 50ms).", + "type": "integer" + }, "sequence": { - "type": "array", + "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]].", "items": { - "type": "array", + "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]).", "items": { "type": "integer" }, - "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])." + "type": "array" }, - "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." + "type": "array" }, "target_descriptions": { - "type": "array", + "description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself.", "items": { "type": "string" }, - "description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself." - }, - "delay_ms": { - "type": "integer", - "description": "Delay between consecutive taps in milliseconds (default 50ms)." + "type": "array" } }, "required": [ "sequence", "target_descriptions" - ] + ], + "type": "object" } - } + }, + "type": "function" }, - "LONG_PRESS_TOOL": { - "type": "function", + "CLICK_TOOL": { "function": { - "name": "long_press", - "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "click", "parameters": { - "type": "object", "properties": { + "delay_ms": { + "description": "Delay in milliseconds between consecutive clicks. Default is 100.", + "type": "integer" + }, "target": { "anyOf": [ { "type": "integer" }, { - "type": "array", "items": { "type": "integer" - } + }, + "type": "array" } ], - "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { - "type": "string", - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index.", + "type": "string" }, - "duration": { - "type": "integer", - "description": "Long press duration in milliseconds (default 1000)." + "times": { + "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", + "type": "integer" } }, "required": [ "target" - ] + ], + "type": "object" } - } + }, + "type": "function" }, "INPUT_TEXT_TOOL": { - "type": "function", "function": { - "name": "input_text", "description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').", + "name": "input_text", "parameters": { - "type": "object", "properties": { - "text": { - "type": "string", - "description": "The text content to input. Supports multi-line content with '\\n'." + "clear_exist": { + "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", + "type": "boolean" }, "target": { "anyOf": [ @@ -133,61 +94,41 @@ "type": "integer" }, { - "type": "array", "items": { "type": "integer" - } + }, + "type": "array" } ], "description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { - "type": "string", - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index.", + "type": "string" }, - "clear_exist": { - "type": "boolean", - "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content." + "text": { + "description": "The text content to input. Supports multi-line content with '\\n'.", + "type": "string" } }, "required": [ "text", "target" - ] + ], + "type": "object" } - } + }, + "type": "function" }, - "SWIPE_TOOL": { - "type": "function", + "LONG_PRESS_TOOL": { "function": { - "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", + "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "long_press", "parameters": { - "type": "object", "properties": { - "direction": { - "type": "string", - "enum": [ - "up", - "down", - "left", - "right" - ], - "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." - }, - "start": { - "type": "array", - "items": { - "type": "integer" - }, - "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." - }, - "end": { - "type": "array", - "items": { - "type": "integer" - }, - "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." + "duration": { + "description": "Long press duration in milliseconds (default 1000).", + "type": "integer" }, "target": { "anyOf": [ @@ -195,120 +136,182 @@ "type": "integer" }, { - "type": "array", "items": { "type": "integer" - } - }, - { - "type": "string" + }, + "type": "array" } ], - "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." - }, - "gesture": { - "anyOf": [ - { - "type": "string", - "enum": [ - "up", - "down", - "left", - "right" - ] - }, - { - "type": "array", - "items": { - "type": "integer" - } - } - ], - "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { - "type": "string", - "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index.", + "type": "string" + } + }, + "required": [ + "target" + ], + "type": "object" + } + }, + "type": "function" + }, + "MANAGE_APP_TOOL": { + "function": { + "description": "[ACTION] Launch or force stop a specified application.", + "name": "manage_app", + "parameters": { + "properties": { + "action": { + "description": "The action type.", + "enum": [ + "launch", + "stop" + ], + "type": "string" }, - "duration": { - "type": "integer", - "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." + "app_name": { + "description": "Display name or package name of the application.", + "type": "string" } }, - "required": [] + "required": [ + "action", + "app_name" + ], + "type": "object" } - } + }, + "type": "function" }, "PRESS_KEY_TOOL": { - "type": "function", "function": { + "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", "name": "press_key", - "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).", "parameters": { - "type": "object", "properties": { "key": { - "type": "string", + "description": "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", "enum": [ "ENTER", "BACK", "HOME", - "APP_SWITCH" + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN" ], - "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)." + "type": "string" } }, "required": [ "key" - ] + ], + "type": "object" } - } + }, + "type": "function" }, - "MANAGE_APP_TOOL": { - "type": "function", + "SWIPE_TOOL": { "function": { - "name": "manage_app", - "description": "[ACTION] Launch or force stop a specified application.", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n\u2022 Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n\u2022 Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", + "name": "swipe", "parameters": { - "type": "object", "properties": { - "action": { - "type": "string", + "direction": { + "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left).", "enum": [ - "launch", - "stop" + "up", + "down", + "left", + "right" ], - "description": "The action type." + "type": "string" }, - "app_name": { - "type": "string", - "description": "Display name or package name of the application." + "duration": { + "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes).", + "type": "integer" + }, + "end": { + "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", + "items": { + "type": "integer" + }, + "type": "array" + }, + "gesture": { + "anyOf": [ + { + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + }, + "start": { + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", + "items": { + "type": "integer" + }, + "type": "array" + }, + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + }, + "target_description": { + "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.", + "type": "string" } }, - "required": [ - "action", - "app_name" - ] + "required": [], + "type": "object" } - } + }, + "type": "function" }, "WAIT_FOR_DELAY_TOOL": { - "type": "function", "function": { + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse\u2014whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "name": "wait_for_delay", - "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "parameters": { - "type": "object", "properties": { "time_in_ms": { - "type": "integer", - "description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes)." + "description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes).", + "type": "integer" } }, "required": [ "time_in_ms" - ] + ], + "type": "object" } - } + }, + "type": "function" } } diff --git a/tests/unit/agents/test_operator.py b/tests/unit/agents/test_operator.py index 5fcfb7c2..cf512642 100644 --- a/tests/unit/agents/test_operator.py +++ b/tests/unit/agents/test_operator.py @@ -1499,13 +1499,19 @@ async def test_operator_press_key_translation_is_case_insensitive(): assert err is None assert actions == [{"action": "press_key", "keycode": "KEYCODE_APP_SWITCH"}] - # Unsupported keys get a self-explanatory error that lists the accepted names. actions, err = node._translate_and_validate_tool( {"name": "press_key", "args": {"key": "volume_up"}}, mock_state ) + assert err is None + assert actions == [{"action": "press_key", "keycode": "KEYCODE_VOLUME_UP"}] + + # Unsupported keys get a self-explanatory error that lists the accepted names. + actions, err = node._translate_and_validate_tool( + {"name": "press_key", "args": {"key": "delete"}}, mock_state + ) assert actions == [] assert err == ( - "Error: Unsupported key 'volume_up'. Supported keys: ENTER, BACK, HOME, APP_SWITCH." + "Error: Unsupported key 'delete'. Supported keys: ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN." ) diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index 59345753..44bc62d6 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -866,3 +866,11 @@ async def test_select_device_android_unchanged(monkeypatch): assert result["status"] == "success" assert result["selected_serial"] == "emulator-5554" set_target.assert_called_once_with("emulator-5554") + + +def test_ios_press_key_vocabulary_passes_the_operator_gate(): + """Every key the iOS guidance advertises must survive the Operator key gate.""" + from artemis.agents.operator.operator import SUPPORTED_PRESS_KEYS + + ios_supported = {"enter", "home", "power", "volume_up", "volume_down", "app_switch"} + assert ios_supported <= {key.lower() for key in SUPPORTED_PRESS_KEYS} From 76f23c31f98f3acbfea2c3d5067e13768892295e Mon Sep 17 00:00:00 2001 From: James Lee Date: Wed, 7 Oct 2026 02:35:59 +0900 Subject: [PATCH 28/32] Refactor iOS support: unify readiness, platform parsing, and shared helpers - Shared physical_ios_ready() predicate across pool, probe, and driver paths (paired + connected-or-absent tunnelState) - Sync and async validate_explicit_serial now share matching/state rules (UDID or device name) - normalize_device_platform/device_pool_for/target_for_platform in adb_endpoint; adopted at CLI, admin, MCP surfaces - BOOTED_SIMULATOR_ID and DEFAULT_MAX_DURATION_SECONDS replace magic literals across iOS code - Shared helpers: reap_process, devicectl_screenshot, pixel_element, device_matches_identifier - MCP worker env now reuses IosTarget.apply_to_environment; IOS_LOCK_SCOPE replaces 'ios' literals - WDA-aware smoke hints and hierarchy_backend='wda' for physical devices - Removed dead code: duplicate replay /api/devices route, ReplayManager.list_devices/_init_device, fetchDevices JS, no-op driver overrides, unused imports - for_ios_device() is the canonical SDK method; for_ios_simulator kept as alias; IosDeviceProbe with stable probe id and class alias - Docstring/wording sweep: 'iOS device' instead of simulator-only phrasing where hardware is supported Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/admin_console/index.html | 23 --- apps/admin_console/replay_manager.py | 166 +++------------- apps/admin_console/routers/replay.py | 6 - apps/admin_console/routers/system.py | 14 +- apps/admin_console/routers/tasks.py | 20 +- .../services/task_queue_service.py | 41 ++-- artemis/controllers/unified_controller.py | 4 +- artemis/core/diagnostics/device_smoke.py | 34 +++- artemis/core/diagnostics/engine.py | 4 +- artemis/core/diagnostics/probes/ios_probe.py | 28 +-- artemis/drivers/factory.py | 3 +- artemis/drivers/ios/bridge.py | 2 +- artemis/drivers/ios/discovery.py | 184 +++++++++++------- artemis/drivers/ios/hierarchy.py | 29 ++- artemis/drivers/ios/physical_driver.py | 131 +++---------- artemis/drivers/ios/physical_recording.py | 29 ++- artemis/drivers/ios/recording.py | 22 +-- artemis/drivers/ios/wda.py | 26 ++- artemis/drivers/ios/xcode_driver.py | 97 +++++---- artemis/interfaces/cli/commands/batch.py | 13 +- artemis/interfaces/cli/commands/run.py | 3 +- artemis/mcp/actuators/ios.py | 2 +- artemis/mcp/adb_server.py | 6 +- artemis/runtime/adb_endpoint.py | 54 +++++ artemis/runtime/ios_device_pool.py | 161 +++++++-------- artemis/sdk/agent.py | 7 +- artemis/sdk/builders/agent_config_builder.py | 8 +- artemis/utils/video.py | 2 +- docs/ios.md | 8 +- mcp_server/background/task_runner.py | 8 +- mcp_server/tools/diagnose.py | 6 +- mcp_server/tools/task_runner.py | 42 ++-- tests/unit/drivers/test_ios_physical.py | 16 +- tests/unit/test_ios_parity.py | 9 +- 34 files changed, 583 insertions(+), 625 deletions(-) diff --git a/apps/admin_console/index.html b/apps/admin_console/index.html index 0d60c4c0..fc0fc2df 100644 --- a/apps/admin_console/index.html +++ b/apps/admin_console/index.html @@ -1442,29 +1442,6 @@

No devices detected'; - return; - } - - devices.forEach(d => { - const opt = document.createElement('option'); - opt.value = d.serial; - opt.innerText = `${d.serial} (${d.status})`; - select.appendChild(opt); - }); - } catch (err) { - console.error("Failed to fetch devices:", err); - } - } - let replayTools = []; async function fetchReplayTools() { diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index fcccecba..50af4250 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -113,21 +113,15 @@ class ReplayManager: def __init__( self, workspace_root: Path = WORKSPACE_ROOT, - device_id: str = None, original_db_path: Path = None, - init_device: bool = False, ): - """Initializes the ReplayManager and autodetects the connected Android device. + """Initializes the ReplayManager. Args: workspace_root (Path, optional): The absolute path to the workspace root. Defaults to WORKSPACE_ROOT. - device_id (str, optional): The serial ID of the target Android - device. If not specified, autodetects the first connected device. original_db_path (Path, optional): Path to the master SQLite database. Defaults to DB_PATH. - init_device (bool, optional): Whether to initialize the ADB device - connection on startup. """ self.workspace_root = workspace_root self.traces_path = TRACES_PATH @@ -137,62 +131,6 @@ def __init__( self.db_path = Path(original_db_path) if original_db_path else DB_PATH self.images_dir = IMAGES_DIR - # Device connection and autodetection - self.device_id = None - self.w = None - self.h = None - self.adb = None - self.ui_client = None - - if init_device: - try: - self._init_device(device_id) - except Exception as e: - print( - "Warning: Connected device could not be initialized during" - f" ReplayManager startup: {e}" - ) - - def _init_device(self, device_id: str = None): - """Initializes connection to the target device and queries screen metrics.""" - from adbutils import AdbClient - - try: - from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient - except ImportError: - raise ImportError( - "Failed to import UIAutomatorClient. Ensure artemis package is installed in path." - ) - - self.adb = AdbClient(host="localhost", port=5037) - try: - devices = self.adb.device_list() - except Exception as adb_err: - raise ConnectionError( - f"Failed to query device list from ADB server: {adb_err}" - ) from adb_err - - if not devices: - raise ConnectionError( - "No active ADB devices connected. Please connect an Android device via ADB." - ) - - if device_id: - matched_device = next((d.serial for d in devices if d.serial == device_id), None) - if not matched_device: - raise ConnectionError( - f"Requested device '{device_id}' is not connected." - f" Connected devices: {[d.serial for d in devices]}" - ) - self.device_id = matched_device - else: - self.device_id = devices[0].serial - - self.ui_client = UIAutomatorClient(device_id=self.device_id) - ui_data = self.ui_client.get_screen_data() - self.w, self.h = ui_data.width, ui_data.height - print(f"Connected to device: {self.device_id} ({self.w}x{self.h})") - def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path: """Chunks traces, screenshots, and metadata for a specific execution session. @@ -396,58 +334,6 @@ def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path print("Chunking trace completion status: success") return output_dir - def list_devices(self) -> list[dict]: - """Connected Android devices, iOS simulators, and physical iOS devices.""" - devices: list[dict] = [] - try: - from adbutils import AdbClient - - adb = AdbClient(host="localhost", port=5037) - devices.extend( - {"serial": d.serial, "status": "online", "platform": "android"} - for d in adb.device_list() - ) - except Exception as e: - print(f"Warning: Failed to query device list from ADB: {e}") - try: - from artemis.drivers.ios.discovery import list_ios_simulators_sync - - for sim in list_ios_simulators_sync() or []: - devices.append( - { - "serial": sim["udid"], - "status": "online" if sim.get("state") == "Booted" else "shutdown", - "platform": "ios", - "model": sim.get("name"), - } - ) - except Exception as e: - print(f"Warning: Failed to query iOS simulators: {e}") - try: - from artemis.drivers.ios.discovery import ( - is_physical_ios, - list_core_devices_sync, - ) - - for device in list_core_devices_sync() or []: - if not is_physical_ios(device): - continue - connected = ( - device.get("pairing_state") == "paired" - and device.get("connection_state") == "connected" - ) - devices.append( - { - "serial": device["udid"], - "status": "online" if connected else "offline", - "platform": "ios", - "model": device.get("name"), - } - ) - except Exception as e: - print(f"Warning: Failed to query iOS physical devices: {e}") - return devices - def load_session_goal( self, session_id: str, step_dir: Path, original_db_path: str = None ) -> str: @@ -1644,6 +1530,30 @@ def _preemptive_clip_video( except Exception as e: print(f"Warning: Failed to update sandbox SQLite database: {e}") + def _session_device_info(self, session_id: str) -> dict: + """The recorded session's ``device_info`` blob (``{}`` when absent).""" + import sqlite3 + + try: + conn = sqlite3.connect(self.db_path) + try: + row = conn.execute( + "SELECT device_info FROM sessions WHERE session_id = ?", + (session_id,), + ).fetchone() + finally: + conn.close() + except sqlite3.Error: + return {} + if not row or not row[0]: + return {} + try: + info = json.loads(row[0]) + except (ValueError, TypeError): + # Malformed device_info: simulate with the defaults. + return {} + return info if isinstance(info, dict) else {} + @staticmethod def _replay_device_context(device_info: dict, override_device_id: str | None): """Build the simulated ``DeviceContext`` for a replayed session. @@ -1755,25 +1665,7 @@ def create_ctx( except ImportError as import_err: raise ImportError(f"Failed to import Artemis core modules: {import_err}") - import sqlite3 - - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - cursor.execute( - "SELECT device_info FROM sessions WHERE session_id = ?", - (session_id,), - ) - row = cursor.fetchone() - conn.close() - - device_info = {} - if row and row[0]: - try: - device_info = json.loads(row[0]) - except (ValueError, TypeError): - # Malformed device_info: simulate with the defaults below. - pass - + device_info = self._session_device_info(session_id) device_context = self._replay_device_context(device_info, override_device_id) sim_device_id = device_context.device_id print( @@ -1832,6 +1724,7 @@ def instantiate_state( pre_image_meta = json.load(f) initial_goal = self.load_session_goal(session_id, step_dir, str(self.db_path)) + device_info = self._session_device_info(session_id) # State has no fields for the app info or device date (extra="forbid"). ui_hier, decisions, _app_info, _dev_date = self.extract_state_prepopulation_data( @@ -1848,7 +1741,10 @@ def instantiate_state( latest_screenshot=str(step_dir / "pre.jpg") if (step_dir / "pre.jpg").exists() else str(step_dir / "post.jpg"), - operator_raw_data={"width": self.w, "height": self.h}, + operator_raw_data={ + "width": device_info.get("device_width"), + "height": device_info.get("device_height"), + }, current_step_id=step_data["step_id"], subagent_calls=step_data.get("subagent_calls") or [], latest_ui_hierarchy=ui_hier, diff --git a/apps/admin_console/routers/replay.py b/apps/admin_console/routers/replay.py index 2a83b6e2..d6de82c9 100644 --- a/apps/admin_console/routers/replay.py +++ b/apps/admin_console/routers/replay.py @@ -30,12 +30,6 @@ router = APIRouter(tags=["replay"]) -@router.get("/api/devices") -async def list_devices(): - """Dynamically queries the ADB server for connected Android devices.""" - return replay_manager.list_devices() - - @router.get("/api/replay/tools") async def get_replay_tools(): """Returns the list of registered tools available for replay.""" diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index f4fef2a4..b6d6685b 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -28,6 +28,7 @@ adb_server_connection, ) from artemis.core.diagnostics.schema import SystemReadinessReport +from artemis.runtime.adb_endpoint import normalize_device_platform router = APIRouter(prefix="/api/system", tags=["system"]) @@ -115,7 +116,13 @@ async def select_active_device(request: SelectDeviceRequest): if not serial: raise HTTPException(status_code=400, detail="Device serial cannot be empty.") - platform = (request.platform or "android").strip().lower() + try: + platform = normalize_device_platform(request.platform) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) if platform == "ios": # iOS has no ADB probe to retarget; validate the UDID and let the # task payload's device_serial + platform carry the binding. @@ -125,11 +132,6 @@ async def select_active_device(request: SelectDeviceRequest): if rejection: raise HTTPException(status_code=400, detail=rejection) return {"status": "success", "selected_serial": serial, "platform": "ios"} - if platform != "android": - raise HTTPException( - status_code=400, - detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", - ) readiness_engine.set_probe_target_serial(serial) # Return updated readiness diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index 549001f6..df2a14d7 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -20,7 +20,11 @@ from fastapi.responses import StreamingResponse from artemis.core.diagnostics import readiness_engine from artemis.runtime import DeviceExecutionLock, device_pool, ios_device_pool -from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.adb_endpoint import ( + IOS_LOCK_SCOPE, + device_pool_for, + normalize_device_platform, +) try: from admin_console.core.state import state @@ -114,8 +118,9 @@ async def run_task(request: RunRequest): "total_queued": len(state.queue_tasks), } - platform = str(request.platform or "android").strip().lower() or "android" - if platform not in ("android", "ios"): + try: + platform = normalize_device_platform(request.platform) + except ValueError: raise HTTPException( status_code=400, detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", @@ -134,12 +139,9 @@ async def run_task(request: RunRequest): # fail downstream with a clear error instead. if request.device_serial: try: - if platform == "ios": - rejection = await ios_device_pool.validate_explicit_serial_async( - request.device_serial - ) - else: - rejection = await device_pool.validate_explicit_serial_async(request.device_serial) + rejection = await device_pool_for(platform).validate_explicit_serial_async( + request.device_serial + ) except Exception: rejection = None if rejection: diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index b27a97cc..3f5b7845 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -52,7 +52,13 @@ request_cancel, trace_store, ) -from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE, IosTarget +from artemis.runtime.adb_endpoint import ( + IOS_LOCK_SCOPE, + IosTarget, + device_pool_for, + normalize_device_platform, + target_for_platform, +) logger = logging.getLogger(__name__) @@ -218,16 +224,16 @@ def _enforce_sync() -> None: @staticmethod def _task_target(task_item: dict[str, Any]) -> AdbTarget | IosTarget: - serial = task_item.get("device_serial") - if str(task_item.get("platform") or "android").lower() == "ios": - return IosTarget(serial=str(serial) if serial else None) + serial = str(task_item.get("device_serial") or "") or None endpoint_data = task_item.get("adb_endpoint") endpoint = ( AdbEndpoint.from_mapping(endpoint_data) if isinstance(endpoint_data, dict) - else current_adb_endpoint() + else None + ) + return target_for_platform( + task_item.get("platform"), serial=serial, endpoint=endpoint ) - return AdbTarget(endpoint=endpoint, serial=str(serial) if serial else None) @classmethod def _broadcast_event(cls, event_type: str, data: Any): @@ -985,14 +991,9 @@ async def _reject_unavailable_device( # the task can proceed and fail downstream with a clear no-device error. if device_serial: try: - if platform == "ios": - from artemis.runtime import ios_device_pool - - rejection = await ios_device_pool.validate_explicit_serial_async(device_serial) - else: - from artemis.runtime import device_pool - - rejection = await device_pool.validate_explicit_serial_async(device_serial) + rejection = await device_pool_for( + platform + ).validate_explicit_serial_async(device_serial) except Exception: rejection = None if rejection: @@ -1088,7 +1089,7 @@ async def enqueue_tasks( or paired physical UDID, the device lock is scoped under ``ios``, and the worker runs with ``--platform ios`` instead of ADB bindings. """ - platform = str(platform or "android").strip().lower() or "android" + platform = normalize_device_platform(platform, strict=False) verification_level = ( str(verification_level).strip().lower() or None if verification_level else None ) @@ -1112,15 +1113,7 @@ async def enqueue_tasks( single_session_id = session_id if (session_id and len(goals) == 1) else None if not device_serial: try: - if platform == "ios": - from artemis.runtime import ios_device_pool - - device_serial = await ios_device_pool.select_device_async() - else: - # Device enumeration may block on ADB. - from artemis.runtime import device_pool - - device_serial = await device_pool.select_device_async() + device_serial = await device_pool_for(platform).select_device_async() except Exception: device_serial = None for i, goal in enumerate(goals): diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 3202b639..82a74ed4 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -689,7 +689,7 @@ def _ios_recording_driver(self) -> "XcodeSimulatorDriver | None": async def _start_ios_recording( self, output_dir: Path | None, max_duration_seconds: int ) -> VideoRecordingResult: - """Start native simctl capture through the iOS driver.""" + """Start native capture through the iOS driver (simctl or devicectl).""" device_id = self._get_device_id() if already_active := recording_already_active(device_id): return already_active @@ -697,7 +697,7 @@ async def _start_ios_recording( if driver is None: return VideoRecordingResult( success=False, - message="iOS recording requires the Xcode simulator driver.", + message="iOS recording requires the iOS driver.", ) try: await driver.start_video_recording( diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 4eff9c22..bbf844a8 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -56,6 +56,7 @@ import time from typing import Any, TypeVar +from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.runtime.device_lock import DeviceExecutionLock from third_party.mobile_use.utils.logger import get_logger @@ -97,7 +98,7 @@ def _uiautomator_fix(serial: str | None) -> list[str]: def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: - """iOS Simulator repair steps (simctl/Xcode instead of adb/UIAutomator).""" + """iOS repair steps (simctl/Xcode for simulators, devicectl/WDA for hardware).""" s = serial or "" text = (error or "").lower() if "busy" in text or "another task" in text: @@ -114,6 +115,20 @@ def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: return [ "Install Xcode 27 or later and select it: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", ] + if ( + "webdriveragent" in text + or "wda" in text + or "physical" in text + or "pair" in text + or "trust" in text + or "devicectl" in text + or "developer mode" in text + ): + return [ + f"List paired devices: xcrun devicectl list devices; check capture: xcrun devicectl device capture screenshot --device {s} --destination /tmp/ios.png", + "Pair the device (plug in, tap Trust), enable Developer Mode under Settings > Privacy & Security, and ensure WebDriverAgent is installed " + "(build-for-testing via Xcode, or set ARTEMIS_IOS_WDA_URL to a running server).", + ] if "simulator" in text or "udid" in text or "boot" in text or "not available" in text: return [ f"List simulators: xcrun simctl list devices; boot one with: xcrun simctl boot {s}", @@ -127,7 +142,7 @@ def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: def fix_for_error(error: str | None, serial: str | None, platform: str = "android") -> list[str]: """Map an error string to concrete repair steps (substring match, case-insensitive).""" - if platform.lower() == "ios": + if normalize_device_platform(platform, strict=False) == "ios": return _ios_fix_for_error(error, serial) if not error: return [] @@ -297,14 +312,15 @@ async def smoke_test_device( ) -> dict[str, Any]: """Observe the device exactly like ``mobile_get_device_state`` and report a verdict. - ``platform="ios"`` exercises the native Xcode 27 simulator path - (``simctl`` + mcpbridge session) instead of ADB/UIAutomator. + ``platform="ios"`` exercises the native iOS path (``simctl`` + mcpbridge + for simulators, ``devicectl`` + WebDriverAgent for physical devices) + instead of ADB/UIAutomator. Never raises. Returns:: {"ok": bool, "serial": str | None, "elapsed_seconds": float, "screenshot_bytes": int | None, "element_count": int | None, - "hierarchy_backend": "helper" | "uiautomator" | "xcode" | None, + "hierarchy_backend": "helper" | "uiautomator" | "xcode" | "wda" | None, "error": str | None, "fix": list[str]} """ started = time.monotonic() @@ -341,7 +357,7 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any " skipping the screen capture so the running task is not disturbed." ) - is_ios = platform.lower() == "ios" + is_ios = normalize_device_platform(platform, strict=False) == "ios" try: from artemis.mcp import adb_server @@ -399,7 +415,11 @@ async def _observe() -> Any: result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None)) result["element_count"] = _count_elements(getattr(device_data, "elements", None)) if is_ios: - result["hierarchy_backend"] = "xcode" + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + result["hierarchy_backend"] = ( + "wda" if isinstance(getattr(controller, "_driver", None), PhysicalIosDriver) else "xcode" + ) else: from artemis.clients.screen_client_factory import describe_backend diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index a46393a7..56b93cf8 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -27,7 +27,7 @@ LLMCredentialsProbe, VisionOCRProbe, ) -from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe +from artemis.core.diagnostics.probes.ios_probe import IosDeviceProbe from artemis.core.diagnostics.probes.runtime_probe import ( PythonRuntimeProbe, SystemConfigProbe, @@ -68,7 +68,7 @@ def __init__(self): self._credentials_probe = LLMCredentialsProbe() self._ocr_probe = VisionOCRProbe() self._adb_probe = AdbDeviceProbe() - self._ios_probe = IosSimulatorProbe() + self._ios_probe = IosDeviceProbe() self._report_cache: SystemReadinessReport | None = None self._report_cache_time = 0.0 self._report_cache_generation = -1 diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py index b87b0272..cd2990bb 100644 --- a/artemis/core/diagnostics/probes/ios_probe.py +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -20,7 +20,6 @@ """ import asyncio -import re from artemis.core.diagnostics.probes.base import BaseProbe from artemis.core.diagnostics.schema import ( @@ -33,16 +32,19 @@ is_physical_ios, list_core_devices, list_ios_simulators, + parse_xcode_version, + physical_ios_ready, run_xcrun, simctl_available, ) -class IosSimulatorProbe(BaseProbe): - """Reports iOS Simulator automation capability (Xcode 27+ on macOS).""" +class IosDeviceProbe(BaseProbe): + """Reports iOS automation capability: Xcode 27+, simulators, physical.""" @property def probe_id(self) -> str: + # Historical id kept stable: persisted reports reference it. return "ios_simulators" @property @@ -63,8 +65,8 @@ async def probe(self) -> ProbeResult: is_blocker=self.is_blocker, summary="Unavailable", description=( - "iOS Simulator automation requires macOS with Xcode 27+; " - "this host cannot enumerate simulators." + "iOS automation requires macOS with Xcode 27+; " + "this host cannot enumerate iOS devices." ), metadata={"platform_supported": False}, ) @@ -73,9 +75,8 @@ async def probe(self) -> ProbeResult: xcode_version: str | None = None try: raw = await asyncio.wait_for(run_xcrun("xcodebuild", "-version"), timeout=15.0) - match = re.search(rb"Xcode\s+(\d+(?:\.\d+)*)", raw) - if match: - xcode_version = match.group(1).decode() + xcode_version = parse_xcode_version(raw) + if xcode_version: xcode_ok = int(xcode_version.split(".")[0]) >= 27 except (OSError, RuntimeError, TimeoutError): xcode_version = None @@ -84,12 +85,7 @@ async def probe(self) -> ProbeResult: booted = [d for d in (simulators or []) if d.get("state") == "Booted"] core_devices = await list_core_devices() physical = [d for d in (core_devices or []) if is_physical_ios(d)] - connected = [ - d - for d in physical - if d.get("pairing_state") == "paired" - and d.get("connection_state") == "connected" - ] + connected = [d for d in physical if physical_ios_ready(d)] metadata = { "platform_supported": True, "xcode_version": xcode_version, @@ -162,3 +158,7 @@ async def probe(self) -> ProbeResult: ), metadata=metadata, ) + + +# Backward-compatible name from when the probe only covered simulators. +IosSimulatorProbe = IosDeviceProbe diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index 33ccf882..d2b709d5 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -38,9 +38,10 @@ def ios_driver_class(device_id: str | None) -> type[BaseDeviceDriver]: ``PhysicalIosDriver``; simulators, ``booted``, and unknown serials keep ``XcodeSimulatorDriver`` so its existing validation and error paths apply. """ + from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver - if device_id and device_id != "booted": + if device_id and device_id != BOOTED_SIMULATOR_ID: from artemis.drivers.ios.discovery import ( find_physical_ios_device_sync, list_ios_simulators_sync, diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py index d488fb56..2490b55a 100644 --- a/artemis/drivers/ios/bridge.py +++ b/artemis/drivers/ios/bridge.py @@ -81,7 +81,7 @@ def xcode_approval_guidance(workspace_path: str | Path | None = None) -> str: "grants can require approval again.\n" "If Xcode has not recorded a request yet, rerun with " "--platform ios --ios-workspace /absolute/path/YourApp.xcodeproj " - "(or pass workspace_path to for_ios_simulator())." + "(or pass workspace_path to for_ios_device())." ) diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py index bc86fed7..4c9a3e34 100644 --- a/artemis/drivers/ios/discovery.py +++ b/artemis/drivers/ios/discovery.py @@ -22,6 +22,8 @@ import asyncio import json +from pathlib import Path +import re import shutil import subprocess import sys @@ -34,34 +36,58 @@ SIMCTL_QUERY_TIMEOUT = 15.0 -# ``simctl list devices`` takes seconds on a busy host and every iOS consumer -# (readiness probe, device pool validation, /api/devices) enumerates it. Share -# one result briefly so polling UIs do not spawn back-to-back simctl calls. -_SIMULATOR_CACHE_TTL = 10.0 -_simulator_cache: list[dict[str, Any]] | None = None -_simulator_cache_time = 0.0 +# ``simctl`` wildcard that resolves to whichever simulator is currently +# booted; also the public sentinel callers pass to mean "the booted one". +BOOTED_SIMULATOR_ID = "booted" +# ``simctl list devices`` / ``devicectl list devices`` take seconds on a busy +# host and every iOS consumer (readiness probe, device pool validation, +# /api/devices) enumerates them. Share one result briefly so polling UIs do +# not spawn back-to-back tool invocations. +_ENUMERATION_CACHE_TTL = 10.0 -def clear_ios_simulator_cache() -> None: - """Drop the cached enumeration (e.g. after the driver boots a simulator).""" - global _simulator_cache, _simulator_cache_time - _simulator_cache = None - _simulator_cache_time = 0.0 +class _TtlCache: + """Monotonic-TTL cache holding one enumeration result (or nothing).""" -def _cache_simulators(devices: list[dict[str, Any]]) -> list[dict[str, Any]]: - global _simulator_cache, _simulator_cache_time - _simulator_cache = devices - _simulator_cache_time = time.monotonic() - return devices + def __init__(self) -> None: + self._devices: list[dict[str, Any]] | None = None + self._time = 0.0 + def clear(self) -> None: + self._devices = None + self._time = 0.0 -def _cached_simulators() -> list[dict[str, Any]] | None: - if _simulator_cache is None: - return None - if time.monotonic() - _simulator_cache_time > _SIMULATOR_CACHE_TTL: - return None - return _simulator_cache + def store(self, devices: list[dict[str, Any]]) -> list[dict[str, Any]]: + self._devices = devices + self._time = time.monotonic() + return devices + + def get(self) -> list[dict[str, Any]] | None: + if self._devices is None: + return None + if time.monotonic() - self._time > _ENUMERATION_CACHE_TTL: + return None + return self._devices + + +_simulator_cache = _TtlCache() +_core_device_cache = _TtlCache() + + +def clear_ios_simulator_cache() -> None: + """Drop the cached enumeration (e.g. after the driver boots a simulator).""" + _simulator_cache.clear() + + +async def reap_process(process: asyncio.subprocess.Process) -> None: + """Kill a still-running child and drain its pipes; never raises.""" + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.communicate() async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: @@ -76,12 +102,7 @@ async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: try: stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) except (TimeoutError, asyncio.CancelledError): - if process.returncode is None: - try: - process.kill() - except ProcessLookupError: - pass - await process.communicate() + await reap_process(process) raise if process.returncode: raise RuntimeError( @@ -111,12 +132,7 @@ async def plist_to_json(payload: bytes, timeout: float = 30.0) -> bytes: try: stdout, stderr = await asyncio.wait_for(process.communicate(input=payload), timeout) except (TimeoutError, asyncio.CancelledError): - if process.returncode is None: - try: - process.kill() - except ProcessLookupError: - pass - await process.communicate() + await reap_process(process) raise if process.returncode != 0: raise RuntimeError( @@ -130,6 +146,13 @@ def simctl_available() -> bool: return sys.platform == "darwin" and shutil.which("xcrun") is not None +def parse_xcode_version(payload: bytes | str) -> str | None: + """Extract the Xcode version string from ``xcodebuild -version`` output.""" + text = payload.decode(errors="replace") if isinstance(payload, bytes) else payload + match = re.search(r"Xcode\s+(\d+(?:\.\d+)*)", text) + return match.group(1) if match else None + + def parse_simctl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: """Flatten ``simctl list devices --json`` into available iOS entries.""" return [ @@ -151,13 +174,13 @@ async def list_ios_simulators( ) -> list[dict[str, Any]] | None: """All available iOS simulators, or ``None`` when enumeration fails. - Successful enumerations are cached for ``_SIMULATOR_CACHE_TTL`` seconds; + Successful enumerations are cached for ``_ENUMERATION_CACHE_TTL`` seconds; failures are never cached so callers retry against live simctl. """ if not simctl_available(): return None if not force_refresh: - cached = _cached_simulators() + cached = _simulator_cache.get() if cached is not None: return cached try: @@ -166,7 +189,7 @@ async def list_ios_simulators( logger.debug(f"simctl device enumeration failed: {exc}") return None try: - return _cache_simulators(parse_simctl_devices(json.loads(raw))) + return _simulator_cache.store(parse_simctl_devices(json.loads(raw))) except (ValueError, TypeError) as exc: logger.debug(f"simctl device list parse failed: {exc}") return None @@ -179,7 +202,7 @@ def list_ios_simulators_sync( if not simctl_available(): return None if not force_refresh: - cached = _cached_simulators() + cached = _simulator_cache.get() if cached is not None: return cached try: @@ -196,7 +219,7 @@ def list_ios_simulators_sync( if completed.returncode != 0: return None try: - return _cache_simulators(parse_simctl_devices(json.loads(completed.stdout))) + return _simulator_cache.store(parse_simctl_devices(json.loads(completed.stdout))) except (ValueError, TypeError) as exc: logger.debug(f"simctl device list parse failed: {exc}") return None @@ -204,31 +227,24 @@ def list_ios_simulators_sync( # --- CoreDevice (physical iPhone/iPad) enumeration ------------------------- -DEVICECTL_QUERY_TIMEOUT = 20.0 -_core_device_cache: list[dict[str, Any]] | None = None -_core_device_cache_time = 0.0 - - -def clear_core_device_cache() -> None: - """Drop the cached CoreDevice enumeration.""" - global _core_device_cache, _core_device_cache_time - _core_device_cache = None - _core_device_cache_time = 0.0 - - -def _cache_core_devices(devices: list[dict[str, Any]]) -> list[dict[str, Any]]: - global _core_device_cache, _core_device_cache_time - _core_device_cache = devices - _core_device_cache_time = time.monotonic() - return devices - - -def _cached_core_devices() -> list[dict[str, Any]] | None: - if _core_device_cache is None: - return None - if time.monotonic() - _core_device_cache_time > _SIMULATOR_CACHE_TTL: - return None - return _core_device_cache +DEVICECTL_ENUMERATE_TIMEOUT = 20.0 + + +async def devicectl_screenshot( + device_id: str, destination: Path, timeout: float +) -> None: + """Capture one PNG frame from a paired device via ``devicectl``.""" + await run_xcrun( + "devicectl", + "device", + "capture", + "screenshot", + "--device", + device_id, + "--destination", + str(destination), + timeout=timeout, + ) def _device_property(device: dict[str, Any], section: str, key: str) -> Any: @@ -295,18 +311,18 @@ async def list_core_devices( if not simctl_available(): return None if not force_refresh: - cached = _cached_core_devices() + cached = _core_device_cache.get() if cached is not None: return cached try: raw = await run_xcrun( - "devicectl", "list", "devices", "--json-output", "-", timeout=DEVICECTL_QUERY_TIMEOUT + "devicectl", "list", "devices", "--json-output", "-", timeout=DEVICECTL_ENUMERATE_TIMEOUT ) except (OSError, RuntimeError, TimeoutError) as exc: logger.debug(f"devicectl device enumeration failed: {exc}") return None try: - return _cache_core_devices(parse_devicectl_devices(json.loads(raw))) + return _core_device_cache.store(parse_devicectl_devices(json.loads(raw))) except (ValueError, TypeError) as exc: logger.debug(f"devicectl device list parse failed: {exc}") return None @@ -319,7 +335,7 @@ def list_core_devices_sync( if not simctl_available(): return None if not force_refresh: - cached = _cached_core_devices() + cached = _core_device_cache.get() if cached is not None: return cached try: @@ -328,7 +344,7 @@ def list_core_devices_sync( stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - timeout=DEVICECTL_QUERY_TIMEOUT, + timeout=DEVICECTL_ENUMERATE_TIMEOUT, ) except (OSError, subprocess.TimeoutExpired) as exc: logger.debug(f"devicectl device enumeration failed: {exc}") @@ -336,7 +352,7 @@ def list_core_devices_sync( if completed.returncode != 0: return None try: - return _cache_core_devices(parse_devicectl_devices(json.loads(completed.stdout))) + return _core_device_cache.store(parse_devicectl_devices(json.loads(completed.stdout))) except (ValueError, TypeError) as exc: logger.debug(f"devicectl device list parse failed: {exc}") return None @@ -350,15 +366,39 @@ def is_physical_ios(device: dict[str, Any]) -> bool: ) +def physical_ios_ready(device: dict[str, Any]) -> bool: + """Whether a CoreDevice entry is a paired, reachable physical iOS device. + + ``connection_state`` (CoreDevice ``tunnelState``) may be absent on paired + USB devices running older iOS — an absent value is acceptable; + ``"disconnected"`` is not. Every consumer (pool validators, readiness + probe, replay manager, driver) must agree on this rule. + """ + if not is_physical_ios(device): + return False + if device.get("pairing_state") != "paired": + return False + return device.get("connection_state") in ("connected", None) + + +def device_matches_identifier(device: dict[str, Any], identifier: str) -> bool: + """Match a device entry by case-insensitive UDID or exact name.""" + if not identifier: + return False + return ( + str(device.get("udid") or "").lower() == identifier.lower() + or device.get("name") == identifier + ) + + def find_physical_ios_device_sync(identifier: str) -> dict[str, Any] | None: """Match a physical iOS device by UDID or exact name, or ``None``.""" devices = list_core_devices_sync() if devices is None or not identifier: return None - needle = identifier.lower() for device in devices: if not is_physical_ios(device): continue - if device.get("udid", "").lower() == needle or device.get("name") == identifier: + if device_matches_identifier(device, identifier): return device return None diff --git a/artemis/drivers/ios/hierarchy.py b/artemis/drivers/ios/hierarchy.py index bddf69e7..b2d12fdf 100644 --- a/artemis/drivers/ios/hierarchy.py +++ b/artemis/drivers/ios/hierarchy.py @@ -35,6 +35,19 @@ def application_bundle(hierarchy: str) -> str | None: return next(iter(bundles)) if len(bundles) == 1 else None +def pixel_element( + *, text: str, resource_id: str, class_name: str, left: int, top: int, right: int, bottom: int +) -> dict[str, Any]: + """Element dict in the shared Android-shape format used by both iOS parsers.""" + return { + "text": text, + "resource_id": resource_id, + "class": class_name, + "bounds": f"[{left},{top}][{right},{bottom}]", + "parsed_bounds": {"left": left, "top": top, "right": right, "bottom": bottom}, + } + + def parse_hierarchy( hierarchy: str, width: int, height: int ) -> tuple[list[dict[str, Any]], tuple[float, float]]: @@ -101,13 +114,15 @@ def parse_hierarchy( text = value_text elif placeholder and placeholder.group(1): text = placeholder.group(1) - element: dict[str, Any] = { - "text": text, - "resource_id": identifier.group(1) if identifier else "", - "class": line.strip().split(",", 1)[0].split(" ", 1)[0], - "bounds": f"[{left},{top}][{right},{bottom}]", - "parsed_bounds": {"left": left, "top": top, "right": right, "bottom": bottom}, - } + element = pixel_element( + text=text, + resource_id=identifier.group(1) if identifier else "", + class_name=line.strip().split(",", 1)[0].split(" ", 1)[0], + left=left, + top=top, + right=right, + bottom=bottom, + ) if hit: hx, hy = map(float, hit.groups()) element["hit_point"] = [round(hx * scale[0]), round(hy * scale[1])] diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py index bef40e40..1f356091 100644 --- a/artemis/drivers/ios/physical_driver.py +++ b/artemis/drivers/ios/physical_driver.py @@ -42,14 +42,14 @@ from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection from artemis.drivers.ios.discovery import ( + BOOTED_SIMULATOR_ID, + device_matches_identifier, + devicectl_screenshot, is_physical_ios, list_core_devices, run_xcrun, ) -from artemis.drivers.ios.physical_recording import ( - IosPhysicalRecordingSession, - PhysicalIosRecorder, -) +from artemis.drivers.ios.physical_recording import PhysicalIosRecorder from artemis.drivers.ios.wda import ( WdaClient, WdaUnavailableError, @@ -64,7 +64,7 @@ logger = get_logger(__name__) -DEVICECTL_QUERY_TIMEOUT = 30.0 +DEVICECTL_OP_TIMEOUT = 30.0 DEVICECTL_LAUNCH_TIMEOUT = 60.0 DEVICECTL_INSTALL_TIMEOUT = 300.0 WDA_START_TIMEOUT = 45.0 @@ -103,7 +103,6 @@ def __init__( super().__init__( device_id=device_id, width=width, height=height, workspace_path=workspace_path ) - self._physical_recorder: PhysicalIosRecorder | None = None self._launched_pids: dict[str, int] = {} self._wda: WdaClient | None = None self._wda_runner_pid: int | None = None @@ -113,7 +112,7 @@ def __init__( async def _resolve_device(self) -> dict[str, Any]: self._validate_workspace() - if self._device_id in ("", "booted"): + if self._device_id in ("", BOOTED_SIMULATOR_ID): raise ValueError( "Physical iOS devices require --device-serial ; " "'booted' only selects simulators. Find UDIDs via 'xcrun devicectl list devices'." @@ -124,17 +123,14 @@ async def _resolve_device(self) -> dict[str, Any]: raise RuntimeError( "Could not enumerate physical devices; 'xcrun devicectl list devices' failed." ) - needle = self._device_id.lower() matches = [ device for device in devices if is_physical_ios(device) - and ( - device.get("udid", "").lower() == needle - or device.get("name") == self._device_id - ) + and device_matches_identifier(device, self._device_id) ] if not matches: + needle = self._device_id.lower() simulator = [ device for device in devices @@ -193,13 +189,13 @@ async def connect(self) -> None: async def disconnect(self) -> None: try: - if self._physical_recorder is not None: - session = self._physical_recorder.session + if self._recorder is not None: + session = self._recorder.session if session is not None: # A failed (non-active) session still owns frames and # conversions — finalize it too so nothing is lost. try: - await self._physical_recorder.stop() + await self._recorder.stop() except Exception as exc: logger.error( f"Physical iOS recording finalization failed during disconnect: {exc}" @@ -230,7 +226,7 @@ async def disconnect(self) -> None: logger.debug(f"WDA runner termination failed: {exc}") def _require_connected(self) -> None: - if not self._session_key or self._device_id in ("", "booted"): + if not self._session_key or self._device_id in ("", BOOTED_SIMULATOR_ID): raise RuntimeError("Connect the physical iOS driver before interacting.") async def _ensure_wda(self) -> WdaClient: @@ -380,7 +376,7 @@ async def _devicectl_json(self, *arguments: str) -> dict[str, Any]: self._device_id, "--json-output", str(target), - timeout=DEVICECTL_QUERY_TIMEOUT, + timeout=DEVICECTL_OP_TIMEOUT, ) payload = json.loads(target.read_text(encoding="utf-8")) result = payload.get("result") @@ -397,16 +393,8 @@ async def _screenshot_png(self) -> bytes: logger.debug(f"WDA screenshot failed, falling back to devicectl: {exc}") with tempfile.TemporaryDirectory(prefix="artemis-shot-") as tmp: target = Path(tmp) / "shot.png" - await run_xcrun( - "devicectl", - "device", - "capture", - "screenshot", - "--device", - self._device_id, - "--destination", - str(target), - timeout=DEVICECTL_QUERY_TIMEOUT, + await devicectl_screenshot( + self._device_id, target, timeout=DEVICECTL_OP_TIMEOUT ) data = target.read_bytes() if not data: @@ -453,18 +441,6 @@ async def _capture(self) -> ScreenData: platform="ios", ) - async def get_screen_data(self, skip_settling: bool = False) -> ScreenData: - async with self._operation_lock: - return await self._capture() - - def _wda_point(self, x: int, y: int) -> tuple[float, float]: - """Convert screenshot pixels to the WDA point coordinate space.""" - if self._scale is None: - raise RuntimeError("Capture an iOS screen before coordinate interaction.") - if not 0 <= x < self._width or not 0 <= y < self._height: - raise ValueError("iOS input coordinates are outside the current screenshot.") - return x / self._scale[0], y / self._scale[1] - # --- Input --- async def tap( @@ -473,36 +449,23 @@ async def tap( if times < 1 or duration_ms < 0 or delay_ms < 0: raise ValueError("Tap count must be positive and durations nonnegative.") async with self._operation_lock: - previous_size = self.screen_size - await self._capture() - if self.screen_size != previous_size: - raise ValueError( - "The iOS screen changed orientation or size. Observe it again before tapping." - ) - point = self._wda_point(x, y) + await self._capture_unchanged("tapping") + point = self._scaled_point(x, y) for index in range(times): await self._wda.tap(*point, hold_ms=duration_ms) if index < times - 1: await asyncio.sleep(delay_ms / 1000) return True - async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool: - return await self.tap(x, y, duration_ms=duration_ms) - async def swipe( self, start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 800 ) -> bool: if duration_ms <= 0: raise ValueError("Swipe duration must be positive.") async with self._operation_lock: - previous_size = self.screen_size - await self._capture() - if self.screen_size != previous_size: - raise ValueError( - "The iOS screen changed orientation or size. Observe it again before swiping." - ) - start = self._wda_point(start_x, start_y) - end = self._wda_point(end_x, end_y) + await self._capture_unchanged("swiping") + start = self._scaled_point(start_x, start_y) + end = self._scaled_point(end_x, end_y) await self._wda.swipe(*start, *end, duration_ms) return True @@ -515,16 +478,12 @@ async def swipe_direction( if duration_ms <= 0: raise ValueError("Swipe duration must be positive.") async with self._operation_lock: + # Rotation-tolerant like the simulator path: recapture and compute + # from the *current* size rather than refusing like tap. await self._capture() - w, h = self.screen_size - points = { - "up": (w // 2, h * 3 // 4, w // 2, h // 4), - "down": (w // 2, h // 4, w // 2, h * 3 // 4), - "left": (w * 3 // 4, h // 2, w // 4, h // 2), - "right": (w // 4, h // 2, w * 3 // 4, h // 2), - } - start = self._wda_point(points[direction][0], points[direction][1]) - end = self._wda_point(points[direction][2], points[direction][3]) + sx, sy, ex, ey = self._direction_points(direction, *self.screen_size) + start = self._scaled_point(sx, sy) + end = self._scaled_point(ex, ey) await self._wda.swipe(*start, *end, duration_ms) return True @@ -572,20 +531,12 @@ async def tap_element( element, center, error = await self.find_element(query, data) if error or element is None or center is None: return False - point = self._wda_point(*center) + point = self._scaled_point(*center) await self._wda.tap( *point, hold_ms=duration_ms if long_press else 0 ) return True - async def find_element( - self, query: ElementQuery, screen_data: ScreenData | None = None - ) -> tuple[dict[str, Any] | None, list[int] | None, str | None]: - element, center, error = await super().find_element(query, screen_data) - if element is not None: - center = element.get("hit_point", center) - return element, center, error - # --- App lifecycle via devicectl --- async def launch_app(self, package_name: str) -> bool: @@ -675,7 +626,7 @@ async def _terminate_pid(self, pid: int, kill: bool = False) -> None: ] if kill: arguments.append("--kill") - await run_xcrun(*arguments, timeout=DEVICECTL_QUERY_TIMEOUT) + await run_xcrun(*arguments, timeout=DEVICECTL_OP_TIMEOUT) async def stop_app(self, package_name: str) -> bool: async with self._operation_lock: @@ -763,7 +714,7 @@ async def open_url(self, url: str) -> bool: "--device", self._device_id, url, - timeout=DEVICECTL_QUERY_TIMEOUT, + timeout=DEVICECTL_OP_TIMEOUT, ) return True @@ -779,26 +730,6 @@ async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> st # --- Recording (devicectl screenshot polling) --- - @property - def recording_session(self) -> IosPhysicalRecordingSession | None: - if self._physical_recorder is None: - return None - return self._physical_recorder.session - - async def start_video_recording( - self, output_dir: Path | None = None, max_duration_seconds: int = 900 - ) -> None: - self._require_connected() - if self._physical_recorder is None: - self._physical_recorder = PhysicalIosRecorder(self._device_id) - await self._physical_recorder.start(output_dir, max_duration_seconds) - - async def seal_recording_segment(self, through_time: float | None = None) -> None: - if self._physical_recorder is not None: - await self._physical_recorder.seal(through_time) - - async def stop_video_recording(self) -> str | None: - if self._physical_recorder is None: - return None - path = await self._physical_recorder.stop() - return str(path) if path is not None else None + def _new_recorder(self) -> PhysicalIosRecorder: + """Physical recorder: devicectl screenshot polling + ffconcat encode.""" + return PhysicalIosRecorder(self._device_id) diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py index 34eed8f0..dd4f5d22 100644 --- a/artemis/drivers/ios/physical_recording.py +++ b/artemis/drivers/ios/physical_recording.py @@ -23,7 +23,6 @@ import asyncio import contextlib -from io import BytesIO from pathlib import Path import tempfile import time @@ -33,8 +32,11 @@ from PIL import Image from artemis.config.paths import get_temp_dir -from artemis.drivers.ios.discovery import run_xcrun -from artemis.drivers.ios.recording import IosRecordingSession +from artemis.drivers.ios.discovery import devicectl_screenshot, reap_process +from artemis.drivers.ios.recording import ( + DEFAULT_MAX_DURATION_SECONDS, + IosRecordingSession, +) from artemis.utils.video import get_ffmpeg_path, probe_video_segment, write_recording_manifest from third_party.mobile_use.utils.logger import get_logger @@ -42,6 +44,8 @@ CAPTURE_TIMEOUT_SECONDS = 15.0 WATCHDOG_INTERVAL_SECONDS = 0.5 +# devicectl frame polls flake more often than the managed simctl process, so +# the failure cap and the ffmpeg budget are looser than in recording.py. MAX_CONSECUTIVE_FAILURES = 5 @@ -62,8 +66,7 @@ async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: process.communicate(), timeout=FFMPEG_TIMEOUT_SECONDS ) except TimeoutError: - process.kill() - await process.wait() + await reap_process(process) return -1, b"ffmpeg timed out" return process.returncode, stderr @@ -85,7 +88,7 @@ def __init__(self, device_id: str): self._session: IosPhysicalRecordingSession | None = None self._output_dir: Path | None = None self._lock = asyncio.Lock() - self._max_duration_seconds = 900 + self._max_duration_seconds = DEFAULT_MAX_DURATION_SECONDS self._consecutive_failures = 0 @property @@ -102,16 +105,8 @@ async def _capture_frame(self, session: IosPhysicalRecordingSession) -> None: assert session.frames_dir is not None session.frame_index += 1 path = session.frames_dir / f"frame_{session.frame_index:06d}.png" - await run_xcrun( - "devicectl", - "device", - "capture", - "screenshot", - "--device", - self._device_id, - "--destination", - str(path), - timeout=CAPTURE_TIMEOUT_SECONDS, + await devicectl_screenshot( + self._device_id, path, timeout=CAPTURE_TIMEOUT_SECONDS ) captured_at = time.monotonic() if not path.exists() or path.stat().st_size == 0: @@ -346,7 +341,7 @@ async def _watchdog(self, session: IosPhysicalRecordingSession) -> None: async def start( self, output_dir: Path | None = None, - max_duration_seconds: int = 900, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, ) -> IosPhysicalRecordingSession: async with self._lock: if self._session is not None and self._session.is_active: diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py index 0bcaaa2c..d345a711 100644 --- a/artemis/drivers/ios/recording.py +++ b/artemis/drivers/ios/recording.py @@ -30,6 +30,7 @@ from uuid import uuid4 from artemis.config.paths import get_temp_dir +from artemis.drivers.ios.discovery import reap_process from artemis.utils.video import ( get_ffmpeg_path, probe_video_segment, @@ -41,6 +42,8 @@ logger = get_logger(__name__) RECORDING_STARTED_MARKER = "Recording started" +# Longest a single iOS capture may run before auto-stopping (15 minutes). +DEFAULT_MAX_DURATION_SECONDS = 900 STARTUP_TIMEOUT_SECONDS = 30.0 SIGINT_FLUSH_TIMEOUT_SECONDS = 10.0 TERMINATE_TIMEOUT_SECONDS = 3.0 @@ -113,13 +116,7 @@ async def probe_display_dimensions(device_id: str) -> tuple[int, int] | None: return _parse_display_dimensions(stdout.decode(errors="replace")) -async def _reap_probe(process: asyncio.subprocess.Process) -> None: - if process.returncode is None: - try: - process.kill() - except ProcessLookupError: - pass - await process.communicate() +_reap_probe = reap_process async def finalize_mov_to_mp4( @@ -187,12 +184,7 @@ async def finalize_mov_to_mp4( probe_video_segment(temporary_path), PROBES_TIMEOUT_SECONDS * 10 ) except TimeoutError: - if process.returncode is None: - try: - process.kill() - except ProcessLookupError: - pass - await process.communicate() + await reap_process(process) raise RuntimeError("iOS recording finalization timed out") valid = ( process.returncode == 0 @@ -225,7 +217,7 @@ def __init__(self, device_id: str): self._session: IosRecordingSession | None = None self._output_dir: Path | None = None self._lock = asyncio.Lock() - self._max_duration_seconds = 900 + self._max_duration_seconds = DEFAULT_MAX_DURATION_SECONDS self._consecutive_failures = 0 self._conversion_semaphore = asyncio.Semaphore(MAX_CONCURRENT_CONVERSIONS) @@ -468,7 +460,7 @@ async def _watchdog(self, session: IosRecordingSession) -> None: async def start( self, output_dir: Path | None = None, - max_duration_seconds: int = 900, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, ) -> IosRecordingSession: async with self._lock: if self._session is not None and self._session.is_active: diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py index 61a6da82..dd3ba394 100644 --- a/artemis/drivers/ios/wda.py +++ b/artemis/drivers/ios/wda.py @@ -38,6 +38,7 @@ import urllib.error import urllib.request +from artemis.drivers.ios.hierarchy import pixel_element from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -376,20 +377,17 @@ def visit(entry: dict[str, Any]) -> None: elif value is not None and str(value): text = str(value) element_type = str(entry.get("type") or "") - element: dict[str, Any] = { - "text": text, - "resource_id": name if isinstance(name, str) else "", - "class": element_type.removeprefix(_ELEMENT_TYPE_PREFIX), - "bounds": f"[{left},{top}][{right},{bottom}]", - "parsed_bounds": { - "left": left, - "top": top, - "right": right, - "bottom": bottom, - }, - "hit_point": [(left + right) // 2, (top + bottom) // 2], - "visible": bool(entry.get("isVisible", True)), - } + element = pixel_element( + text=text, + resource_id=name if isinstance(name, str) else "", + class_name=element_type.removeprefix(_ELEMENT_TYPE_PREFIX), + left=left, + top=top, + right=right, + bottom=bottom, + ) + element["hit_point"] = [(left + right) // 2, (top + bottom) // 2] + element["visible"] = bool(entry.get("isVisible", True)) if value is not None and str(value) != text: element["value"] = str(value) elements.append(element) diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index 29884cf8..79379517 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -11,15 +11,16 @@ """iOS Simulator support using Xcode 27's native MCP and simctl tools.""" +from __future__ import annotations + import asyncio import base64 from io import BytesIO import json from pathlib import Path import plistlib -import re import sys -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from uuid import uuid4 from PIL import Image @@ -28,16 +29,25 @@ from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, XcodeBridge from artemis.drivers.ios.discovery import ( clear_ios_simulator_cache, + BOOTED_SIMULATOR_ID, parse_simctl_devices, + parse_xcode_version, plist_to_json, run_xcrun, ) from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy -from artemis.drivers.ios.recording import IosRecordingSession, IosScreenRecorder +from artemis.drivers.ios.recording import ( + DEFAULT_MAX_DURATION_SECONDS, + IosRecordingSession, + IosScreenRecorder, +) from third_party.mobile_use.controllers.types import ElementQuery from third_party.mobile_use.utils.logger import get_logger from third_party.mobile_use.utils.video import get_active_session, remove_active_session +if TYPE_CHECKING: + from artemis.drivers.ios.physical_recording import PhysicalIosRecorder + logger = get_logger(__name__) @@ -65,7 +75,7 @@ class XcodeSimulatorDriver(BaseDeviceDriver): def __init__( self, - device_id: str = "booted", + device_id: str = BOOTED_SIMULATOR_ID, width: int = 0, height: int = 0, *, @@ -81,7 +91,7 @@ def __init__( self._bridge = XcodeBridge() self._connect_lock = asyncio.Lock() self._operation_lock = asyncio.Lock() - self._recorder: IosScreenRecorder | None = None + self._recorder: IosScreenRecorder | PhysicalIosRecorder | None = None @property def device_id(self) -> str: @@ -101,9 +111,8 @@ async def _require_ios_host(self) -> None: """Validate the macOS/Xcode host requirements shared by all iOS drivers.""" if sys.platform != "darwin": raise RuntimeError("iOS support requires macOS and Xcode 27 or later.") - version = (await run_xcrun("xcodebuild", "-version")).decode() - match = re.search(r"Xcode\s+(\d+)", version) - if not match or int(match.group(1)) < 27: + version = parse_xcode_version(await run_xcrun("xcodebuild", "-version")) + if version is None or int(version.split(".")[0]) < 27: raise RuntimeError( "Native iOS interaction requires Xcode 27 or later. Set DEVELOPER_DIR to select it." ) @@ -122,7 +131,7 @@ async def _resolve_device(self) -> dict[str, Any]: await self._require_ios_host() devices = json.loads(await run_xcrun("simctl", "list", "devices", "--json")) available = parse_simctl_devices(devices) - if self._device_id == "booted": + if self._device_id == BOOTED_SIMULATOR_ID: candidates = [device for device in available if device.get("state") == "Booted"] if len(candidates) != 1: raise ValueError( @@ -265,7 +274,7 @@ async def _synthesize(self, command: str = "", activation: str | None = None) -> return await self._bridge.call("DeviceInteractionSynthesize", arguments) def _require_connected(self) -> None: - if not self._session_key or self._device_id == "booted": + if not self._session_key or self._device_id == BOOTED_SIMULATOR_ID: raise RuntimeError("Connect the iOS simulator driver before interacting.") async def _capture(self) -> ScreenData: @@ -299,12 +308,37 @@ async def get_screen_data(self, skip_settling: bool = False) -> ScreenData: async with self._operation_lock: return await self._capture() - def _point(self, x: int, y: int) -> str: + async def _capture_unchanged(self, action: str) -> ScreenData: + """Capture, refusing to continue when the screen geometry changed.""" + previous_size = self.screen_size + data = await self._capture() + if self.screen_size != previous_size: + raise ValueError( + f"The iOS screen changed orientation or size. Observe it again before {action}." + ) + return data + + def _scaled_point(self, x: int, y: int) -> tuple[float, float]: + """Convert screenshot pixels to the driver's logical point space.""" if self._scale is None: raise RuntimeError("Capture an iOS screen before coordinate interaction.") if not 0 <= x < self._width or not 0 <= y < self._height: raise ValueError("iOS input coordinates are outside the current screenshot.") - return f"{x / self._scale[0]:.4f} {y / self._scale[1]:.4f}" + return x / self._scale[0], y / self._scale[1] + + def _point(self, x: int, y: int) -> str: + px, py = self._scaled_point(x, y) + return f"{px:.4f} {py:.4f}" + + @staticmethod + def _direction_points(direction: str, w: int, h: int) -> tuple[int, int, int, int]: + points = { + "up": (w // 2, h * 3 // 4, w // 2, h // 4), + "down": (w // 2, h // 4, w // 2, h * 3 // 4), + "left": (w * 3 // 4, h // 2, w // 4, h // 2), + "right": (w // 4, h // 2, w * 3 // 4, h // 2), + } + return points[direction] @staticmethod def _activation_for(data: ScreenData, x: int, y: int) -> str | None: @@ -334,12 +368,7 @@ async def tap( if times < 1 or duration_ms < 0 or delay_ms < 0: raise ValueError("Tap count must be positive and durations nonnegative.") async with self._operation_lock: - previous_size = self.screen_size - data = await self._capture() - if self.screen_size != previous_size: - raise ValueError( - "The iOS screen changed orientation or size. Observe it again before tapping." - ) + data = await self._capture_unchanged("tapping") command = f"t {self._point(x, y)} {duration_ms / 1000:.3f}" for index in range(times): await self._synthesize(command, activation=self._activation_for(data, x, y)) @@ -356,12 +385,7 @@ async def swipe( if duration_ms <= 0: raise ValueError("Swipe duration must be positive.") async with self._operation_lock: - previous_size = self.screen_size - data = await self._capture() - if self.screen_size != previous_size: - raise ValueError( - "The iOS screen changed orientation or size. Observe it again before swiping." - ) + data = await self._capture_unchanged("swiping") await self._synthesize( f"t {self._point(start_x, start_y)} f {self._point(end_x, end_y)} {duration_ms / 1000:.3f}", activation=self._activation_for(data, start_x, start_y), @@ -377,15 +401,10 @@ async def swipe_direction( if duration_ms <= 0: raise ValueError("Swipe duration must be positive.") async with self._operation_lock: + # swipe_direction is rotation-tolerant by design: recapture and + # compute from the *current* size rather than refusing like tap. data = await self._capture() - w, h = self.screen_size - points = { - "up": (w // 2, h * 3 // 4, w // 2, h // 4), - "down": (w // 2, h // 4, w // 2, h * 3 // 4), - "left": (w * 3 // 4, h // 2, w // 4, h // 2), - "right": (w // 4, h // 2, w * 3 // 4, h // 2), - } - sx, sy, ex, ey = points[direction] + sx, sy, ex, ey = self._direction_points(direction, *self.screen_size) await self._synthesize( f"t {self._point(sx, sy)} f {self._point(ex, ey)} {duration_ms / 1000:.3f}", activation=self._activation_for(data, sx, sy), @@ -497,7 +516,11 @@ async def tap_element( async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: raise NotImplementedError("Android shell commands are unavailable on iOS Simulator.") - # --- Recording (native simctl segmented capture) --- + # --- Recording (native segmented capture) --- + + def _new_recorder(self) -> IosScreenRecorder | PhysicalIosRecorder: + """Recorder implementation for this driver (sim: simctl recordVideo).""" + return IosScreenRecorder(self._device_id) @property def recording_session(self) -> IosRecordingSession | None: @@ -507,13 +530,15 @@ def recording_session(self) -> IosRecordingSession | None: return self._recorder.session async def start_video_recording( - self, output_dir: Path | None = None, max_duration_seconds: int = 900 + self, + output_dir: Path | None = None, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, ) -> None: self._require_connected() if not self._device_id: - raise RuntimeError("iOS recording requires a pinned simulator UDID") + raise RuntimeError("iOS recording requires a pinned device UDID") if self._recorder is None: - self._recorder = IosScreenRecorder(self._device_id) + self._recorder = self._new_recorder() await self._recorder.start(output_dir, max_duration_seconds) async def seal_recording_segment(self, through_time: float | None = None) -> None: diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 4c0eb432..9ba99f55 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -21,6 +21,8 @@ from typing import Annotated from artemis.config import initialize_llm_config +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID +from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.sdk import Agent from artemis.sdk.builders import Builders from third_party.mobile_use.sdk.types.task import AgentProfile @@ -65,9 +67,9 @@ async def run_batch_tasks( config_builder.with_verification_level(verification_level) if explorer_pro_mode is not None: config_builder.with_explorer(pro_mode=explorer_pro_mode) - if platform.lower() == "ios": + if normalize_device_platform(platform, strict=False) == "ios": config_builder.for_ios_simulator( - device_id=device_serial or "booted", + device_id=device_serial or BOOTED_SIMULATOR_ID, workspace_path=ios_workspace, ) elif device_serial: @@ -204,13 +206,14 @@ def batch_command( ] = None, ) -> None: """Execute multiple automation tasks in sequence.""" - platform = platform.strip().lower() - if platform not in ("android", "ios"): + try: + platform = normalize_device_platform(platform) + except ValueError as exc: typer.secho( f"Error: --platform must be 'android' or 'ios' (got '{platform}').", fg=typer.colors.RED, ) - raise typer.Exit(1) + raise typer.Exit(1) from exc if ios_workspace is not None and platform != "ios": typer.secho("Error: --ios-workspace requires --platform ios.", fg=typer.colors.RED) raise typer.Exit(1) diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 6c642ece..b628178b 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -23,6 +23,7 @@ from artemis.config import checker_overrides_for_level, settings from artemis.context import DevicePlatform from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, xcode_approval_guidance +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.utils.startup_progress import publish_startup_progress from third_party.mobile_use.main import ( GoalArgument, @@ -147,7 +148,7 @@ async def execute_task( target_serial = device_serial if platform == DevicePlatform.IOS: - target_serial = target_serial or "booted" + target_serial = target_serial or BOOTED_SIMULATOR_ID else: target_serial = ( target_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") diff --git a/artemis/mcp/actuators/ios.py b/artemis/mcp/actuators/ios.py index 57b0eef6..a650b525 100644 --- a/artemis/mcp/actuators/ios.py +++ b/artemis/mcp/actuators/ios.py @@ -46,7 +46,7 @@ async def input_text( return ActionResult.failure("input_text", error) success = await self.controller.type_text(text, clear_existing=clear_exist) if not success: - return ActionResult.failure("input_text", "Failed to type text on the iOS simulator.") + return ActionResult.failure("input_text", "Failed to type text on the iOS device.") return ActionResult.success("input_text", f"Typed '{text}'.") async def press_key(self, key: str) -> ActionResult: diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 3ee15c44..15749fd0 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -44,6 +44,8 @@ from artemis.context import ArtemisContext, DeviceContext, DevicePlatform from artemis.controllers.unified_controller import UnifiedMobileController from artemis.platform import platform +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID +from artemis.runtime.adb_endpoint import normalize_device_platform from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries @@ -129,7 +131,7 @@ def _get_controller(device_serial: str | None = None, target_platform: str | Non target_serial = ( device_serial or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL") ) - if (target_platform or "").lower() == "ios": + if normalize_device_platform(target_platform, strict=False) == "ios": cache_key = f"ios:{target_serial or 'booted'}" if cache_key in _CONTROLLERS: return _CONTROLLERS[cache_key] @@ -139,7 +141,7 @@ def _get_controller(device_serial: str | None = None, target_platform: str | Non device=DeviceContext( host_platform=platform.os_type.name, mobile_platform=DevicePlatform.IOS, - device_id=target_serial or "booted", + device_id=target_serial or BOOTED_SIMULATOR_ID, device_width=1206, device_height=2622, ), diff --git a/artemis/runtime/adb_endpoint.py b/artemis/runtime/adb_endpoint.py index 7c136f0b..029018b2 100644 --- a/artemis/runtime/adb_endpoint.py +++ b/artemis/runtime/adb_endpoint.py @@ -37,11 +37,65 @@ _LOCAL_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"}) _SAFE_HOST_PATTERN = re.compile(r"^[A-Za-z0-9._:\-\[\]]+$") +# User-facing platform vocabulary shared by CLI, console, daemon, and MCP. +SUPPORTED_PLATFORMS: tuple[str, ...] = ("android", "ios") +DEFAULT_PLATFORM = "android" + + +def normalize_device_platform( + value: object, + *, + default: str = DEFAULT_PLATFORM, + strict: bool = True, +) -> str: + """Canonicalize a platform token to ``"android"`` or ``"ios"``. + + Strips whitespace and case-folds. ``strict=True`` (entry points) raises + ``ValueError`` on anything else; ``strict=False`` (fail-open internals) + falls back to *default*. Surfaces adapt the ``ValueError`` to their own + error type (HTTP 400, Typer error, blocked verdict, ...). + """ + text = str(value).strip().lower() if value is not None else "" + if not text: + return default + if text in SUPPORTED_PLATFORMS: + return text + if strict: + raise ValueError(f"Unsupported platform '{value}'. Expected 'android' or 'ios'.") + return default + class InvalidAdbEndpoint(ValueError): """Raised when an ADB server endpoint is malformed.""" +def device_pool_for(platform: object): + """The device pool for a platform token (lazy imports avoid a cycle). + + Both pools expose the same admission surface (``validate_explicit_serial``, + ``validate_explicit_serial_async``, ``select_device_async``, + ``list_devices_async``). + """ + if normalize_device_platform(platform, strict=False) == "ios": + from artemis.runtime.ios_device_pool import ios_device_pool + + return ios_device_pool + from artemis.runtime.device_pool import device_pool + + return device_pool + + +def target_for_platform( + platform: object, + serial: str | None = None, + endpoint: AdbEndpoint | None = None, +) -> AdbTarget | IosTarget: + """Build the execution target for a platform; ``lock_scope`` comes with it.""" + if normalize_device_platform(platform, strict=False) == "ios": + return IosTarget(serial=serial) + return AdbTarget(endpoint=endpoint or current_adb_endpoint(), serial=serial) + + @dataclass(frozen=True, slots=True) class AdbEndpoint: """Network address of one ADB server.""" diff --git a/artemis/runtime/ios_device_pool.py b/artemis/runtime/ios_device_pool.py index a71cabbc..2077bf2a 100644 --- a/artemis/runtime/ios_device_pool.py +++ b/artemis/runtime/ios_device_pool.py @@ -30,6 +30,7 @@ list_core_devices_sync, list_ios_simulators, list_ios_simulators_sync, + physical_ios_ready, ) from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.runtime.device_lock import DeviceExecutionLock @@ -46,20 +47,65 @@ PHYSICAL_ACCEPTABLE_STATES = frozenset({"device"}) +def _ios_lock_owners() -> dict: + """Active lock owners scoped to iOS, keyed by normalized device id. + + Owner map keys are only scope-prefixed for multi-owner collisions, so + match by owner payload: an iOS lock always carries lock_scope="ios", + and an Android lock on the same text must never mark a device busy. + """ + return { + DeviceExecutionLock._normalize_device_id(o.device_id): o + for o in DeviceExecutionLock.get_active_owners().values() + if o and getattr(o, "lock_scope", None) == IOS_LOCK_SCOPE + } + + +def _owner_fields(owner) -> dict: + return { + "is_busy": owner is not None, + "active_pid": owner.pid if owner else None, + "active_task_desc": owner.description if owner else None, + "active_session_id": owner.session_id if owner else None, + "acquired_at": owner.acquired_at if owner else None, + } + + +def _match_status( + devices: list[DeviceStatus], requested_serial: str +) -> DeviceStatus | None: + """Find the status matching a requested UDID or device name.""" + needle = str(requested_serial).lower() + return next( + ( + d + for d in devices + if d.serial.lower() == needle or (d.model or "") == requested_serial + ), + None, + ) + + +def _state_rejection(match: DeviceStatus, requested_serial: str) -> str | None: + """The shared explicit-target state check for both validator variants.""" + acceptable = ( + ACCEPTABLE_STATES if match.is_emulator else PHYSICAL_ACCEPTABLE_STATES + ) + if match.state not in acceptable: + kind = "simulator" if match.is_emulator else "physical device" + return ( + f"iOS {kind} '{requested_serial}' is in state '{match.state}' " + "and cannot be used." + ) + return None + + class IosDevicePool: """Discovers iOS simulators and physical devices and reports lock state.""" @staticmethod def _build_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: - active_owners = DeviceExecutionLock.get_active_owners() - # Owner map keys are only scope-prefixed for multi-owner collisions, so - # match by owner payload: an iOS lock always carries lock_scope="ios", - # and an Android lock on the same text must never mark a sim busy. - ios_owners = { - DeviceExecutionLock._normalize_device_id(o.device_id): o - for o in active_owners.values() - if o and getattr(o, "lock_scope", None) == IOS_LOCK_SCOPE - } + ios_owners = _ios_lock_owners() devices: list[DeviceStatus] = [] for device in raw_devices: udid = str(device.get("udid") or "") @@ -79,33 +125,29 @@ def _build_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: "-", " " ), is_emulator=True, - is_busy=owner is not None, - active_pid=owner.pid if owner else None, - active_task_desc=owner.description if owner else None, - active_session_id=owner.session_id if owner else None, - acquired_at=owner.acquired_at if owner else None, platform="ios", + **_owner_fields(owner), ) ) return devices @staticmethod def _build_physical_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: - """CoreDevice entries -> statuses; reachable devices read as "device".""" - active_owners = DeviceExecutionLock.get_active_owners() - ios_owners = { - DeviceExecutionLock._normalize_device_id(o.device_id): o - for o in active_owners.values() - if o and getattr(o, "lock_scope", None) == IOS_LOCK_SCOPE - } + """CoreDevice entries -> statuses; usable devices read as "device".""" + ios_owners = _ios_lock_owners() devices: list[DeviceStatus] = [] for device in raw_devices: if not is_physical_ios(device): continue udid = str(device.get("udid") or "") - paired = device.get("pairing_state") == "paired" - connected = device.get("connection_state") == "connected" - state = "device" if connected and paired else "unpaired" if not paired else "offline" + ready = physical_ios_ready(device) + state = ( + "device" + if ready + else "unpaired" + if device.get("pairing_state") != "paired" + else "offline" + ) owner = ios_owners.get(DeviceExecutionLock._normalize_device_id(udid)) devices.append( DeviceStatus( @@ -114,12 +156,8 @@ def _build_physical_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: model=device.get("name"), product=f"iOS {device.get('os_version') or '?'} physical", is_emulator=False, - is_busy=owner is not None, - active_pid=owner.pid if owner else None, - active_task_desc=owner.description if owner else None, - active_session_id=owner.session_id if owner else None, - acquired_at=owner.acquired_at if owner else None, platform="ios", + **_owner_fields(owner), ) ) return devices @@ -154,15 +192,7 @@ async def validate_explicit_serial_async(self, requested_serial: str) -> str | N devices = self._build_statuses(raw or []) + self._build_physical_statuses( core or [] ) - needle = str(requested_serial).lower() - match = next( - ( - d - for d in devices - if d.serial.lower() == needle or (d.model or "") == requested_serial - ), - None, - ) + match = _match_status(devices, requested_serial) if match is None: # A failed enumeration cannot prove the serial is absent — # defer to the driver's own resolution rather than reject. @@ -172,56 +202,31 @@ async def validate_explicit_serial_async(self, requested_serial: str) -> str | N f"iOS device '{requested_serial}' is not available. " f"Known devices: {sorted(d.serial for d in devices)}." ) - acceptable = ( - ACCEPTABLE_STATES if match.is_emulator else PHYSICAL_ACCEPTABLE_STATES - ) - if match.state not in acceptable: - kind = "simulator" if match.is_emulator else "physical device" - return ( - f"iOS {kind} '{requested_serial}' is in state '{match.state}' " - "and cannot be used." - ) - return None + return _state_rejection(match, requested_serial) def validate_explicit_serial(self, requested_serial: str) -> str | None: - """Synchronous validator for non-async admission paths (MCP tools).""" - devices = list_ios_simulators_sync() - statuses = {d["udid"].lower(): d.get("state") for d in devices or []} - needle = str(requested_serial).lower() - state = statuses.get(needle) - if state is not None: - if state != "Booted" and state != "Shutdown": - return ( - f"iOS simulator '{requested_serial}' is in state '{state}' " - "and cannot be used." - ) - return None + """Synchronous validator for non-async admission paths (MCP tools). + + Shares the async validator's matching and state rules so both + admission paths accept and reject the same targets. + """ + raw = list_ios_simulators_sync() core = list_core_devices_sync() - physical = [d for d in core or [] if is_physical_ios(d)] - match = next( - ( - d - for d in physical - if str(d.get("udid") or "").lower() == needle - or d.get("name") == requested_serial - ), - None, + if raw is None and core is None: + return None + devices = self._build_statuses(raw or []) + self._build_physical_statuses( + core or [] ) + match = _match_status(devices, requested_serial) if match is None: # An enumeration that could not answer must not reject the serial. - if devices is None or core is None: - return None - if not statuses and not physical: + if raw is None or core is None or not devices: return None return ( f"iOS device '{requested_serial}' is not available. " - f"Known devices: {sorted([*statuses, *[d['udid'] for d in physical]])}." + f"Known devices: {sorted(d.serial for d in devices)}." ) - if match.get("pairing_state") != "paired": - return f"iOS device '{requested_serial}' is not paired with this Mac." - if match.get("connection_state") != "connected": - return f"iOS device '{requested_serial}' is offline; attach it and retry." - return None + return _state_rejection(match, requested_serial) async def select_device_async(self, preferred_serial: str | None = None) -> str | None: """Pick a simulator UDID for task execution. diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index ae09ae5c..a3b43264 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -148,6 +148,7 @@ async def _init_internal( if self._initialized: return True from artemis.drivers.factory import ios_driver_class + from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID publish_startup_progress( "device_check", "Checking the iOS device", session_id=self._session_id @@ -155,15 +156,15 @@ async def _init_internal( # The picker runs simctl/devicectl subprocesses — keep them off the # event loop so init timeouts and progress stays responsive. driver_class = await asyncio.to_thread( - ios_driver_class, self._config.device_id or "booted" + ios_driver_class, self._config.device_id or BOOTED_SIMULATOR_ID ) driver = driver_class( - device_id=self._config.device_id or "booted", + device_id=self._config.device_id or BOOTED_SIMULATOR_ID, workspace_path=getattr(self._config, "ios_workspace_path", None), ) self._ios_driver = driver try: - # Resolve the simulator without booting it or opening a native UI + # Resolve the device without booting it or opening a native UI # session. Mutating setup waits for run_task's execution lease. await driver.resolve_device() except (OSError, ValueError, RuntimeError, TimeoutError, asyncio.CancelledError): diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 97008748..d6d641aa 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -24,6 +24,7 @@ load_agent_config, settings, ) +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.context import DevicePlatform from artemis.sdk.types.agent import AgentConfig, ServerConfig from artemis.utils.video import detect_video_tools_enabled @@ -78,14 +79,17 @@ def for_device_serial(self, device_serial: str) -> "AgentConfigBuilder": """Target a specific Android device by its ADB serial number.""" return self.for_device(DevicePlatform.ANDROID, device_serial) - def for_ios_simulator( - self, device_id: str = "booted", *, workspace_path: str | Path | None = None + def for_ios_device( + self, device_id: str = BOOTED_SIMULATOR_ID, *, workspace_path: str | Path | None = None ) -> "AgentConfigBuilder": """Target an iOS device UDID (simulator or paired physical), or the booted simulator.""" if workspace_path is not None: self.with_ios_workspace(workspace_path) return self.for_device(DevicePlatform.IOS, device_id) + # Backward-compatible alias from when iOS support was simulator-only. + for_ios_simulator = for_ios_device + def with_ios_workspace(self, workspace_path: str | Path | None) -> "AgentConfigBuilder": """Set an existing Xcode project/workspace for iOS first-run approval.""" self._ios_workspace_path = ( diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 01c5a4b9..9627cfd7 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -603,7 +603,7 @@ def detect_video_tools_enabled(platform: str = "android") -> bool: iOS uses native ``xcrun simctl io recordVideo`` capture plus the bundled FFmpeg for post-processing; Android requires scrcpy plus FFmpeg. """ - if platform == "ios": + if str(platform).strip().lower() == "ios": return ( sys.platform == "darwin" and shutil.which("xcrun") is not None and is_ffmpeg_installed() ) diff --git a/docs/ios.md b/docs/ios.md index d987e7f9..50aa224f 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -138,6 +138,10 @@ endpoint on the CoreDevice tunnel address and `127.0.0.1:8100` (for `xcodebuild test-without-building` (the canonical WDA session — a bare runner app launch does not start the HTTP server) +Simulator bridging note: `xcrun mcpbridge` is spawned with a minimal +environment; `DEVELOPER_DIR` and `MCP_XCODE_PID` are forwarded when set so a +specific Xcode toolchain can be pinned. + Differences from simulators: - The device must already be paired, trusted, and connected; Artemis never @@ -188,7 +192,7 @@ Configure iOS through the embedded SDK's builder: from artemis.sdk import Agent from artemis.sdk.builders import AgentConfigBuilder -config = AgentConfigBuilder().for_ios_simulator("").build() +config = AgentConfigBuilder().for_ios_device("").build() agent = Agent(config=config) ``` @@ -198,7 +202,7 @@ existing project or workspace through the builder: ```python config = ( AgentConfigBuilder() - .for_ios_simulator( + .for_ios_device( "", workspace_path="/absolute/path/MyApp.xcodeproj", ) diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index bcdc11d5..b986b031 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -42,6 +42,8 @@ load_dotenv(os.path.join(PROJECT_ROOT, ".env")) from artemis.runtime import trace_store +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID +from artemis.runtime.adb_endpoint import normalize_device_platform from mcp_server.notifiers import notify from mcp_server.utils import device_utils @@ -156,7 +158,7 @@ async def run_task( print("--------------------------------------------------") agent = None - is_ios = platform.lower() == "ios" + is_ios = normalize_device_platform(platform, strict=False) == "ios" adb_path = None if is_ios else device_utils.resolve_adb_path() target_serial = device_serial @@ -176,7 +178,7 @@ async def run_task( if is_ios: # The iOS driver resolves "booted" against `simctl` at connect # time; an explicit UDID was already validated at admission. - target_serial = device_serial or "booted" + target_serial = device_serial or BOOTED_SIMULATOR_ID print(f"✅ Targeting iOS simulator: '{target_serial}'.") elif device_serial: target_serial = device_serial @@ -231,7 +233,7 @@ async def run_task( if is_ios: config_builder.for_ios_simulator( - device_id=target_serial or "booted", + device_id=target_serial or BOOTED_SIMULATOR_ID, workspace_path=ios_workspace, ) elif target_serial: diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index 88f9e978..dae86210 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -44,6 +44,7 @@ ) from artemis.core.diagnostics.schema import ProbeResult, ProbeStatus, SystemReadinessReport from artemis.runtime import DeviceExecutionLock, trace_store +from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.runtime.helper_manager import helper_manager from artemis.utils.credentials_validator import validate_api_key from third_party.mobile_use.utils.logger import get_logger @@ -1251,8 +1252,9 @@ async def mobile_diagnose( hardware); ADB-only extras (AVD launch, accessibility helper) are skipped. """ fixes_applied: list[dict[str, Any]] = [] - platform_name = (platform or "android").strip().lower() - if platform_name not in ("android", "ios"): + try: + platform_name = normalize_device_platform(platform) + except ValueError: return { "verdict": "blocked", "error": f"Invalid platform '{platform}'. Expected 'android' or 'ios'.", diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index cfd6cbee..8d045e54 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -29,13 +29,14 @@ from artemis.config import ExplorerVersion, checker_overrides_for_level from artemis.config.runtime import read_ipc_port from artemis.runtime import ( + IOS_LOCK_SCOPE, DeviceExecutionLock, - device_pool, + IosTarget, ensure_daemon_running, - ios_device_pool, submit_task_to_daemon, trace_store, ) +from artemis.runtime.adb_endpoint import device_pool_for, normalize_device_platform # Seconds the spawned runner gets to finish its imports and open its log files # before the spawn is declared dead. Normal startup creates stdout.log within a @@ -155,12 +156,13 @@ def _validate_device_serial(device_serial: str, platform: str = "android") -> di try: # The shared validators fail open on an indeterminate/empty enumeration: # the task proceeds and fails downstream with a clear no-device error. - if platform == "ios": - detail = ios_device_pool.validate_explicit_serial(device_serial) - inspect_hint = "Run `xcrun simctl list devices` (simulators) or `xcrun devicectl list devices` (physical devices)" - else: - detail = device_pool.validate_explicit_serial(device_serial) - inspect_hint = "Run `adb devices -l` to inspect attached hardware" + detail = device_pool_for(platform).validate_explicit_serial(device_serial) + inspect_hint = ( + "Run `xcrun simctl list devices` (simulators) or " + "`xcrun devicectl list devices` (physical devices)" + if platform == "ios" + else "Run `adb devices -l` to inspect attached hardware" + ) except Exception: return None if detail is None: @@ -296,10 +298,8 @@ def mobile_run_task( if model.lower() not in ("flash", "pro"): raise ValueError(f"Invalid model '{model}'. Must be either 'Flash' or 'Pro'.") canonical_model = "Flash" if model.lower() == "flash" else "Pro" - # 0a. Validate and normalize platform: 'android' (ADB) or 'ios' (Xcode Simulator). - platform = (platform or "android").strip().lower() - if platform not in ("android", "ios"): - raise ValueError(f"Invalid platform '{platform}'. Must be 'android' or 'ios'.") + # 0a. Validate and normalize platform: 'android' (ADB) or 'ios' (Xcode). + platform = normalize_device_platform(platform) if platform == "ios" and locked_app_package: return { "trace_id": None, @@ -472,7 +472,7 @@ def mobile_run_task( if platform == "ios": # iOS workers queue under the ios__ scope so they never collide # with an Android device sharing the same textual identifier. - reserve_kwargs["lock_scope"] = "ios" + reserve_kwargs["lock_scope"] = IOS_LOCK_SCOPE queue_ticket = DeviceExecutionLock.reserve(**reserve_kwargs) # 5. Spawn the background task runner as an independent subprocess @@ -511,14 +511,14 @@ def mobile_run_task( env["ARTEMIS_SESSION_ID"] = trace_id env["ARTEMIS_TASK_INGRESS"] = "mcp" if platform == "ios": - # An iOS worker never touches ADB: a stale serial must not leak in. - env.pop("ADB_DEVICE_SERIAL", None) - if device_serial: - if platform == "ios": - env["ARTEMIS_DEVICE_ID"] = device_serial - else: - env["ADB_DEVICE_SERIAL"] = device_serial + # iOS workers run under the ios lock scope; the target scrubs + # stale ADB serials so an Android id can never leak in. + IosTarget(serial=device_serial).apply_to_environment(env) + if device_serial: env["ARTEMIS_DEVICE_ID"] = device_serial + elif device_serial: + env["ADB_DEVICE_SERIAL"] = device_serial + env["ARTEMIS_DEVICE_ID"] = device_serial env[DeviceExecutionLock.QUEUE_TICKET_ENV] = queue_ticket try: ipc_port = read_ipc_port() @@ -541,7 +541,7 @@ def mobile_run_task( "ingress": "mcp", } if platform == "ios": - transfer_kwargs["lock_scope"] = "ios" + transfer_kwargs["lock_scope"] = IOS_LOCK_SCOPE if device_serial: transfer_kwargs["device_id"] = device_serial diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py index fc2a1bad..f8d67af4 100644 --- a/tests/unit/drivers/test_ios_physical.py +++ b/tests/unit/drivers/test_ios_physical.py @@ -507,13 +507,11 @@ async def test_recorder_polls_frames_and_assembles_mp4(tmp_path, monkeypatch): """Frames become one timestamped segment through the ffconcat demuxer.""" ffmpeg_calls = [] - async def fake_xcrun(*arguments, timeout=30.0): + async def fake_screenshot(device_id, destination, timeout=30.0): # Pace like real devicectl round-trips; an instant fake lets the poll # loop flood tmp_path with frames and stalls cleanup for minutes. await asyncio.sleep(0.02) - destination = Path(arguments[arguments.index("--destination") + 1]) _png(destination) - return b"" async def fake_ffmpeg(arguments): ffmpeg_calls.append(arguments) @@ -529,7 +527,7 @@ async def fake_manifest(output_dir, paths, offsets): manifest.write_text("{}") return manifest - monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_recording, "devicectl_screenshot", fake_screenshot) monkeypatch.setattr(physical_recording, "_run_ffmpeg", fake_ffmpeg) monkeypatch.setattr(physical_recording, "probe_video_segment", fake_probe) monkeypatch.setattr(physical_recording, "write_recording_manifest", fake_manifest) @@ -542,13 +540,11 @@ async def fake_manifest(output_dir, paths, offsets): @pytest.mark.asyncio async def test_recorder_partitions_frames_at_the_seam(tmp_path, monkeypatch): - async def fake_xcrun(*arguments, timeout=30.0): + async def fake_screenshot(device_id, destination, timeout=30.0): await asyncio.sleep(0.02) - destination = Path(arguments[arguments.index("--destination") + 1]) _png(destination) - return b"" - monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_recording, "devicectl_screenshot", fake_screenshot) recorder = PhysicalIosRecorder(IPHONE_UDID) session = await recorder.start(tmp_path) anchor = session.anchor_monotonic @@ -566,11 +562,11 @@ async def fake_xcrun(*arguments, timeout=30.0): @pytest.mark.asyncio async def test_recorder_fails_closed_when_devicectl_never_delivers(tmp_path, monkeypatch): - async def fake_xcrun(*arguments, timeout=30.0): + async def fake_screenshot(device_id, destination, timeout=30.0): await asyncio.sleep(0.01) raise RuntimeError("device disconnected") - monkeypatch.setattr(physical_recording, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_recording, "devicectl_screenshot", fake_screenshot) monkeypatch.setattr(physical_recording, "MAX_CONSECUTIVE_FAILURES", 2) recorder = PhysicalIosRecorder(IPHONE_UDID) with pytest.raises(RuntimeError): diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index dd3e7673..f7923bd5 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -610,22 +610,25 @@ async def test_stream_target_none_without_devices(monkeypatch): def test_mcp_validate_device_serial_ios_uses_ios_pool(monkeypatch): + from artemis.runtime import device_pool as adb_pool + from artemis.runtime.ios_device_pool import ios_device_pool from mcp_server.tools import task_runner ios_validate = MagicMock(return_value=None) adb_validate = MagicMock(return_value="should not be called") - monkeypatch.setattr(task_runner.ios_device_pool, "validate_explicit_serial", ios_validate) - monkeypatch.setattr(task_runner.device_pool, "validate_explicit_serial", adb_validate) + monkeypatch.setattr(ios_device_pool, "validate_explicit_serial", ios_validate) + monkeypatch.setattr(adb_pool, "validate_explicit_serial", adb_validate) assert task_runner._validate_device_serial("AAAA-1111", "ios") is None ios_validate.assert_called_once_with("AAAA-1111") adb_validate.assert_not_called() def test_mcp_validate_device_serial_ios_rejection(monkeypatch): + from artemis.runtime.ios_device_pool import ios_device_pool from mcp_server.tools import task_runner monkeypatch.setattr( - task_runner.ios_device_pool, + ios_device_pool, "validate_explicit_serial", lambda serial: f"iOS simulator '{serial}' is not available.", ) From 34bba22369e2bc394ed54361a55e602feff6ad02 Mon Sep 17 00:00:00 2001 From: James Lee Date: Wed, 7 Oct 2026 06:16:36 +0900 Subject: [PATCH 29/32] fix: harden native iOS execution and remote SDK compatibility Lease and close one-shot iOS observations; drain cancelled native and WDA requests; reap incomplete simulator recording starts and bound recovery loops. Preserve modern CoreDevice schema, unambiguous targets, per-platform queue identity, and physical-device selection without changing Android driver code. Add remote SDK iOS platform/workspace forwarding with capabilities preflight to reject legacy hosts before submission. Keep Android payloads unchanged. Add regressions and resolve protected-core typing diagnostics without changing quality thresholds. Verification: 2900 deterministic tests passed, 11 skipped; protected-core pyright reports 0 errors; Ruff and quality ratchets pass. Live simulator acceptance is blocked by Xcode agent consent; physical/Android hardware and the compatible frontend Node runtime are unavailable. No production-readiness claim. --- apps/admin_console/routers/tasks.py | 24 +- .../services/task_queue_service.py | 29 +- .../src/app/pages/home/home.component.html | 24 +- .../src/app/services/system.service.spec.ts | 192 +++++ .../src/app/services/system.service.ts | 35 +- artemis/agents/diagnoser/diagnoser.py | 4 +- artemis/core/diagnostics/device_smoke.py | 88 +- artemis/core/diagnostics/probes/ios_probe.py | 8 +- artemis/drivers/factory.py | 9 +- artemis/drivers/ios/bridge.py | 2 +- artemis/drivers/ios/discovery.py | 105 ++- artemis/drivers/ios/physical_driver.py | 218 +++-- artemis/drivers/ios/physical_recording.py | 96 ++- artemis/drivers/ios/recording.py | 126 ++- artemis/drivers/ios/wda.py | 119 ++- artemis/drivers/ios/xcode_driver.py | 17 +- artemis/interfaces/cli/commands/batch.py | 5 +- artemis/interfaces/cli/commands/run.py | 4 +- artemis/mcp/action_specs.py | 4 +- artemis/mcp/adb_server.py | 24 +- artemis/runtime/ios_device_pool.py | 88 +- artemis/runtime/ios_observation.py | 94 +++ artemis/sdk/agent.py | 4 +- artemis/utils/video.py | 37 +- artemis/utils/visualization.py | 6 +- docs/ios.md | 48 +- mcp_server/background/task_runner.py | 11 +- mcp_server/tools/device_state.py | 33 +- mcp_server/tools/diagnose.py | 115 ++- .../src/artemis_client/client.py | 38 + .../src/artemis_client/models.py | 33 + packages/artemis-client/tests/test_client.py | 184 +++++ .../admin_console/test_task_queue_service.py | 80 ++ tests/unit/core/test_device_smoke.py | 124 ++- tests/unit/drivers/test_ios_driver.py | 30 + tests/unit/drivers/test_ios_physical.py | 774 +++++++++++++++++- tests/unit/drivers/test_ios_recording.py | 132 +++ tests/unit/mcp/test_diagnose_tool.py | 145 ++++ tests/unit/mcp/test_mcp_tools.py | 58 ++ tests/unit/runtime/test_device_lock.py | 4 +- .../runtime/test_ios_observation_lifecycle.py | 266 ++++++ .../sdk/test_agent_config_builder_tuning.py | 68 +- tests/unit/test_ios_integration.py | 16 +- tests/unit/test_ios_parity.py | 163 ++++ tests/unit/utils/test_video_manifest.py | 94 +++ 45 files changed, 3324 insertions(+), 454 deletions(-) create mode 100644 artemis/runtime/ios_observation.py create mode 100644 tests/unit/runtime/test_ios_observation_lifecycle.py diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index df2a14d7..d655581e 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -71,6 +71,26 @@ async def get_task_catalog(): } +@router.get("/api/v1/capabilities") +async def get_capabilities(): + """Static wire-feature advertisement — no device or service probing. + + ``platform.ios`` means the API accepts iOS submissions; it does not + imply Xcode approval or attached hardware on this host. + """ + return { + "api_version": "1", + "features": [ + "tasks.submit", + "tasks.get", + "tasks.stop", + "devices.list", + "system.readiness", + "platform.ios", + ], + } + + @router.post("/api/run") async def run_task(request: RunRequest): incoming_goals = [] @@ -305,7 +325,9 @@ async def get_status(): "pid": owner.pid, "ingress": owner.ingress, "acquired_at": owner.acquired_at, - "platform": "ios" if getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE else "android", + "platform": "ios" + if getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE + else "android", } for owner in active_owners.values() ] diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index 3f5b7845..93e7613b 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -225,15 +225,14 @@ def _enforce_sync() -> None: @staticmethod def _task_target(task_item: dict[str, Any]) -> AdbTarget | IosTarget: serial = str(task_item.get("device_serial") or "") or None + if normalize_device_platform(task_item.get("platform"), strict=False) == "ios": + # iOS targets never read a stale/malformed Android endpoint blob. + return IosTarget(serial=serial) endpoint_data = task_item.get("adb_endpoint") endpoint = ( - AdbEndpoint.from_mapping(endpoint_data) - if isinstance(endpoint_data, dict) - else None - ) - return target_for_platform( - task_item.get("platform"), serial=serial, endpoint=endpoint + AdbEndpoint.from_mapping(endpoint_data) if isinstance(endpoint_data, dict) else None ) + return target_for_platform(task_item.get("platform"), serial=serial, endpoint=endpoint) @classmethod def _broadcast_event(cls, event_type: str, data: Any): @@ -927,6 +926,8 @@ def _find_duplicate_submission( device_serial: str | None, endpoint: AdbEndpoint, now: float, + *, + platform: str = "android", ) -> dict[str, Any] | None: """Return the short-circuit response for a duplicate submission, if any.""" # 1. Deduplication by session_id: if session_id is already running or queued, do not re-enqueue @@ -965,8 +966,14 @@ def _find_duplicate_submission( if isinstance(item, dict) and item.get("status") == "pending" and item.get("goal") == first_goal + and normalize_device_platform(item.get("platform"), strict=False) == platform and (not device_serial or item.get("device_serial") == device_serial) - and item.get("adb_endpoint", {}).get("identity") == endpoint.identity + # iOS items carry no ADB endpoint snapshot; matching the + # platform + serial is the whole identity there. + and ( + platform == "ios" + or (item.get("adb_endpoint") or {}).get("identity") == endpoint.identity + ) and (now - float(item.get("created_at", 0))) < 1.0 ), None, @@ -991,9 +998,9 @@ async def _reject_unavailable_device( # the task can proceed and fail downstream with a clear no-device error. if device_serial: try: - rejection = await device_pool_for( - platform - ).validate_explicit_serial_async(device_serial) + rejection = await device_pool_for(platform).validate_explicit_serial_async( + device_serial + ) except Exception: rejection = None if rejection: @@ -1101,7 +1108,7 @@ async def enqueue_tasks( endpoint = current_adb_endpoint() duplicate_response = cls._find_duplicate_submission( - goals, session_id, device_serial, endpoint, now + goals, session_id, device_serial, endpoint, now, platform=platform ) if duplicate_response is not None: return duplicate_response diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index bf2c8ab3..571e0c82 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -674,7 +674,7 @@

AI Model Setup

Device & Emulator Connection

-

Android device or emulator via ADB, or an iOS Simulator (Xcode 27+)

+

Android device or emulator via ADB, or an iOS Simulator / paired iPhone or iPad (Xcode 27+)

@if (deviceGuidePlatform() === 'android') { @@ -710,7 +710,7 @@

Device & Emulator Connection

- +

(click)="deviceGuidePlatform.set('ios')" > phone_iphone - iOS Simulator + iOS Devices @if (iosDevices().length > 0) { {{ iosDevices().length }} } @@ -1708,7 +1708,7 @@

No Android Virtual Devices (AVD) Detected

} @else { - +
@@ -1725,7 +1725,7 @@

No Android Virtual Devices (AVD) Detected

{{ sim.model || sim.serial }}
- iOS Simulator + {{ sim.is_emulator ? 'iOS Simulator' : 'Physical iOS Device' }} · {{ sim.serial }} @if (sim.product) { @@ -1736,21 +1736,21 @@

No Android Virtual Devices (AVD) Detected

- {{ sim.state === 'device' ? 'Booted' : 'Shutdown — boots on run' }} + {{ sim.state === 'device' ? (sim.is_emulator ? 'Booted' : 'Connected') : (sim.is_emulator ? 'Shutdown — boots on run' : sim.state) }}
} - + @if (iosDevices().length > 0) {
phone_iphone
- iOS Simulators + iOS Devices - {{ iosDevices().length }} available — select one to run iOS tasks + {{ iosDevices().length }} available — select a simulator or paired iPhone/iPad to run iOS tasks
@@ -1766,7 +1766,7 @@

No Android Virtual Devices (AVD) Detected

{{ dev.model?.includes('iPad') ? 'tablet_mac' : 'phone_iphone' }} {{ dev.model || dev.serial }} - + @if (dev.serial === selectedDeviceSerial()) { ✓ Active } @@ -1778,9 +1778,9 @@

No Android Virtual Devices (AVD) Detected

phone_iphone
-
No iOS Simulators Available
+
No iOS Devices Available
- {{ iosProbe()?.description || 'iOS Simulator automation requires macOS with Xcode 27+ and at least one available simulator.' }} + {{ iosProbe()?.description || 'iOS automation requires macOS with Xcode 27+ and an available simulator or a paired iPhone/iPad. Physical devices also need WebDriverAgent on the device.' }}
diff --git a/apps/showcase_ui/src/app/services/system.service.spec.ts b/apps/showcase_ui/src/app/services/system.service.spec.ts index 40748721..f75ac47f 100644 --- a/apps/showcase_ui/src/app/services/system.service.spec.ts +++ b/apps/showcase_ui/src/app/services/system.service.spec.ts @@ -278,6 +278,198 @@ describe('SystemService readiness polling', () => { expect(service.selectedIosDevice()).toBeNull(); }); + const iosProbeWith = (metadata: any) => ({ + id: 'ios_simulators', + category: 'device', + title: 'iOS Devices', + status: 'pass', + is_blocker: false, + summary: 'iOS ready', + description: '', + metadata, + actions: [] + }); + + const envProbe = (id: string, status = 'pass') => ({ + id, + category: 'runtime', + title: id, + status, + is_blocker: true, + summary: status, + description: '', + metadata: {}, + actions: [] + }); + + it('merges paired physical iOS devices alongside simulators', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(90), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { + udid: 'SIM-1', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + } + ], + connected_physical_devices: [ + { + udid: 'PHYS-1', + name: 'Office iPhone', + os_version: '27.1', + product_type: 'iPhone17,2' + }, + { name: 'No UDID — skipped', os_version: '27.0' } + ] + }) + ] + }); + + const devices = service.connectedDevices(); + const physical = devices.find(d => d.serial === 'PHYS-1'); + expect(devices.filter(d => d.platform === 'ios').length).toBe(2); + expect(physical?.platform).toBe('ios'); + expect(physical?.is_emulator).toBeFalse(); + expect(physical?.state).toBe('device'); + expect(physical?.product).toBe('iOS 27.1'); + expect(physical?.android_version).toBeNull(); + }); + + it('selects a physical iOS device by serial and posts its platform', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(91), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + connected_physical_devices: [ + { udid: 'PHYS-9', name: 'Office iPhone', os_version: '27.0' } + ] + }) + ] + }); + + service.selectDevice('PHYS-9', 'ios').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'PHYS-9', platform: 'ios' }); + request.flush({ status: 'success', selected_serial: 'PHYS-9', platform: 'ios' }); + + expect(service.selectedIosDevice()?.serial).toBe('PHYS-9'); + expect(service.selectedIosDevice()?.is_emulator).toBeFalse(); + expect(service.isDeviceReady()).toBeTrue(); + }); + + it('reverts the optimistic iOS selection when the server rejects it', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(92), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { + udid: 'SIM-2', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'r', + isAvailable: true + } + ] + }) + ] + }); + + service.selectDevice('SIM-2', 'ios').subscribe({ error: () => {} }); + expect(service.selectedIosDevice()?.serial).toBe('SIM-2'); // optimistic + http + .expectOne('/api/system/devices/select') + .flush('boom', { status: 500, statusText: 'Internal Server Error' }); + expect(service.selectedIosDevice()).toBeNull(); + }); + + it('gates iOS readiness on macOS + Xcode 27, not the ADB toolchain', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(93), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + { ...envProbe('android_adb', 'fail'), metadata: { installed: false } }, + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { udid: 'SIM-3', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + expect(service.isEnvironmentReady()).toBeFalse(); // no iOS selection yet + + service.selectDevice('SIM-3', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + + expect(service.isEnvironmentReady()).toBeTrue(); + }); + + it('keeps an iOS selection not-ready when Xcode is too old', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(94), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + iosProbeWith({ + xcode_27_or_newer: false, + simulators: [ + { udid: 'SIM-4', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + + service.selectDevice('SIM-4', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + + expect(service.isEnvironmentReady()).toBeFalse(); + }); + + it('restores ADB gates when an Android device is selected', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(95), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + { ...envProbe('android_adb', 'fail'), metadata: { installed: false } }, + { ...envProbe('toolchain'), is_blocker: false }, + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { udid: 'SIM-5', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + + service.selectDevice('SIM-5', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + expect(service.isEnvironmentReady()).toBeTrue(); + + service.selectDevice('emulator-5554', 'android').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + expect(service.selectedIosDevice()).toBeNull(); + expect(service.isEnvironmentReady()).toBeFalse(); + }); + it('restores the standard local ADB server explicitly', () => { service.useLocalAdbServer(true).subscribe(); diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index 7b795268..af1e916a 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -65,10 +65,16 @@ export class SystemService { public isEnvironmentReady = computed(() => { const py = this.pythonProbe(); const cfg = this.configProbe(); - const adb = this.adbProbe(); - const tc = this.toolchainProbe(); const pyOk = py?.status === 'pass'; const cfgOk = cfg?.status === 'pass'; + if (this.selectedIosDevice() != null) { + // An iOS target replaces the ADB toolchain gates: it needs macOS with + // Xcode 27+, never an ADB installation. + const iosOk = this.iosProbe()?.metadata?.['xcode_27_or_newer'] === true; + return pyOk && cfgOk && this.osType() === 'darwin' && iosOk; + } + const adb = this.adbProbe(); + const tc = this.toolchainProbe(); const adbInstalled = adb?.metadata?.['installed'] ?? (adb?.status !== 'fail'); const tcOk = tc?.status === 'pass' || tc?.is_blocker === false; return pyOk && cfgOk && adbInstalled && tcOk; @@ -152,6 +158,22 @@ export class SystemService { }); } } + if (iosMeta && Array.isArray(iosMeta['connected_physical_devices'])) { + for (const p of iosMeta['connected_physical_devices'] as any[]) { + if (!p?.udid) continue; + devices.push({ + serial: p.udid, + state: 'device', + model: p.name ?? null, + product: p.os_version ? `iOS ${p.os_version}` : 'iOS physical device', + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: false, + platform: 'ios', + }); + } + } return devices; }); @@ -538,7 +560,10 @@ export class SystemService { // apply it optimistically since UDID validation can take seconds and the // error path below reverts it if the server rejects the pick. if (platform === 'ios') { - const dev = this.connectedDevices().find(d => d.serial === serial) ?? null; + const dev = + this.connectedDevices().find( + d => d.serial === serial && d.platform === 'ios' + ) ?? null; this.selectedIosDevice.set(dev); } this.isLoading.set(true); @@ -548,7 +573,9 @@ export class SystemService { if (platform === 'ios') { this.selectedIosDevice.set( res?.status === 'success' - ? this.connectedDevices().find(d => d.serial === serial) ?? this.selectedIosDevice() + ? this.connectedDevices().find( + d => d.serial === serial && d.platform === 'ios' + ) ?? this.selectedIosDevice() : null ); } else if (platform === 'android') { diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 3518a7f7..f77113fc 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -115,9 +115,7 @@ def _build_traced_tools(self) -> list: ] if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": # logcat-backed analysis can never succeed on iOS. - all_tools = [ - t for t in all_tools if t.name not in ("run_adb_command", "analyze_logs") - ] + all_tools = [t for t in all_tools if t.name not in ("run_adb_command", "analyze_logs")] if not self.is_device_online: logger.info( "Diagnoser running in OFFLINE environment: stripping adb short" diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index bbf844a8..d2469346 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -58,6 +58,7 @@ from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.runtime.device_lock import DeviceExecutionLock +from artemis.runtime.ios_observation import observe_ios_controller from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -99,6 +100,8 @@ def _uiautomator_fix(serial: str | None) -> list[str]: def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: """iOS repair steps (simctl/Xcode for simulators, devicectl/WDA for hardware).""" + if not error: + return [] s = serial or "" text = (error or "").lower() if "busy" in text or "another task" in text: @@ -324,7 +327,13 @@ async def smoke_test_device( "error": str | None, "fix": list[str]} """ started = time.monotonic() - requested_serial = _resolve_requested_serial(device_serial) + is_ios = normalize_device_platform(platform, strict=False) == "ios" + # iOS targets are UDIDs; ADB_DEVICE_SERIAL is Android-only fallback. + requested_serial = ( + (device_serial or os.environ.get("ARTEMIS_DEVICE_ID")) + if is_ios + else _resolve_requested_serial(device_serial) + ) result: dict[str, Any] = { "ok": False, "serial": requested_serial, @@ -346,7 +355,9 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any ) return result - busy = _find_busy_owner(requested_serial) + # iOS skips this unscoped sweep: observe_ios_controller performs atomic, + # scope-correct lease admission on the resolved UDID itself. + busy = None if is_ios else _find_busy_owner(requested_serial) if busy is not None: key, owner = busy description = getattr(owner, "description", "") or "unknown task" @@ -357,25 +368,30 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any " skipping the screen capture so the running task is not disturbed." ) - is_ios = normalize_device_platform(platform, strict=False) == "ios" try: from artemis.mcp import adb_server - # Keep the Android call shape identical; the platform kwarg only - # exists for iOS. - controller = await _run_in_daemon_thread( - lambda: adb_server._get_controller( - device_serial=device_serial, - **({"target_platform": platform} if is_ios else {}), - ), - timeout_seconds, - "controller-init", - ) + if is_ios: + # iOS controller construction enumerates simctl/devicectl lazily; + # offload the sync factory instead of the daemon-thread hop used + # for the blocking Android UIAutomator handshake. + controller = await asyncio.wait_for( + asyncio.to_thread( + adb_server._get_controller, + device_serial=device_serial, + target_platform="ios", + ), + timeout_seconds, + ) + else: + controller = await _run_in_daemon_thread( + lambda: adb_server._get_controller(device_serial=device_serial), + timeout_seconds, + "controller-init", + ) except TimeoutError: label = "iOS driver" if is_ios else "UIAutomator/controller" - return _finish( - f"{label} initialization did not respond within {timeout_seconds:g}s" - ) + return _finish(f"{label} initialization did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught @@ -391,18 +407,22 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any remaining = max(0.5, timeout_seconds - (time.monotonic() - started)) try: - - async def _observe() -> Any: - if is_ios: - # The native Xcode session opens lazily on first use. - await controller._driver.connect() - return await controller.get_screen_data() - - device_data = await _run_in_daemon_thread( - lambda: _run_coroutine_blocking(_observe), - remaining, - "screen-data", - ) + if is_ios: + # Runs on the caller's loop: the helper owns lease + connect + + # capture + disconnect + release and is cancellation-safe, unlike + # the Android coroutine which must be bounced onto a private + # thread/loop because UIAutomator blocks the loop thread. + device_data = await asyncio.wait_for(observe_ios_controller(controller), remaining) + else: + + async def _observe() -> Any: + return await controller.get_screen_data() + + device_data = await _run_in_daemon_thread( + lambda: _run_coroutine_blocking(_observe), + remaining, + "screen-data", + ) except TimeoutError: label = "iOS session capture" if is_ios else "UIAutomator/screen capture" return _finish(f"{label} did not respond within {timeout_seconds:g}s") @@ -417,8 +437,14 @@ async def _observe() -> Any: if is_ios: from artemis.drivers.ios.physical_driver import PhysicalIosDriver + # Report the canonical UDID the observation lease actually resolved. + resolved_serial = getattr(device, "device_id", None) + if isinstance(resolved_serial, str) and resolved_serial: + result["serial"] = resolved_serial result["hierarchy_backend"] = ( - "wda" if isinstance(getattr(controller, "_driver", None), PhysicalIosDriver) else "xcode" + "wda" + if isinstance(getattr(controller, "_driver", None), PhysicalIosDriver) + else "xcode" ) else: from artemis.clients.screen_client_factory import describe_backend @@ -435,9 +461,7 @@ async def _observe() -> Any: ) if not result["element_count"]: backend = "Xcode accessibility tree" if is_ios else "UIAutomator hierarchy dump" - return _finish( - f"{backend} returned no UI elements (screenshot worked, hierarchy did not)" - ) + return _finish(f"{backend} returned no UI elements (screenshot worked, hierarchy did not)") return _finish(None) diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py index cd2990bb..bb4c7865 100644 --- a/artemis/core/diagnostics/probes/ios_probe.py +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -95,7 +95,13 @@ async def probe(self) -> ProbeResult: "simulators": simulators or [], "physical_device_count": len(physical), "connected_physical_devices": [ - {"udid": d.get("udid"), "name": d.get("name")} for d in connected + { + "udid": d.get("udid"), + "name": d.get("name"), + "os_version": d.get("os_version"), + "product_type": d.get("product_type"), + } + for d in connected ], } diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index d2b709d5..e997b813 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -41,15 +41,16 @@ def ios_driver_class(device_id: str | None) -> type[BaseDeviceDriver]: from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver - if device_id and device_id != BOOTED_SIMULATOR_ID: + if device_id and device_id.strip().lower() != BOOTED_SIMULATOR_ID: from artemis.drivers.ios.discovery import ( + device_matches_identifier, find_physical_ios_device_sync, list_ios_simulators_sync, ) simulators = list_ios_simulators_sync() if simulators is None or not any( - device.get("udid", "").lower() == device_id.lower() for device in simulators + device_matches_identifier(device, device_id) for device in simulators ): if find_physical_ios_device_sync(device_id) is not None: from artemis.drivers.ios.physical_driver import PhysicalIosDriver @@ -63,9 +64,7 @@ def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": raise ValueError("iOS support is local only; cloud mode targets Android.") config = getattr(ctx, "agent_config", None) - workspace_path = ( - getattr(config, "ios_workspace_path", None) if config is not None else None - ) + workspace_path = getattr(config, "ios_workspace_path", None) if config is not None else None driver_class = ios_driver_class(ctx.device.device_id) return driver_class( device_id=ctx.device.device_id, diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py index 2490b55a..105db8d5 100644 --- a/artemis/drivers/ios/bridge.py +++ b/artemis/drivers/ios/bridge.py @@ -222,7 +222,7 @@ async def close(self) -> None: except Exception as exc: # Owner failures already propagate to requesters via futures; a # close() must never mask the error its caller is handling. - logger.debug("Xcode bridge owner exited with an error: %s", exc) + logger.debug(f"Xcode bridge owner exited with an error: {exc}") if self._ready is not None and self._ready.done() and not self._ready.cancelled(): self._ready.exception() # consume an initialization error on cancellation self.tools.clear() diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py index 4c9a3e34..1860a599 100644 --- a/artemis/drivers/ios/discovery.py +++ b/artemis/drivers/ios/discovery.py @@ -135,9 +135,7 @@ async def plist_to_json(payload: bytes, timeout: float = 30.0) -> bytes: await reap_process(process) raise if process.returncode != 0: - raise RuntimeError( - f"plutil conversion failed: {stderr.decode(errors='replace').strip()}" - ) + raise RuntimeError(f"plutil conversion failed: {stderr.decode(errors='replace').strip()}") return stdout @@ -230,9 +228,7 @@ def list_ios_simulators_sync( DEVICECTL_ENUMERATE_TIMEOUT = 20.0 -async def devicectl_screenshot( - device_id: str, destination: Path, timeout: float -) -> None: +async def devicectl_screenshot(device_id: str, destination: Path, timeout: float) -> None: """Capture one PNG frame from a paired device via ``devicectl``.""" await run_xcrun( "devicectl", @@ -265,6 +261,29 @@ def _device_property(device: dict[str, Any], section: str, key: str) -> Any: return nested.get(key) if isinstance(nested, dict) else None +def _modern_property(device: dict[str, Any], section: str, key: str) -> Any: + """Read ``properties.
.`` from Xcode 27's devicectl shape. + + The modern ``properties`` map uses short section names (``hardware``, + ``software``, ``state``, ``connection``) rather than the deprecated + ``hardwareProperties``/``deviceProperties``/``connectionProperties`` + spellings. Sections may be present-but-null, so guard every hop. + """ + properties = device.get("properties") + if not isinstance(properties, dict): + return None + nested = properties.get(section) + return nested.get(key) if isinstance(nested, dict) else None + + +def _modern_os_version(device: dict[str, Any]) -> Any: + """``properties.software.osVersionNumber`` may carry a stringValue map.""" + version = _modern_property(device, "software", "osVersionNumber") + if isinstance(version, dict): + return version.get("stringValue") or version.get("string") + return version + + def parse_devicectl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: """Flatten ``devicectl list devices --json-output -`` into device entries.""" devices = [] @@ -277,28 +296,32 @@ def parse_devicectl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: for device in entries: if not isinstance(device, dict): continue - udid = _device_property(device, "hardwareProperties", "udid") + udid = ( + _device_property(device, "hardwareProperties", "udid") + or _modern_property(device, "hardware", "udid") + or device.get("identifier") + ) if not udid: continue devices.append( { "udid": udid, - "name": _device_property(device, "deviceProperties", "name"), - "os_version": _device_property( - device, "deviceProperties", "osVersionNumber" - ), - "platform": _device_property(device, "hardwareProperties", "platform"), - "reality": _device_property(device, "hardwareProperties", "reality"), - "product_type": _device_property( - device, "hardwareProperties", "productType" - ), - "connection_state": _device_property( - device, "connectionProperties", "tunnelState" - ), - "pairing_state": _device_property( - device, "connectionProperties", "pairingState" - ), - "visibility": device.get("visibilityClass"), + "name": _device_property(device, "deviceProperties", "name") + or _modern_property(device, "state", "name"), + "os_version": _device_property(device, "deviceProperties", "osVersionNumber") + or _modern_os_version(device), + "platform": _device_property(device, "hardwareProperties", "platform") + or _modern_property(device, "hardware", "platform"), + "reality": _device_property(device, "hardwareProperties", "reality") + or _modern_property(device, "hardware", "reality"), + "product_type": _device_property(device, "hardwareProperties", "productType") + or _modern_property(device, "hardware", "productType"), + "connection_state": _device_property(device, "connectionProperties", "tunnelState") + or _modern_property(device, "connection", "state"), + "pairing_state": _device_property(device, "connectionProperties", "pairingState") + or _modern_property(device, "connection", "pairingState"), + "visibility": device.get("visibilityClass") + or _modern_property(device, "state", "visibilityClass"), } ) return devices @@ -316,7 +339,12 @@ async def list_core_devices( return cached try: raw = await run_xcrun( - "devicectl", "list", "devices", "--json-output", "-", timeout=DEVICECTL_ENUMERATE_TIMEOUT + "devicectl", + "list", + "devices", + "--json-output", + "-", + timeout=DEVICECTL_ENUMERATE_TIMEOUT, ) except (OSError, RuntimeError, TimeoutError) as exc: logger.debug(f"devicectl device enumeration failed: {exc}") @@ -360,10 +388,7 @@ def list_core_devices_sync( def is_physical_ios(device: dict[str, Any]) -> bool: """Whether a CoreDevice entry is a physical iPhone/iPad (not a simulator).""" - return ( - device.get("platform") in ("iOS", "iPadOS") - and device.get("reality") == "physical" - ) + return device.get("platform") in ("iOS", "iPadOS") and device.get("reality") == "physical" def physical_ios_ready(device: dict[str, Any]) -> bool: @@ -392,13 +417,23 @@ def device_matches_identifier(device: dict[str, Any], identifier: str) -> bool: def find_physical_ios_device_sync(identifier: str) -> dict[str, Any] | None: - """Match a physical iOS device by UDID or exact name, or ``None``.""" + """Match a physical iOS device by UDID or exact name, or ``None``. + + Raises ``ValueError`` when the identifier matches more than one device — + silently picking the first would drive an arbitrary phone. + """ devices = list_core_devices_sync() if devices is None or not identifier: return None - for device in devices: - if not is_physical_ios(device): - continue - if device_matches_identifier(device, identifier): - return device - return None + matches = [ + device + for device in devices + if is_physical_ios(device) and device_matches_identifier(device, identifier) + ] + if len(matches) > 1: + udids = sorted(str(device.get("udid") or "?") for device in matches) + raise ValueError( + f"{len(matches)} physical iOS devices match {identifier!r} " + f"({', '.join(udids)}); use the device UDID instead." + ) + return matches[0] if matches else None diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py index 1f356091..1566a8cc 100644 --- a/artemis/drivers/ios/physical_driver.py +++ b/artemis/drivers/ios/physical_driver.py @@ -28,7 +28,6 @@ import asyncio import base64 -import contextlib from io import BytesIO import json import os @@ -47,6 +46,7 @@ devicectl_screenshot, is_physical_ios, list_core_devices, + reap_process, run_xcrun, ) from artemis.drivers.ios.physical_recording import PhysicalIosRecorder @@ -112,7 +112,7 @@ def __init__( async def _resolve_device(self) -> dict[str, Any]: self._validate_workspace() - if self._device_id in ("", BOOTED_SIMULATOR_ID): + if not self._device_id.strip() or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: raise ValueError( "Physical iOS devices require --device-serial ; " "'booted' only selects simulators. Find UDIDs via 'xcrun devicectl list devices'." @@ -126,16 +126,14 @@ async def _resolve_device(self) -> dict[str, Any]: matches = [ device for device in devices - if is_physical_ios(device) - and device_matches_identifier(device, self._device_id) + if is_physical_ios(device) and device_matches_identifier(device, self._device_id) ] if not matches: needle = self._device_id.lower() simulator = [ device for device in devices - if device.get("udid", "").lower() == needle - and device.get("reality") == "simulated" + if device.get("udid", "").lower() == needle and device.get("reality") == "simulated" ] if simulator: raise ValueError( @@ -145,6 +143,12 @@ async def _resolve_device(self) -> dict[str, Any]: f"No paired physical iOS device matches {self._device_id!r}. " "Attach it, trust this Mac, and verify 'xcrun devicectl list devices'." ) + if len(matches) > 1: + udids = sorted(device.get("udid", "?") for device in matches) + raise ValueError( + f"{len(matches)} physical iOS devices match {self._device_id!r} " + f"({', '.join(udids)}); target the device UDID instead." + ) candidate = matches[0] self._device_id = candidate["udid"] return candidate @@ -171,21 +175,41 @@ async def connect(self) -> None: return candidate = await self._resolve_device() await self._prepare_device(candidate) + connected = False try: self._wda = await self._ensure_wda() + # The reachable WDA endpoint must belong to THIS device before + # we open a session or send input: names corroborate identity + # (uuid is identifierForVendor, not the UDID). + info = await self._wda.device_info() + expected_name = candidate.get("name") or "" + if ( + info.get("isSimulator") is not False + or not expected_name + or info.get("name") != expected_name + ): + raise RuntimeError( + f"The WebDriverAgent at {self._wda.base_url} does not report " + f"the selected physical device {self._device_id} " + f"(expected name {expected_name!r}, got " + f"{info.get('name')!r}, isSimulator={info.get('isSimulator')!r}). " + "Point ARTEMIS_IOS_WDA_URL at a WDA server running on the " + "selected device." + ) self._session_key = await self._wda.open_session() await self.get_screen_data(skip_settling=True) - # Any failure here must release a half-started WDA runner, - # xcodebuild session, or server-side session. - except (Exception, asyncio.CancelledError): - try: - await self.disconnect() - except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: - logger.warning( - "Could not release the WDA session after a connection failure: " - f"{cleanup_error}", - ) - raise + connected = True + finally: + # Any incomplete setup must release a half-started WDA + # runner, xcodebuild session, or server-side session. + if not connected: + try: + await self.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: + logger.warning( + "Could not release the WDA session after a connection failure: " + f"{cleanup_error}", + ) async def disconnect(self) -> None: try: @@ -196,7 +220,7 @@ async def disconnect(self) -> None: # conversions — finalize it too so nothing is lost. try: await self._recorder.stop() - except Exception as exc: + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: logger.error( f"Physical iOS recording finalization failed during disconnect: {exc}" ) @@ -210,15 +234,24 @@ async def disconnect(self) -> None: client, self._wda = self._wda, None runner_pid, self._wda_runner_pid = self._wda_runner_pid, None test_process, self._wda_test_process = self._wda_test_process, None + # Each cleanup step is isolated so a failure in one never skips + # the owned WDA session, the xcodebuild child, or the runner pid. if client is not None: - await client.close_session() + try: + await client.close_session() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug(f"WDA session close failed during disconnect: {exc}") if test_process is not None and test_process.returncode is None: try: test_process.terminate() await asyncio.wait_for(test_process.wait(), timeout=10.0) except (TimeoutError, OSError): - with contextlib.suppress(OSError): - test_process.kill() + # Graceful terminate failed — reap (kill + drain) the + # xcodebuild child so it cannot outlive the driver. + try: + await reap_process(test_process) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"WDA xcodebuild reap failed: {exc}") if runner_pid is not None: try: await self._terminate_pid(runner_pid) @@ -226,9 +259,17 @@ async def disconnect(self) -> None: logger.debug(f"WDA runner termination failed: {exc}") def _require_connected(self) -> None: - if not self._session_key or self._device_id in ("", BOOTED_SIMULATOR_ID): + if not self._session_key or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: raise RuntimeError("Connect the physical iOS driver before interacting.") + def _require_wda(self) -> WdaClient: + """Bound WDA client for input paths; fails clearly when detached.""" + self._require_connected() + client = self._wda + if client is None: + raise RuntimeError("Connect the physical iOS driver before interacting.") + return client + async def _ensure_wda(self) -> WdaClient: """Attach to a reachable WDA server, starting one when possible. @@ -249,9 +290,7 @@ async def _ensure_wda(self) -> WdaClient: await self._start_xctest_session(xctestrun) elif runner is not None: try: - self._wda_runner_pid = await self._launch_bundle( - runner, terminate_existing=False - ) + self._wda_runner_pid = await self._launch_bundle(runner, terminate_existing=False) logger.info( f"Launched WebDriverAgent runner {runner} on {self._device_id}; " "waiting for its HTTP server" @@ -277,9 +316,7 @@ async def _ensure_wda(self) -> WdaClient: ) client = await probe_wda(candidates, timeout=min(5.0, remaining)) if client is not None: - logger.info( - f"WebDriverAgent attached at {client.base_url} for {self._device_id}" - ) + logger.info(f"WebDriverAgent attached at {client.base_url} for {self._device_id}") return client await asyncio.sleep(min(1.0, remaining)) @@ -343,9 +380,7 @@ async def _wda_runner_bundle(self) -> str | None: matches = sorted(b for b in bundles if WDA_RUNNER_PATTERN.search(b)) return matches[0] if matches else None - async def _launch_bundle( - self, bundle: str, terminate_existing: bool = True - ) -> int | None: + async def _launch_bundle(self, bundle: str, terminate_existing: bool = True) -> int | None: arguments = [ "devicectl", "device", @@ -393,9 +428,7 @@ async def _screenshot_png(self) -> bytes: logger.debug(f"WDA screenshot failed, falling back to devicectl: {exc}") with tempfile.TemporaryDirectory(prefix="artemis-shot-") as tmp: target = Path(tmp) / "shot.png" - await devicectl_screenshot( - self._device_id, target, timeout=DEVICECTL_OP_TIMEOUT - ) + await devicectl_screenshot(self._device_id, target, timeout=DEVICECTL_OP_TIMEOUT) data = target.read_bytes() if not data: raise RuntimeError("devicectl produced an empty screenshot.") @@ -429,9 +462,7 @@ async def _capture(self) -> ScreenData: ) self._scale = scale tree = await self._wda.source_json() - elements = parse_wda_elements( - tree, self._scale, self._width, self._height - ) + elements = parse_wda_elements(tree, self._scale, self._width, self._height) return ScreenData( screenshot_bytes=screenshot, screenshot_base64=base64.b64encode(screenshot).decode("ascii"), @@ -449,10 +480,11 @@ async def tap( if times < 1 or duration_ms < 0 or delay_ms < 0: raise ValueError("Tap count must be positive and durations nonnegative.") async with self._operation_lock: + wda = self._require_wda() await self._capture_unchanged("tapping") point = self._scaled_point(x, y) for index in range(times): - await self._wda.tap(*point, hold_ms=duration_ms) + await wda.tap(*point, hold_ms=duration_ms) if index < times - 1: await asyncio.sleep(delay_ms / 1000) return True @@ -463,10 +495,11 @@ async def swipe( if duration_ms <= 0: raise ValueError("Swipe duration must be positive.") async with self._operation_lock: + wda = self._require_wda() await self._capture_unchanged("swiping") start = self._scaled_point(start_x, start_y) end = self._scaled_point(end_x, end_y) - await self._wda.swipe(*start, *end, duration_ms) + await wda.swipe(*start, *end, duration_ms) return True async def swipe_direction( @@ -480,11 +513,12 @@ async def swipe_direction( async with self._operation_lock: # Rotation-tolerant like the simulator path: recapture and compute # from the *current* size rather than refusing like tap. + wda = self._require_wda() await self._capture() sx, sy, ex, ey = self._direction_points(direction, *self.screen_size) start = self._scaled_point(sx, sy) end = self._scaled_point(ex, ey) - await self._wda.swipe(*start, *end, duration_ms) + await wda.swipe(*start, *end, duration_ms) return True async def input_text(self, text: str, clear_existing: bool = True) -> bool: @@ -494,30 +528,37 @@ async def input_text(self, text: str, clear_existing: bool = True) -> bool: "UI or pass clear_existing=False. This matches the simulator behavior." ) async with self._operation_lock: - self._require_connected() - await self._wda.type_text(text) + wda = self._require_wda() + await wda.type_text(text) return True async def press_key(self, key: KeyCode | str | int) -> bool: key = key.value if isinstance(key, KeyCode) else str(key).lower() async with self._operation_lock: - self._require_connected() + wda = self._require_wda() if key in ("home", "app_switch"): presses = 2 if key == "app_switch" else 1 for index in range(presses): - if not await self._wda.press_button("home"): - await self._wda.homescreen() + if not await wda.press_button("home"): + if key == "app_switch": + # A backend that cannot press Home cannot + # double-press for app switching — never report + # success from a homescreen fallback. + raise NotImplementedError( + "App switching is unavailable from this WebDriverAgent backend." + ) + await wda.homescreen() if index < presses - 1: await asyncio.sleep(0.4) elif key == "enter": - await self._wda.type_text("\n") + await wda.type_text("\n") else: buttons = { "volume_up": "volumeUp", "volume_down": "volumeDown", "power": "power", } - if key not in buttons or not await self._wda.press_button(buttons[key]): + if key not in buttons or not await wda.press_button(buttons[key]): raise NotImplementedError( f"Key {key!r} is not supported by the physical iOS driver." ) @@ -527,14 +568,13 @@ async def tap_element( self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 ) -> bool: async with self._operation_lock: + wda = self._require_wda() data = await self._capture() element, center, error = await self.find_element(query, data) if error or element is None or center is None: return False point = self._scaled_point(*center) - await self._wda.tap( - *point, hold_ms=duration_ms if long_press else 0 - ) + await wda.tap(*point, hold_ms=duration_ms if long_press else 0) return True # --- App lifecycle via devicectl --- @@ -578,40 +618,42 @@ def _normalize_executable(raw: Any) -> str: return value.removeprefix("/private") or value async def _resolve_pid(self, package_name: str) -> int: - """Find a running process id for the bundle, tracking launches first.""" + """Verified live pid: the executable must sit under the app's own URL. + + A cached launch pid only prioritizes among processes already proven to + belong to the app — a recycled pid must never select a foreign process. + """ + apps = await self._devicectl_json("info", "apps") + url_prefix = "" + for app in apps.get("apps", []): + if isinstance(app, dict) and app.get("bundleIdentifier") == package_name: + url_prefix = self._normalize_executable(app.get("url")).rstrip("/") + break + if not url_prefix: + raise ValueError( + f"{package_name!r} is not installed on {self._device_id} or its app URL " + "could not be determined; cannot verify a process to terminate." + ) + processes = await self._devicectl_json("info", "processes") + candidates: list[int] = [] + for process in processes.get("runningProcesses", []): + if not isinstance(process, dict): + continue + executable = self._normalize_executable(process.get("executable")) + if executable != url_prefix and not executable.startswith(url_prefix + "/"): + continue + pid = process.get("processIdentifier") or process.get("pid") or process.get("processID") + if isinstance(pid, int): + candidates.append(pid) + if not candidates: + raise ValueError( + f"No running process found for {package_name!r} on {self._device_id}; " + "launch it with launch_app before stopping." + ) tracked = self._launched_pids.get(package_name) - if tracked is not None: + if tracked is not None and tracked in candidates: return tracked - try: - apps = await self._devicectl_json("info", "apps") - url_prefix = "" - for app in apps.get("apps", []): - if isinstance(app, dict) and app.get("bundleIdentifier") == package_name: - url_prefix = self._normalize_executable(app.get("url")).rstrip("/") - break - processes = await self._devicectl_json("info", "processes") - for process in processes.get("runningProcesses", []): - if not isinstance(process, dict): - continue - executable = self._normalize_executable(process.get("executable")) - matched = ( - executable.startswith(url_prefix) - if url_prefix - else package_name in executable - ) - pid = ( - process.get("processIdentifier") - or process.get("pid") - or process.get("processID") - ) - if matched and isinstance(pid, int): - return pid - except (OSError, RuntimeError, TimeoutError, ValueError, TypeError) as exc: - logger.debug(f"devicectl process enumeration failed: {exc}") - raise ValueError( - f"No running process found for {package_name!r} on {self._device_id}; " - "launch it with launch_app before stopping." - ) + return candidates[0] async def _terminate_pid(self, pid: int, kill: bool = False) -> None: arguments = [ @@ -631,17 +673,11 @@ async def _terminate_pid(self, pid: int, kill: bool = False) -> None: async def stop_app(self, package_name: str) -> bool: async with self._operation_lock: self._require_connected() - tracked = self._launched_pids.pop(package_name, None) - if tracked is not None: - try: - await self._terminate_pid(tracked, kill=True) - return True - except (OSError, RuntimeError, TimeoutError) as exc: - # A stale or recycled PID must not poison stop_app — - # fall through to the live process scan. - logger.debug(f"Tracked-PID terminate failed for {package_name}: {exc}") + # Verify the live pid first — a recycled cached pid could belong to + # a different app now, and terminating it would kill the wrong app. pid = await self._resolve_pid(package_name) await self._terminate_pid(pid, kill=True) + self._launched_pids.pop(package_name, None) return True async def install_app(self, app_path: Path) -> str: diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py index dd4f5d22..06cc9f51 100644 --- a/artemis/drivers/ios/physical_recording.py +++ b/artemis/drivers/ios/physical_recording.py @@ -35,6 +35,7 @@ from artemis.drivers.ios.discovery import devicectl_screenshot, reap_process from artemis.drivers.ios.recording import ( DEFAULT_MAX_DURATION_SECONDS, + SEGMENT_PROBE_TIMEOUT_SECONDS, IosRecordingSession, ) from artemis.utils.video import get_ffmpeg_path, probe_video_segment, write_recording_manifest @@ -68,7 +69,17 @@ async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: except TimeoutError: await reap_process(process) return -1, b"ffmpeg timed out" - return process.returncode, stderr + except asyncio.CancelledError: + await reap_process(process) + # Drop only our own in-flight .part output before propagating. + part = Path(str(arguments[-1])) + if part.suffix == ".mp4" and ".part" in part.name: + part.unlink(missing_ok=True) + raise + rc = process.returncode + if rc is None: + raise RuntimeError("ffmpeg completed without an exit code") + return rc, stderr class IosPhysicalRecordingSession(IosRecordingSession): @@ -105,9 +116,7 @@ async def _capture_frame(self, session: IosPhysicalRecordingSession) -> None: assert session.frames_dir is not None session.frame_index += 1 path = session.frames_dir / f"frame_{session.frame_index:06d}.png" - await devicectl_screenshot( - self._device_id, path, timeout=CAPTURE_TIMEOUT_SECONDS - ) + await devicectl_screenshot(self._device_id, path, timeout=CAPTURE_TIMEOUT_SECONDS) captured_at = time.monotonic() if not path.exists() or path.stat().st_size == 0: raise RuntimeError("devicectl produced no screenshot frame") @@ -171,9 +180,7 @@ async def _poll_first_frame(self, session: IosPhysicalRecordingSession) -> None: deadline = time.monotonic() + CAPTURE_TIMEOUT_SECONDS * 2 while session.anchor_monotonic is None: if time.monotonic() > deadline: - raise RuntimeError( - "Timed out waiting for the first devicectl screenshot frame." - ) + raise RuntimeError("Timed out waiting for the first devicectl screenshot frame.") if not session.is_active: raise RuntimeError( session.errors[-1] @@ -206,23 +213,31 @@ def _seal_current_segment( session.segment_index += 1 return record + @staticmethod + def _ffconcat_file_line(path: Path) -> str: + """Quote a frame path for ffconcat: ' inside a quoted name -> '\\''.""" + escaped = path.as_posix().replace("'", "'\\''") + return f"file '{escaped}'" + def _write_concat(self, session: IosPhysicalRecordingSession, record: dict[str, Any]) -> Path: """Write an ffconcat list preserving real capture timestamps.""" frames = record["frames"] end_monotonic = record["end"] + (session.anchor_monotonic or 0.0) lines = ["ffconcat version 1.0"] for index, frame in enumerate(frames): - lines.append(f"file '{Path(frame['path']).as_posix()}'") + lines.append(self._ffconcat_file_line(Path(frame["path"]))) following = frames[index + 1]["at"] if index + 1 < len(frames) else end_monotonic lines.append(f"duration {max(0.001, following - frame['at']):.3f}") # The concat demuxer ignores the final frame's duration unless the file # line repeats, so list the last frame twice to keep its tail visible. - lines.append(f"file '{Path(frames[-1]['path']).as_posix()}'") + lines.append(self._ffconcat_file_line(Path(frames[-1]["path"]))) concat_path = Path(frames[0]["path"]).parent / "frames.txt" concat_path.write_text("\n".join(lines) + "\n", encoding="utf-8") return concat_path - async def _convert_record(self, session: IosPhysicalRecordingSession, record: dict[str, Any]) -> None: + async def _convert_record( + self, session: IosPhysicalRecordingSession, record: dict[str, Any] + ) -> None: output = Path(record["output_path"]) width = max(2, int(record["width"] or 1080)) // 2 * 2 height = max(2, int(record["height"] or 1920)) // 2 * 2 @@ -258,7 +273,9 @@ async def _convert_record(self, session: IosPhysicalRecordingSession, record: di str(temporary), ] ) - metadata = await probe_video_segment(temporary) + metadata = await probe_video_segment( + temporary, timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS + ) valid = ( returncode == 0 and temporary.exists() @@ -275,6 +292,11 @@ async def _convert_record(self, session: IosPhysicalRecordingSession, record: di f"Physical iOS recording finalization failed (code {returncode}): " f"{stderr.decode(errors='replace')[-2000:]}" ) + except TimeoutError as exc: + # A bounded probe that expires records a failure — never report a + # segment whose metadata could not be verified as converted. + record["conversion_error"] = f"metadata probe timed out: {exc}" + logger.error(f"Physical iOS recording probe timed out: {exc}") except OSError as exc: record["conversion_error"] = str(exc) logger.error(f"Physical iOS recording finalization failed: {exc}") @@ -283,9 +305,7 @@ async def _convert_record(self, session: IosPhysicalRecordingSession, record: di temporary.unlink() except OSError: pass - session.errors.append( - f"Segment {record['output_path']} failed finalization; frames kept" - ) + session.errors.append(f"Segment {record['output_path']} failed finalization; frames kept") async def _roll( self, session: IosPhysicalRecordingSession, end_monotonic: float, reason: str @@ -300,8 +320,11 @@ async def _roll( ) if session.is_active: await self._start_segment(session) + frames_dir = session.frames_dir + if frames_dir is None: + raise RuntimeError("Physical iOS recording roll found no frame directory.") for frame in leftover: - target = session.frames_dir / Path(frame["path"]).name + target = frames_dir / Path(frame["path"]).name Path(frame["path"]).rename(target) frame["path"] = target session.frames = leftover @@ -345,15 +368,11 @@ async def start( ) -> IosPhysicalRecordingSession: async with self._lock: if self._session is not None and self._session.is_active: - raise RuntimeError( - f"iOS recording is already active on {self._device_id}" - ) + raise RuntimeError(f"iOS recording is already active on {self._device_id}") output = ( Path(output_dir) if output_dir is not None - else Path( - tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings")) - ) + else Path(tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings"))) ) output.mkdir(parents=True, exist_ok=True) self._output_dir = output @@ -370,18 +389,22 @@ async def start( session.poll_task = asyncio.create_task(self._poll_loop(session)) # The first-frame wait must run outside the lock: the poll loop takes # the same lock to append captured frames. + completed = False try: await self._poll_first_frame(session) - except BaseException: - session.is_active = False - if session.poll_task and not session.poll_task.done(): - session.poll_task.cancel() - if session.poll_task is not None: - with contextlib.suppress(asyncio.CancelledError, Exception): - await session.poll_task - raise - session.watchdog_task = asyncio.create_task(self._watchdog(session)) - return session + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + completed = True + return session + finally: + if not completed: + session.is_active = False + if session.poll_task and not session.poll_task.done(): + session.poll_task.cancel() + if session.poll_task is not None: + with contextlib.suppress( + asyncio.CancelledError, OSError, ValueError, RuntimeError, TimeoutError + ): + await session.poll_task async def seal(self, through_time: float | None = None) -> None: """Seal the current segment so its final MP4 can be read safely.""" @@ -439,7 +462,16 @@ async def _finalize(self, session: IosPhysicalRecordingSession) -> Path | None: if record.get("conversion_done") and Path(record["output_path"]).exists() } output_dir = mp4_paths[0].parent - manifest = await write_recording_manifest(output_dir, mp4_paths, offsets) + try: + manifest = await write_recording_manifest( + output_dir, + mp4_paths, + offsets, + probe_timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + session.errors.append(f"Recording manifest probe failed: {exc}") + return None if manifest is None: session.errors.append("Recording manifest has no valid segments") return None diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py index d345a711..b056e044 100644 --- a/artemis/drivers/ios/recording.py +++ b/artemis/drivers/ios/recording.py @@ -48,6 +48,7 @@ SIGINT_FLUSH_TIMEOUT_SECONDS = 10.0 TERMINATE_TIMEOUT_SECONDS = 3.0 PROBES_TIMEOUT_SECONDS = 3.0 +SEGMENT_PROBE_TIMEOUT_SECONDS = 30.0 FFMPEG_TIMEOUT_SECONDS = 120.0 WATCHDOG_INTERVAL_SECONDS = 0.5 MAX_CONSECUTIVE_FAILURES = 3 @@ -177,15 +178,20 @@ async def finalize_mov_to_mp4( stderr=asyncio.subprocess.PIPE, ) try: - _stdout, stderr = await asyncio.wait_for( - process.communicate(), FFMPEG_TIMEOUT_SECONDS - ) - metadata = await asyncio.wait_for( - probe_video_segment(temporary_path), PROBES_TIMEOUT_SECONDS * 10 + _stdout, stderr = await asyncio.wait_for(process.communicate(), FFMPEG_TIMEOUT_SECONDS) + metadata = await probe_video_segment( + temporary_path, timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS ) except TimeoutError: await reap_process(process) raise RuntimeError("iOS recording finalization timed out") + except asyncio.CancelledError: + # Reap the owned ffmpeg child and drop the partial .mp4 before + # cancellation propagates — never leave an orphan encoder behind. + await reap_process(process) + if temporary_path.exists(): + temporary_path.unlink() + raise valid = ( process.returncode == 0 and temporary_path.exists() @@ -320,34 +326,45 @@ async def _spawn_recorder(self, session: IosRecordingSession) -> None: ) first_frame: asyncio.Future = asyncio.get_running_loop().create_future() session.stderr_task = asyncio.create_task(self._drain_stderr(process, session, first_frame)) + started = False try: marker_wall, marker_monotonic = await asyncio.wait_for( asyncio.shield(first_frame), STARTUP_TIMEOUT_SECONDS ) - except BaseException: - if not first_frame.done(): - first_frame.cancel() - session.stderr_task.cancel() - reap = asyncio.ensure_future(self._reap(process)) - try: - await asyncio.shield(reap) - except asyncio.CancelledError: - await reap # finish reaping even when the caller was cancelled - raise - session.process = process - session.local_video_path = source_path - session.segment_started_at = marker_wall - session.segment_started_monotonic = marker_monotonic - if session.anchor_monotonic is None: - session.start_time = marker_wall - session.anchor_monotonic = marker_monotonic - dimensions = await probe_display_dimensions(self._device_id) - if dimensions: - session.capture_width, session.capture_height = dimensions - logger.info( - f"iOS recording segment {index} first frame " - f"{marker_wall - session.start_time:.2f}s after anchor on {self._device_id}" - ) + session.process = process + session.local_video_path = source_path + session.segment_started_at = marker_wall + session.segment_started_monotonic = marker_monotonic + if session.anchor_monotonic is None: + session.start_time = marker_wall + session.anchor_monotonic = marker_monotonic + dimensions = await probe_display_dimensions(self._device_id) + if dimensions: + session.capture_width, session.capture_height = dimensions + logger.info( + f"iOS recording segment {index} first frame " + f"{marker_wall - session.start_time:.2f}s after anchor on {self._device_id}" + ) + started = True + finally: + if not started: + # Startup never completed: the spawned recorder is still + # owned by this coroutine — drain stderr and reap it before + # the original exception/cancellation propagates. + if not first_frame.done(): + first_frame.cancel() + session.stderr_task.cancel() + try: + await session.stderr_task + except asyncio.CancelledError: + pass + except (OSError, RuntimeError, ValueError, TimeoutError) as stderr_error: + logger.debug(f"iOS recorder stderr drain ended with an error: {stderr_error}") + reap = asyncio.ensure_future(self._reap(process)) + try: + await asyncio.shield(reap) + except asyncio.CancelledError: + await reap # finish reaping even when the caller was cancelled def _seal_current_segment( self, session: IosRecordingSession, end_monotonic: float @@ -402,19 +419,29 @@ async def _roll(self, session: IosRecordingSession, end_monotonic: float, reason logger.info(f"Rolling iOS recording segment after {reason}") restarted = False if session.is_active: - try: - await self._spawn_recorder(session) - restarted = True - except asyncio.CancelledError: - raise - except (OSError, RuntimeError, TimeoutError) as exc: - self._consecutive_failures += 1 - session.errors.append(f"Recorder restart failed: {exc}") + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + # Spawns that keep dying young are a crash loop: stop + # respawning (the sealed segment still converts below). + session.is_active = False + session.errors.append("Recording recovery limit reached; session stopped") + else: + try: + await self._spawn_recorder(session) + restarted = True + except asyncio.CancelledError: + raise + except (OSError, RuntimeError, TimeoutError) as exc: + self._consecutive_failures += 1 + session.errors.append(f"Recorder restart failed: {exc}") if record is not None: session.conversion_tasks.append( asyncio.create_task(self._convert_record(session, record)) ) - if not restarted and self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + if ( + not restarted + and session.is_active + and self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES + ): session.errors.append("Recording recovery limit reached; session stopped") session.is_active = False @@ -482,13 +509,15 @@ async def start( is_active=True, ) self._session = session + started = False try: await self._spawn_recorder(session) - except BaseException: - session.is_active = False - raise - session.watchdog_task = asyncio.create_task(self._watchdog(session)) - return session + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + started = True + return session + finally: + if not started: + session.is_active = False async def seal(self, through_time: float | None = None) -> None: """Seal the current segment through ``through_time`` (recording-relative).""" @@ -555,7 +584,16 @@ async def _finalize(self, session: IosRecordingSession) -> Path | None: if record.get("conversion_done") and Path(record["output_path"]).exists() } output_dir = mp4_paths[0].parent - manifest = await write_recording_manifest(output_dir, mp4_paths, offsets) + try: + manifest = await write_recording_manifest( + output_dir, + mp4_paths, + offsets, + probe_timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + session.errors.append(f"Recording manifest probe failed: {exc}") + return None if manifest is None: session.errors.append("Recording manifest has no valid segments") return None diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py index dd3ba394..ede29c9e 100644 --- a/artemis/drivers/ios/wda.py +++ b/artemis/drivers/ios/wda.py @@ -106,6 +106,15 @@ def _sync_request( if isinstance(value, dict) and value.get("error"): message = value.get("message") or value["error"] raise RuntimeError(f"WebDriverAgent {method} {path} failed: {message}") + # WDA reports the live session in the outer /status envelope, not + # inside value — surface it so a foreign session is detectable. + if ( + path == "/status" + and isinstance(value, dict) + and isinstance(payload_out.get("sessionId"), str) + and payload_out["sessionId"] + ): + value["sessionId"] = payload_out["sessionId"] return value return payload_out @@ -116,9 +125,22 @@ async def _request( payload: dict[str, Any] | None = None, timeout: float | None = None, ) -> Any: - return await asyncio.to_thread( - self._sync_request, method, path, payload, timeout or self._timeout + request_task = asyncio.create_task( + asyncio.to_thread(self._sync_request, method, path, payload, timeout or self._timeout) ) + try: + return await asyncio.shield(request_task) + except asyncio.CancelledError: + # The blocking urllib call keeps running on its thread; drain it + # before propagating so a cancelled input cannot still land on the + # device after the caller (and its device lease) moved on. + try: + await request_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"WDA request drain failed while cancelling: {drain_error}") + raise # --- Session lifecycle --- @@ -128,9 +150,38 @@ async def status(self, timeout: float = 5.0) -> dict[str, Any] | None: except (RuntimeError, WdaUnavailableError, OSError) as error: logger.debug(f"WDA status probe failed for {self._base}: {error}") return None - return value if isinstance(value, dict) else {"value": value} + # Reject non-JSON/foreign bodies instead of wrapping them: a proxy or + # unrelated service answering on this port is not WebDriverAgent. + return value if isinstance(value, dict) else None - async def open_session(self) -> str: + async def device_info(self, timeout: float = 10.0) -> dict[str, Any]: + """GET /wda/device/info — available without a session. + + ``uuid`` here is ``identifierForVendor``, not the device UDID, so + identity checks must rely on ``name``/``isSimulator`` instead. + """ + value = await self._request("GET", "/wda/device/info", timeout=timeout) + if not isinstance(value, dict): + raise RuntimeError( + f"WebDriverAgent /wda/device/info returned a non-JSON response: {value!r:.300}" + ) + return value + + async def _create_owned_session(self) -> str: + """Status preflight, POST /session, and ID assignment as one unit.""" + if self._session_id: + return self._session_id + status = await self.status() + if status is None: + raise WdaUnavailableError(f"WebDriverAgent at {self._base} did not answer /status.") + active = status.get("sessionId") + if isinstance(active, str) and active: + raise RuntimeError( + "Refusing to replace the active WebDriverAgent session " + f"({active}), which this client does not own — POST /session " + "would kill it. Point ARTEMIS_IOS_WDA_URL at a dedicated WDA " + "server or close the existing session first." + ) payload = { "capabilities": { "alwaysMatch": {"platformName": "iOS"}, @@ -150,6 +201,34 @@ async def open_session(self) -> str: self._session_id = session_id return session_id + async def open_session(self) -> str: + """Create a WDA session, refusing to take over a foreign one. + + Cancellation-safe: when the caller is cancelled while creation is in + flight, the request is drained and any session it produced is closed + before the CancelledError propagates. + """ + create_task = asyncio.ensure_future(self._create_owned_session()) + try: + return await asyncio.shield(create_task) + except asyncio.CancelledError: + session_id = None + try: + session_id = await create_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"WDA session-create drain failed while cancelling: {drain_error}") + if session_id: + try: + await self.close_session() + except (OSError, ValueError, RuntimeError, TimeoutError) as close_error: + # Best-effort teardown of the orphaned session. + logger.debug( + f"WDA session close after cancelled open_session failed: {close_error}" + ) + raise + async def close_session(self) -> None: session_id, self._session_id = self._session_id, None if session_id is None: @@ -246,9 +325,7 @@ async def press_button(self, name: str) -> bool: """WDA hardware buttons: home, volumeUp, volumeDown, power.""" session = self._require_session() try: - await self._request( - "POST", f"/session/{session}/wda/pressButton", {"name": name} - ) + await self._request("POST", f"/session/{session}/wda/pressButton", {"name": name}) return True except WdaUnavailableError: # The transport died — propagate so callers don't misreport an @@ -289,17 +366,17 @@ def normalize_wda_url(raw: str) -> str: value = f"[{value}]" value = f"http://{value}" value = value.rstrip("/") - authority = value.split("://", 1)[1] - host_port = authority.split("/", 1)[0] + scheme, _, remainder = value.partition("://") + host_port, _, path = remainder.partition("/") # A port is present when ':' follows the host (or the IPv6 ']' bracket). has_port = ( - host_port.rsplit("]", 1)[-1].startswith(":") - if "]" in host_port - else ":" in host_port + host_port.rsplit("]", 1)[-1].startswith(":") if "]" in host_port else ":" in host_port ) - if not has_port: - value = f"{value}:{WDA_DEFAULT_PORT}" - return value + if has_port: + return value + # The default port belongs to the authority, ahead of any path suffix. + suffix = f"/{path}" if path else "" + return f"{scheme}://{host_port}:{WDA_DEFAULT_PORT}{suffix}" def wda_url_candidates( @@ -312,7 +389,11 @@ def wda_url_candidates( for raw in ( env_url if env_url is not None else os.environ.get(WDA_URL_ENV), env_host if env_host is not None else os.environ.get(WDA_HOST_ENV), - (f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" if ":" in tunnel_ip else f"{tunnel_ip}:{WDA_DEFAULT_PORT}") + ( + f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" + if ":" in tunnel_ip + else f"{tunnel_ip}:{WDA_DEFAULT_PORT}" + ) if tunnel_ip else None, f"127.0.0.1:{WDA_DEFAULT_PORT}", @@ -325,9 +406,7 @@ def wda_url_candidates( return candidates -async def probe_wda( - candidates: list[str], timeout: float = 5.0 -) -> WdaClient | None: +async def probe_wda(candidates: list[str], timeout: float = 5.0) -> WdaClient | None: """Return a client bound to the first endpoint that answers ``/status``.""" for url in candidates: client = WdaClient(url) @@ -357,6 +436,8 @@ def parse_wda_elements( def visit(entry: dict[str, Any]) -> None: rect = entry.get("rect") + if not isinstance(rect, dict): + rect = {} try: x = float(rect.get("x") or 0.0) y = float(rect.get("y") or 0.0) diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py index 79379517..14360229 100644 --- a/artemis/drivers/ios/xcode_driver.py +++ b/artemis/drivers/ios/xcode_driver.py @@ -30,6 +30,7 @@ from artemis.drivers.ios.discovery import ( clear_ios_simulator_cache, BOOTED_SIMULATOR_ID, + device_matches_identifier, parse_simctl_devices, parse_xcode_version, plist_to_json, @@ -131,7 +132,7 @@ async def _resolve_device(self) -> dict[str, Any]: await self._require_ios_host() devices = json.loads(await run_xcrun("simctl", "list", "devices", "--json")) available = parse_simctl_devices(devices) - if self._device_id == BOOTED_SIMULATOR_ID: + if self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: candidates = [device for device in available if device.get("state") == "Booted"] if len(candidates) != 1: raise ValueError( @@ -139,11 +140,15 @@ async def _resolve_device(self) -> dict[str, Any]: ) else: candidates = [ - device - for device in available - if device.get("udid", "").lower() == self._device_id.lower() + device for device in available if device_matches_identifier(device, self._device_id) ] - if len(candidates) != 1: + if len(candidates) > 1: + udids = sorted(str(d.get("udid") or "?") for d in candidates) + raise ValueError( + f"{len(candidates)} iOS simulators match {self._device_id!r} " + f"({', '.join(udids)}); use the simulator UDID instead." + ) + if not candidates: raise ValueError(f"Unavailable iOS simulator UDID: {self._device_id}") self._device_id = candidates[0]["udid"] return candidates[0] @@ -274,7 +279,7 @@ async def _synthesize(self, command: str = "", activation: str | None = None) -> return await self._bridge.call("DeviceInteractionSynthesize", arguments) def _require_connected(self) -> None: - if not self._session_key or self._device_id == BOOTED_SIMULATOR_ID: + if not self._session_key or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: raise RuntimeError("Connect the iOS simulator driver before interacting.") async def _capture(self) -> ScreenData: diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 9ba99f55..7215ecee 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -21,7 +21,6 @@ from typing import Annotated from artemis.config import initialize_llm_config -from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.sdk import Agent from artemis.sdk.builders import Builders @@ -68,8 +67,8 @@ async def run_batch_tasks( if explorer_pro_mode is not None: config_builder.with_explorer(pro_mode=explorer_pro_mode) if normalize_device_platform(platform, strict=False) == "ios": - config_builder.for_ios_simulator( - device_id=device_serial or BOOTED_SIMULATOR_ID, + config_builder.for_ios_device( + device_id=device_serial or "booted", workspace_path=ios_workspace, ) elif device_serial: diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index b628178b..5d0ab5c1 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -350,9 +350,7 @@ def run_command( if platform == DevicePlatform.IOS: if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": - raise typer.BadParameter( - "iOS support is local only; cloud mode targets Android." - ) + raise typer.BadParameter("iOS support is local only; cloud mode targets Android.") if locked_app_package: raise typer.BadParameter("--locked-app is unavailable for iOS tasks.") ensure_video_recording_available(with_video_recording_tools) diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 60a63779..d52c5349 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -570,7 +570,9 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action params=( ParamSpec( "key", - Literal["ENTER", "BACK", "HOME", "APP_SWITCH", "POWER", "VOLUME_UP", "VOLUME_DOWN"], + Literal[ + "ENTER", "BACK", "HOME", "APP_SWITCH", "POWER", "VOLUME_UP", "VOLUME_DOWN" + ], "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", ), ), diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 15749fd0..c6e82354 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -48,6 +48,11 @@ from artemis.runtime.adb_endpoint import normalize_device_platform from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries +# Placeholder iOS screen metrics for the lazy context — the driver replaces +# them with the real display size when it connects. +_LAZY_IOS_PLACEHOLDER_WIDTH = 1206 +_LAZY_IOS_PLACEHOLDER_HEIGHT = 2622 + def configure_stdio_mode() -> None: """Applies process-wide settings required when serving MCP over stdio. @@ -128,22 +133,25 @@ def _get_controller(device_serial: str | None = None, target_platform: str | Non the native session opens lazily on first use. """ global _GLOBAL_CONTROLLER, _CONTROLLERS - target_serial = ( - device_serial or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL") - ) - if normalize_device_platform(target_platform, strict=False) == "ios": - cache_key = f"ios:{target_serial or 'booted'}" + is_ios_target = normalize_device_platform(target_platform, strict=False) == "ios" + # ADB_DEVICE_SERIAL is Android-only and must never leak into an iOS target. + target_serial = device_serial or os.environ.get("ARTEMIS_DEVICE_ID") + if not is_ios_target: + target_serial = target_serial or os.environ.get("ADB_DEVICE_SERIAL") + if is_ios_target: + cache_key = f"ios:{target_serial or BOOTED_SIMULATOR_ID}" if cache_key in _CONTROLLERS: return _CONTROLLERS[cache_key] - logger.info("Initializing lazy iOS simulator controller...") + logger.info("Initializing lazy iOS device controller...") ctx = ArtemisContext( trace_id="mcp-session", device=DeviceContext( host_platform=platform.os_type.name, mobile_platform=DevicePlatform.IOS, device_id=target_serial or BOOTED_SIMULATOR_ID, - device_width=1206, - device_height=2622, + # Placeholder until the driver reports real screen metrics. + device_width=_LAZY_IOS_PLACEHOLDER_WIDTH, + device_height=_LAZY_IOS_PLACEHOLDER_HEIGHT, ), ) controller = UnifiedMobileController(ctx=ctx) diff --git a/artemis/runtime/ios_device_pool.py b/artemis/runtime/ios_device_pool.py index 2077bf2a..3a6576e0 100644 --- a/artemis/runtime/ios_device_pool.py +++ b/artemis/runtime/ios_device_pool.py @@ -25,6 +25,7 @@ from __future__ import annotations from artemis.drivers.ios.discovery import ( + BOOTED_SIMULATOR_ID, is_physical_ios, list_core_devices, list_core_devices_sync, @@ -71,32 +72,48 @@ def _owner_fields(owner) -> dict: } -def _match_status( - devices: list[DeviceStatus], requested_serial: str -) -> DeviceStatus | None: - """Find the status matching a requested UDID or device name.""" +def _match_statuses(devices: list[DeviceStatus], requested_serial: str) -> list[DeviceStatus]: + """All statuses matching a requested UDID or device name.""" needle = str(requested_serial).lower() - return next( - ( - d - for d in devices - if d.serial.lower() == needle or (d.model or "") == requested_serial - ), - None, + return [d for d in devices if d.serial.lower() == needle or (d.model or "") == requested_serial] + + +def _match_or_reject( + devices: list[DeviceStatus], requested_serial: str +) -> tuple[DeviceStatus | None, str | None]: + """Return (match, None), (None, ambiguity rejection), or (None, None).""" + matches = _match_statuses(devices, requested_serial) + if len(matches) > 1: + serials = sorted(d.serial for d in matches) + return None, ( + f"{len(matches)} iOS devices match '{requested_serial}' " + f"({', '.join(serials)}); pass the device UDID instead." + ) + return (matches[0] if matches else None), None + + +def _booted_rejection(raw: list[dict] | None, requested_serial: str) -> str | None: + """``booted`` needs exactly one booted simulator to resolve safely.""" + if str(requested_serial).strip().lower() != BOOTED_SIMULATOR_ID: + return None + if raw is None: + # Enumeration cannot answer: fail open to the driver's own check. + return None + booted = [d for d in raw if d.get("state") == "Booted"] + if len(booted) == 1: + return None + return ( + f"'{BOOTED_SIMULATOR_ID}' requires exactly one booted iOS simulator " + f"({len(booted)} found); pass an explicit simulator UDID instead." ) def _state_rejection(match: DeviceStatus, requested_serial: str) -> str | None: """The shared explicit-target state check for both validator variants.""" - acceptable = ( - ACCEPTABLE_STATES if match.is_emulator else PHYSICAL_ACCEPTABLE_STATES - ) + acceptable = ACCEPTABLE_STATES if match.is_emulator else PHYSICAL_ACCEPTABLE_STATES if match.state not in acceptable: kind = "simulator" if match.is_emulator else "physical device" - return ( - f"iOS {kind} '{requested_serial}' is in state '{match.state}' " - "and cannot be used." - ) + return f"iOS {kind} '{requested_serial}' is in state '{match.state}' and cannot be used." return None @@ -189,10 +206,14 @@ async def validate_explicit_serial_async(self, requested_serial: str) -> str | N core = await list_core_devices() if raw is None and core is None: return None - devices = self._build_statuses(raw or []) + self._build_physical_statuses( - core or [] - ) - match = _match_status(devices, requested_serial) + if str(requested_serial).strip().lower() == BOOTED_SIMULATOR_ID: + # The documented selector: accepted by exactly one booted + # simulator, rejected otherwise, fail-open without enumeration. + return _booted_rejection(raw, requested_serial) + devices = self._build_statuses(raw or []) + self._build_physical_statuses(core or []) + match, ambiguity = _match_or_reject(devices, requested_serial) + if ambiguity is not None: + return ambiguity if match is None: # A failed enumeration cannot prove the serial is absent — # defer to the driver's own resolution rather than reject. @@ -214,10 +235,12 @@ def validate_explicit_serial(self, requested_serial: str) -> str | None: core = list_core_devices_sync() if raw is None and core is None: return None - devices = self._build_statuses(raw or []) + self._build_physical_statuses( - core or [] - ) - match = _match_status(devices, requested_serial) + if str(requested_serial).strip().lower() == BOOTED_SIMULATOR_ID: + return _booted_rejection(raw, requested_serial) + devices = self._build_statuses(raw or []) + self._build_physical_statuses(core or []) + match, ambiguity = _match_or_reject(devices, requested_serial) + if ambiguity is not None: + return ambiguity if match is None: # An enumeration that could not answer must not reject the serial. if raw is None or core is None or not devices: @@ -240,6 +263,19 @@ async def select_device_async(self, preferred_serial: str | None = None) -> str target "booted". """ if preferred_serial: + if str(preferred_serial).strip().lower() != BOOTED_SIMULATOR_ID: + return preferred_serial + # "booted" is a documented selector: pin it to the unique booted + # simulator when enumeration can answer. Zero/multiple booted sims + # keep the literal so the driver's resolution reports the + # explicit-UDID guidance; a failed enumeration fails open the same + # way. Physical hardware is still never auto-selected. + booted_devices = await self.try_list_devices_async() + if booted_devices is None: + return preferred_serial + booted_only = [d for d in booted_devices if d.is_emulator and d.state == "device"] + if len(booted_only) == 1: + return booted_only[0].serial return preferred_serial devices = await self.try_list_devices_async() or [] simulators = [d for d in devices if d.is_emulator] diff --git a/artemis/runtime/ios_observation.py b/artemis/runtime/ios_observation.py new file mode 100644 index 00000000..60ce665a --- /dev/null +++ b/artemis/runtime/ios_observation.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared iOS observation lifecycle for one-shot native captures. + +``mobile_get_device_state`` and the device smoke check both need a screen +capture from an iOS driver that was constructed lazily — no lease, no native +session. This helper performs the complete, cancellation-safe lifecycle: +resolve the device identity, take the real iOS-scoped lease, connect, capture, +then disconnect and release in that order, even when the caller is cancelled. +""" + +import asyncio +from typing import Any + +from artemis.controllers.device_controller import ScreenDataResponse +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.device_lock import DeviceExecutionLock +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +_DISCONNECT_ERRORS = (OSError, ValueError, RuntimeError, TimeoutError) + + +async def observe_ios_controller(controller: Any) -> ScreenDataResponse: + """Capture screen data from an unconnected iOS controller. + + Acquires the ``ios``-scoped device lease before any native setup, connects + the driver, captures, then always disconnects (when connect was attempted) + and releases the lease — including on cancellation, where cleanup is + shielded from the caller's CancelledError before it propagates. + """ + driver = controller._driver + resolved = await driver.resolve_device() + controller.ctx.device.device_id = resolved + lease = DeviceExecutionLock( + resolved, + description="Artemis iOS observation", + ingress="mcp", + concurrency_mode="per_device", + lock_scope=IOS_LOCK_SCOPE, + ) + connect_attempted = False + + async def _cleanup() -> None: + try: + if connect_attempted: + try: + await driver.disconnect() + except _DISCONNECT_ERRORS as exc: + logger.warning(f"iOS observation native disconnect failed: {exc}") + finally: + lease.release() + + acquire_task = asyncio.create_task(asyncio.to_thread(lease.acquire, blocking=False)) + try: + try: + # A cancelled waiter must still drain the acquisition thread: if the + # lease landed concurrently, skipping the drain would leak it. + await asyncio.shield(acquire_task) + except asyncio.CancelledError: + try: + await acquire_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"iOS observation lease drain failed while cancelling: {drain_error}") + raise + connect_attempted = True + await driver.connect() + data = await controller.get_screen_data() + device = controller.ctx.device + device.device_width = data.width + device.device_height = data.height + return data + finally: + cleanup = asyncio.ensure_future(_cleanup()) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + await cleanup + raise diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index a3b43264..85cb25c4 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -182,9 +182,7 @@ async def _init_internal( # Android read-only ADB probes have no iOS equivalent yet; native # simctl recording is supported and honors the configured flag. self._config = self._config.model_copy(update={"disable_device_probes": True}) - publish_startup_progress( - "device_ready", "iOS device selected", session_id=self._session_id - ) + publish_startup_progress("device_ready", "iOS device selected", session_id=self._session_id) asyncio.create_task(self._prewarm_llm_connections(api_key)) self._initialized = True return True diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 9627cfd7..3743b2a2 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -319,8 +319,14 @@ def _probe_video_segment_cv2(video_path: Path) -> dict[str, float | int]: } -async def probe_video_segment(video_path: Path) -> dict[str, float | int]: - """Read duration and coded dimensions for a finalized segment.""" +async def probe_video_segment( + video_path: Path, *, timeout_seconds: float | None = None +) -> dict[str, float | int]: + """Read duration and coded dimensions for a finalized segment. + + ``timeout_seconds`` bounds the owned ffprobe child: on timeout or + cancellation the subprocess is killed and drained before propagating. + """ try: process = await asyncio.create_subprocess_exec( get_ffprobe_path(), @@ -336,7 +342,23 @@ async def probe_video_segment(video_path: Path) -> dict[str, float | int]: ) except FileNotFoundError: return await asyncio.to_thread(_probe_video_segment_cv2, video_path) - stdout, _stderr = await process.communicate() + if timeout_seconds is None: + # Default callers keep the original unbounded communicate behaviour. + stdout, _stderr = await process.communicate() + else: + try: + stdout, _stderr = await asyncio.wait_for(process.communicate(), timeout_seconds) + except (TimeoutError, asyncio.CancelledError): + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + try: + await process.communicate() + except (OSError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"ffprobe drain after kill failed: {drain_error}") + raise if process.returncode != 0: return {} try: @@ -397,6 +419,7 @@ async def write_recording_manifest( output_dir: Path, mp4_paths: list[Path], segment_offsets: dict[Path, float] | None = None, + probe_timeout_seconds: float | None = None, ) -> Path | None: """Write the browser playlist used for orientation-aware playback. @@ -418,7 +441,13 @@ async def write_recording_manifest( for path in mp4_paths: if not path.exists(): continue - metadata = await probe_video_segment(path) + # Forward the timeout only when supplied so default callers keep the + # historical probe signature. + metadata = await ( + probe_video_segment(path, timeout_seconds=probe_timeout_seconds) + if probe_timeout_seconds is not None + else probe_video_segment(path) + ) duration = float(metadata.get("duration", 0)) width = int(metadata.get("width", 0)) height = int(metadata.get("height", 0)) diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py index 6a90d00c..ba398d20 100644 --- a/artemis/utils/visualization.py +++ b/artemis/utils/visualization.py @@ -556,8 +556,10 @@ def _resolve_coordinates(coord: Any, width: int, height: int) -> tuple[int, int] if 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0: px = int(round(x * width)) py = int(round(y * height)) - elif 0 <= x <= 1000 and 0 <= y <= 1000 and ( - width > 1000 or height > 1000 or x > width or y > height + elif ( + 0 <= x <= 1000 + and 0 <= y <= 1000 + and (width > 1000 or height > 1000 or x > width or y > height) ): # Normalized 0-1000 space — also on small (logical-size) iOS # screenshots where the values exceed the pixel bounds anyway. diff --git a/docs/ios.md b/docs/ios.md index 50aa224f..eab43cfb 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -138,6 +138,23 @@ endpoint on the CoreDevice tunnel address and `127.0.0.1:8100` (for `xcodebuild test-without-building` (the canonical WDA session — a bare runner app launch does not start the HTTP server) +Endpoint ownership limits: an explicitly configured or forwarded WDA +endpoint (`ARTEMIS_IOS_WDA_URL`/`ARTEMIS_IOS_WDA_HOST`) must belong to the +selected physical device. Before opening a session, Artemis reads WDA's +sessionless `GET /wda/device/info` and refuses to connect when it reports a +simulator or a device name different from the selected device — the name +corroborates the selection but is not a cryptographic proof of unique-device +identity (WDA's `uuid` is `identifierForVendor`, not the device UDID). Artemis +also refuses to replace a WDA session that belongs to another client, since +WDA's `POST /session` unconditionally kills the active session; close the +existing session first. + +Protocol details verified against WebDriverAgent 16.14.0 (commit `d177824`, +checked 2026-10-07): +[FBSessionCommands.m](https://github.com/appium/WebDriverAgent/blob/d177824/WebDriverAgentLib/Commands/FBSessionCommands.m), +[FBResponsePayload.m](https://github.com/appium/WebDriverAgent/blob/d177824/WebDriverAgentLib/Routing/FBResponsePayload.m), +[FBCustomCommands.m](https://github.com/appium/WebDriverAgent/blob/d177824/WebDriverAgentLib/Commands/FBCustomCommands.m). + Simulator bridging note: `xcrun mcpbridge` is spawned with a minimal environment; `DEVELOPER_DIR` and `MCP_XCODE_PID` are forwarded when set so a specific Xcode toolchain can be pinned. @@ -219,17 +236,38 @@ The generic builder also accepts imported from `artemis.context`. Supplying only a `device_serial` without an iOS configuration retains the existing Android selection behavior. +The dependency-free remote client submits iOS tasks to an iOS-capable host — +the remote host needs macOS, Xcode, and (for hardware) WDA set up as above; +`ios_workspace` is an optional host-side path, not resolved locally: + +```python +from artemis_client import ArtemisClient + +client = ArtemisClient( + base_url="http://mac-host:8000", + device_serial="", + platform="ios", + # optional: ios_workspace="/abs/host/path/MyApp.xcworkspace", +) +result = await client.run("Open Settings and inspect the General page") +``` + +iOS submissions first check `GET /api/v1/capabilities` and refuse to POST +when the host does not advertise `platform.ios` — upgrade the host instead +of silently running on Android. + ## Daemon, web console, and batch submission Without `--standalone`, `artemis run --platform ios` forwards `platform`/`ios_workspace`/`device_serial` to the running Artemis Daemon, which -queues the task against the simulator under the shared `ios` lock scope and -spawns the worker with `--platform ios` — no ADB endpoint or Android readiness +queues the task against the selected iOS device under the shared `ios` +lock scope and spawns the worker with `--platform ios` — no ADB endpoint or Android readiness probe is involved. `artemis batch` accepts the same `--platform`, `--device-serial`, and `--ios-workspace` flags for goal lists. -The Admin Console `/api/run` accepts `platform: "ios"`, a simulator UDID in -`device_serial`, and an optional `ios_workspace`; `/api/devices` lists Android +The Admin Console `/api/run` accepts `platform: "ios"`, an iOS device UDID +(simulator or paired physical hardware) in `device_serial`, and an optional +`ios_workspace`; `/api/devices` lists Android devices and iOS devices (simulators and paired physical hardware) together, each tagged with its `platform`. The live screen view (`/api/stream/device-live`) streams frames captured with `simctl io @@ -277,7 +315,7 @@ instead of acquiring a second one. | Tap, long press, and swipe | Native synthesized touch events | WDA W3C pointer actions | | Text entry | Native keyboard synthesis with `clear_exist=false` | WDA `/wda/keys`, `clear_exist=false` | | Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | WDA `/wda/keys` (Enter), `/wda/homescreen`, and `/wda/pressButton` | -| App install, launch, and terminate | `simctl` with simulator `.app` bundles | `devicectl` with signed `.app`/`.ipa`; terminate uses the launch PID or a live process scan | +| App install, launch, and terminate | `simctl` with simulator `.app` bundles | `devicectl` with signed `.app`/`.ipa`; terminate re-verifies the running executable against the installed app URL (a cached launch PID is only a hint — recycled PIDs are never trusted) | | Screen recording | `simctl io recordVideo` (VFR H.264) | `devicectl` screenshot polling assembled to timestamped MP4 | The native hierarchy is normalized into the element tree used by Artemis's diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index b986b031..cad7d71b 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -42,7 +42,6 @@ load_dotenv(os.path.join(PROJECT_ROOT, ".env")) from artemis.runtime import trace_store -from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.runtime.adb_endpoint import normalize_device_platform from mcp_server.notifiers import notify from mcp_server.utils import device_utils @@ -178,8 +177,8 @@ async def run_task( if is_ios: # The iOS driver resolves "booted" against `simctl` at connect # time; an explicit UDID was already validated at admission. - target_serial = device_serial or BOOTED_SIMULATOR_ID - print(f"✅ Targeting iOS simulator: '{target_serial}'.") + target_serial = device_serial or "booted" + print(f"✅ Targeting iOS device: '{target_serial}'.") elif device_serial: target_serial = device_serial if connected_devices and device_serial not in connected_devices: @@ -232,8 +231,8 @@ async def run_task( config_builder.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) if is_ios: - config_builder.for_ios_simulator( - device_id=target_serial or BOOTED_SIMULATOR_ID, + config_builder.for_ios_device( + device_id=target_serial or "booted", workspace_path=ios_workspace, ) elif target_serial: @@ -453,7 +452,7 @@ async def run_task( "--platform", default="android", choices=["android", "ios"], - help="Target platform ('android' default; 'ios' targets a Simulator via Xcode)", + help="Target platform ('android' default; 'ios' targets simulators or physical devices via Xcode/devicectl)", ) parser.add_argument( "--ios-workspace", diff --git a/mcp_server/tools/device_state.py b/mcp_server/tools/device_state.py index df8860dd..916f6cda 100644 --- a/mcp_server/tools/device_state.py +++ b/mcp_server/tools/device_state.py @@ -14,11 +14,14 @@ """MCP Tool: mobile_get_device_state.""" +import asyncio import base64 import os from mcp_server.base import mcp from artemis.mcp.adb_server import _get_controller +from artemis.runtime.adb_endpoint import normalize_device_platform +from artemis.runtime.ios_observation import observe_ios_controller from mcp_server.utils import env_utils from artemis.utils.ocr_xml_fusion import ( fuse_ocr_with_xml, @@ -55,23 +58,31 @@ async def mobile_get_device_state( device UDID (simulator or paired physical device); omit it to use the single booted simulator. """ try: - controller = _get_controller( - device_serial=device_serial, - **({"target_platform": platform} if platform else {}), - ) + # Strict at the public boundary: a misspelled platform must never fall + # through to the Android controller. None/empty stays Android. + canonical_platform = normalize_device_platform(platform) + except ValueError as e: + return f"Error: Failed to initialize/lock device controller: {e}" + try: + if canonical_platform == "ios": + # iOS resolution enumerates simctl/devicectl synchronously; offload + # the blocking factory so the MCP loop stays responsive. + controller = await asyncio.to_thread( + _get_controller, device_serial=device_serial, target_platform="ios" + ) + else: + controller = _get_controller(device_serial=device_serial) is_ios = str(getattr(controller.ctx.device, "mobile_platform", "android")).lower() == "ios" except Exception as e: return f"Error: Failed to initialize/lock device controller: {e}" try: if is_ios: - # The native Xcode session opens lazily; connect before observing. - await controller._driver.connect() - device_data = await controller.get_screen_data() - if is_ios: - # Real capture dims supersede the provisional ctx placeholders. - controller.ctx.device.device_width = device_data.width - controller.ctx.device.device_height = device_data.height + # Owns the whole one-shot lifecycle — device lease, native connect, + # capture, disconnect, release — all cancellation-safe. + device_data = await observe_ios_controller(controller) + else: + device_data = await controller.get_screen_data() device_width = controller.ctx.device.device_width device_height = controller.ctx.device.device_height latest_screenshot_b64 = device_data.base64 diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index dae86210..305bac52 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -475,6 +475,59 @@ async def _device_smoke_test( return await smoke_test_device(device_serial, platform=platform) +def _ios_requested_device( + ios_result: ProbeResult | None, requested_device: str | None +) -> dict[str, Any] | None: + """Canonical iOS selection from probe metadata, or None when ambiguous. + + Mirrors the driver/pool contract: UDID compare is case-insensitive, names + match exactly, exactly one match is required, and physical hardware is + never picked implicitly. Returns the selected entry as + ``{serial, name, state, is_emulator}``. + """ + metadata = ios_result.metadata if ios_result else {} + usable_sims = [ + d + for d in (metadata.get("simulators") or []) + if d.get("udid") and d.get("state") in ("Booted", "Shutdown") + ] + physical = [d for d in (metadata.get("connected_physical_devices") or []) if d.get("udid")] + booted = [d for d in usable_sims if d.get("state") == "Booted"] + + def _entry(device: dict[str, Any], *, is_emulator: bool) -> dict[str, Any]: + state = device.get("state") or "" + return { + "serial": str(device.get("udid")), + "name": device.get("name"), + "state": ("device" if not is_emulator or state == "Booted" else state or "unknown"), + "is_emulator": is_emulator, + } + + if requested_device: + if requested_device.strip().lower() == "booted": + return _entry(booted[0], is_emulator=True) if len(booted) == 1 else None + needle = requested_device.lower() + matches = [ + (d, True) + for d in usable_sims + if str(d.get("udid")).lower() == needle or d.get("name") == requested_device + ] + matches += [ + (d, False) + for d in physical + if str(d.get("udid")).lower() == needle or d.get("name") == requested_device + ] + if len(matches) != 1: + return None + device, is_emulator = matches[0] + return _entry(device, is_emulator=is_emulator) + if len(booted) == 1: + return _entry(booted[0], is_emulator=True) + if len(usable_sims) == 1: + return _entry(usable_sims[0], is_emulator=True) + return None + + async def _run_device_probe( adb_result: ProbeResult | None, requested_device: str | None, @@ -482,29 +535,25 @@ async def _run_device_probe( platform: str = "android", ) -> dict[str, Any]: if platform == "ios": - simulators = (ios_result.metadata.get("simulators") if ios_result else None) or [] - physical = ( - ios_result.metadata.get("connected_physical_devices") if ios_result else None - ) or [] - ready_udids = [str(d.get("udid")) for d in simulators if d.get("udid")] + [ - str(d.get("udid")) for d in physical if d.get("udid") - ] - if requested_device and requested_device not in ready_udids: + selected = _ios_requested_device(ios_result, requested_device) + if selected is None: + if requested_device: + return _probe_unavailable( + requested_device, + f"requested iOS device '{requested_device}' does not resolve to exactly " + "one usable iOS device; pass the simulator or paired physical device " + "UDID listed in the ios_simulators check facts", + ) return _probe_unavailable( - requested_device, - f"requested iOS device '{requested_device}' is not available " - "(simulators plus paired+connected physical devices); nothing to probe", + None, + "no single unambiguous iOS device is available; pass device_serial with an " + "explicit simulator or paired physical device UDID", ) - if not ready_udids: - return _probe_unavailable(None, "no available iOS device found; nothing to probe") - # Auto-pick only a lone simulator — never silently choose hardware. - sim_udids = [str(d.get("udid")) for d in simulators if d.get("udid")] - serial = requested_device or (sim_udids[0] if len(sim_udids) == 1 else None) try: - return await _device_smoke_test(serial, platform="ios") + return await _device_smoke_test(selected["serial"], platform="ios") except Exception as exc: return _probe_unavailable( - serial, f"iOS smoke test raised {exc.__class__.__name__}: {exc}" + selected["serial"], f"iOS smoke test raised {exc.__class__.__name__}: {exc}" ) devices = (adb_result.metadata.get("devices") if adb_result else None) or [] @@ -1054,14 +1103,7 @@ def _requested_device_ready( ios_result = _find(results, "ios_simulators") if ios_result is None: return False - simulators = ios_result.metadata.get("simulators") or [] - if any( - d.get("udid") == requested_device and d.get("state") in ("Booted", "Shutdown") - for d in simulators - ): - return True - physical = ios_result.metadata.get("connected_physical_devices") or [] - return any(d.get("udid") == requested_device for d in physical) + return _ios_requested_device(ios_result, requested_device) is not None adb_result = _find(results, "android_adb") if adb_result is None: return False @@ -1301,9 +1343,24 @@ async def mobile_diagnose( accessibility_helper=accessibility_helper, platform=platform_name, ) - device = _compact_device(report) - if device is not None and accessibility_helper is not None: - device["accessibility_helper"] = accessibility_helper + if platform_name == "ios": + # The response device is the iOS target diagnose would smoke/probe, + # not the Android active_device from the readiness report. + selected_ios = _ios_requested_device(_find(results, "ios_simulators"), requested_device) + device = ( + { + "serial": selected_ios["serial"], + "model": selected_ios["name"], + "platform": "ios", + "is_emulator": selected_ios["is_emulator"], + } + if selected_ios + else None + ) + else: + device = _compact_device(report) + if device is not None and accessibility_helper is not None: + device["accessibility_helper"] = accessibility_helper env_file = host.metadata.get("env_file") return { "verdict": verdict, diff --git a/packages/artemis-client/src/artemis_client/client.py b/packages/artemis-client/src/artemis_client/client.py index d1d3e2d4..b4525583 100644 --- a/packages/artemis-client/src/artemis_client/client.py +++ b/packages/artemis-client/src/artemis_client/client.py @@ -38,6 +38,8 @@ def request( #: Pro-profile Explorer perception versions accepted by ``/api/run`` (``explorer_mode``). ExplorerMode = Literal["flash", "pro", "ultra"] EXPLORER_MODES: tuple[str, ...] = ("flash", "pro", "ultra") +#: Device platforms accepted by ``/api/run`` (``platform``). +PLATFORMS: tuple[str, ...] = ("android", "ios") def _normalize_choice(value: str | None, name: str, choices: tuple[str, ...]) -> str | None: @@ -84,9 +86,15 @@ def __init__( concurrency_mode: str = "per_device", max_concurrency: int | None = None, standalone: bool = False, + platform: str | None = None, + ios_workspace: str | None = None, ) -> None: if poll_interval <= 0: raise ValueError("poll_interval must be greater than zero") + normalized_platform = _normalize_choice(platform, "platform", PLATFORMS) + workspace = str(ios_workspace).strip() if ios_workspace is not None else "" + if workspace and normalized_platform != "ios": + raise ValueError("ios_workspace requires platform='ios'") resolved_base_url = base_url or os.environ.get("ARTEMIS_BASE_URL") if not resolved_base_url: daemon_host = os.environ.get("ARTEMIS_DAEMON_HOST", "127.0.0.1") @@ -100,6 +108,9 @@ def __init__( self.concurrency_mode = str(concurrency_mode).strip().lower() self.max_concurrency = max_concurrency self.standalone = standalone + self._platform = normalized_platform + # Host-side path only: it is never resolved against the local disk. + self._ios_workspace = workspace or None self._transport = transport or JsonTransport( resolved_base_url, token=token, @@ -186,6 +197,8 @@ async def submit( task_id: str | None = None, verification_level: VerificationLevel | None = None, explorer_mode: ExplorerMode | None = None, + platform: str | None = None, + ios_workspace: str | None = None, options: Mapping[str, Any] | None = None, ) -> TaskHandle: """Submit one task and return immediately after scheduler admission. @@ -205,6 +218,13 @@ async def submit( verification_level, "verification_level", VERIFICATION_LEVELS ) resolved_mode = _normalize_choice(explorer_mode, "explorer_mode", EXPLORER_MODES) + resolved_platform = _normalize_choice(platform, "platform", PLATFORMS) or self._platform + call_workspace = str(ios_workspace).strip() if ios_workspace is not None else "" + if call_workspace and resolved_platform != "ios": + raise ValueError("ios_workspace requires platform='ios'") + resolved_workspace = call_workspace or ( + self._ios_workspace if resolved_platform == "ios" else None + ) if task_id is None: resolved_task_id = str(uuid.uuid4()) else: @@ -229,10 +249,22 @@ async def submit( "conversation_id": conversation_id, "verification_level": resolved_level, "explorer_mode": resolved_mode, + "platform": resolved_platform, + "ios_workspace": resolved_workspace, "options": dict(options) if options is not None else None, } payload.update({key: value for key, value in optional_values.items() if value is not None}) + if resolved_platform == "ios": + # A legacy host silently drops unknown platform fields — refuse + # before POSTing unless it advertises iOS wire support. + advertised = await self.capabilities() + if not advertised.supports("platform.ios"): + raise TaskRejectedError( + "This Artemis host does not advertise 'platform.ios' " + "support; upgrade the host before submitting iOS tasks." + ) + response = self._mapping( await self._request("POST", "/api/run", json_body=payload), endpoint="/api/run", @@ -306,6 +338,8 @@ async def run( task_id: str | None = None, verification_level: VerificationLevel | None = None, explorer_mode: ExplorerMode | None = None, + platform: str | None = None, + ios_workspace: str | None = None, options: Mapping[str, Any] | None = None, timeout: float = 1800.0, poll_interval: float | None = None, @@ -323,6 +357,8 @@ async def run( task_id=task_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, options=options, ) return await self.wait_for_task( @@ -341,6 +377,8 @@ async def run_task(self, task: Any, **overrides: Any) -> TaskResult: "device_serial": getattr(task, "device_serial", None) or getattr(task, "device_id", None), "locked_app_package": getattr(task, "locked_package", None), + "platform": getattr(task, "platform", None), + "ios_workspace": getattr(task, "ios_workspace", None), } values.update(overrides) return await self.run(goal, **values) diff --git a/packages/artemis-client/src/artemis_client/models.py b/packages/artemis-client/src/artemis_client/models.py index e9c4e2d4..05378a21 100644 --- a/packages/artemis-client/src/artemis_client/models.py +++ b/packages/artemis-client/src/artemis_client/models.py @@ -52,6 +52,24 @@ def _device_from_payload(payload: Mapping[str, Any]) -> str | None: return None +def _platform_from_payload(payload: Mapping[str, Any]) -> str: + """Normalized platform ('android'|'ios'|...) from raw or device_info.""" + direct = _string(payload.get("platform")) + if direct: + return direct.lower() + device_info = payload.get("device_info") + if isinstance(device_info, str): + try: + device_info = json.loads(device_info) + except (TypeError, ValueError, json.JSONDecodeError): + device_info = None + if isinstance(device_info, Mapping): + nested = _string(device_info.get("mobile_platform")) + if nested: + return nested.lower() + return "android" + + @dataclass(frozen=True, slots=True) class TaskHandle: """A task accepted by the remote Artemis scheduler.""" @@ -66,6 +84,11 @@ def session_id(self) -> str: """Compatibility alias for servers that call a task a session.""" return self.task_id + @property + def platform(self) -> str: + """Recorded device platform; defaults to 'android' when absent.""" + return _platform_from_payload(self.raw) + @classmethod def from_payload(cls, payload: Mapping[str, Any]) -> TaskHandle: task_id = _string(payload.get("task_id") or payload.get("session_id") or payload.get("id")) @@ -111,6 +134,11 @@ def done(self) -> bool: def succeeded(self) -> bool: return self.status in SUCCESS_TASK_STATUSES + @property + def platform(self) -> str: + """Recorded device platform; defaults to 'android' when absent.""" + return _platform_from_payload(self.raw) + @classmethod def from_payload( cls, @@ -160,6 +188,11 @@ class Device: busy: bool = False raw: Mapping[str, Any] = field(default_factory=dict, repr=False, compare=False) + @property + def platform(self) -> str: + """Recorded device platform; defaults to 'android' when absent.""" + return _platform_from_payload(self.raw) + @classmethod def from_payload(cls, payload: Mapping[str, Any]) -> Device: serial = _string( diff --git a/packages/artemis-client/tests/test_client.py b/packages/artemis-client/tests/test_client.py index a5480a7a..c3cceb63 100644 --- a/packages/artemis-client/tests/test_client.py +++ b/packages/artemis-client/tests/test_client.py @@ -15,9 +15,12 @@ from artemis_client import ( ArtemisClient, + Device, NotFoundError, ProtocolError, + TaskHandle, TaskRejectedError, + TaskResult, TaskTimeoutError, ) @@ -285,5 +288,186 @@ async def test_stop_targets_one_session(self) -> None: ) +class IosPlatformSubmissionTests(unittest.IsolatedAsyncioTestCase): + """Platform/workspace forwarding and iOS capability preflight.""" + + CAPABLE = { + "api_version": "1", + "features": ["tasks.submit", "platform.ios"], + } + + def _client(self, **kwargs) -> ArtemisClient: + self.transport = FakeTransport() + return ArtemisClient( + "https://artemis.example.test", + poll_interval=0.001, + transport=self.transport, + **kwargs, + ) + + @staticmethod + def _admitted(platform=None): + task = {"session_id": "task-1", "status": "pending"} + if platform: + task["platform"] = platform + return {"status": "started", "tasks": [task]} + + async def test_ios_submit_preflights_capabilities_then_posts_platform(self): + client = self._client(device_serial="SIM-UDID", platform="ios") + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + + handle = await client.submit("Open Settings") + + get_call, post_call = self.transport.calls + self.assertEqual(get_call[0], "GET") + self.assertEqual(post_call[0], "POST") + body = post_call[2] + self.assertEqual(body["platform"], "ios") + self.assertNotIn("ios_workspace", body) + self.assertEqual(handle.platform, "ios") + + async def test_android_submit_sends_no_platform_and_no_preflight(self): + client = self._client() + self.transport.add("POST", "/api/run", self._admitted()) + + await client.submit("Open Settings") + + self.assertEqual(len(self.transport.calls), 1) + method, path, body = self.transport.calls[0] + self.assertEqual((method, path), ("POST", "/api/run")) + self.assertNotIn("platform", body) + self.assertNotIn("ios_workspace", body) + + async def test_ios_workspace_is_forwarded_only_for_ios(self): + client = self._client(platform="ios", ios_workspace="/host/MyApp.xcworkspace") + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + + await client.submit("Open Settings") + + body = self.transport.calls[1][2] + self.assertEqual(body["platform"], "ios") + self.assertEqual(body["ios_workspace"], "/host/MyApp.xcworkspace") + + # A blank per-call token must inherit, never erase, the ctor choice. + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + await client.submit("Open Settings", platform=" ") + self.assertEqual(self.transport.calls[3][2]["platform"], "ios") + + def test_constructor_workspace_requires_ios_platform(self): + with self.assertRaises(ValueError): + self._client(ios_workspace="/host/MyApp.xcworkspace") + with self.assertRaises(ValueError): + self._client(platform="android", ios_workspace="/host/MyApp.xcworkspace") + + async def test_workspace_on_non_ios_call_rejects_before_transport(self): + client = self._client() + with self.assertRaises(ValueError): + await client.submit("goal", ios_workspace="/host/ws.xcworkspace") + self.assertEqual(self.transport.calls, []) + + async def test_invalid_platform_rejects_before_transport(self): + client = self._client() + with self.assertRaises(ValueError): + await client.submit("goal", platform="tvos") + self.assertEqual(self.transport.calls, []) + + async def test_legacy_server_blocks_ios_before_posting(self): + client = self._client(platform="ios") + self.transport.add("GET", "/api/v1/capabilities", NotFoundError(404, "not implemented")) + + with self.assertRaises(TaskRejectedError): + await client.submit("Open Settings") + + methods = [call[0] for call in self.transport.calls] + self.assertEqual(methods, ["GET"]) # no POST /api/run attempted + + async def test_per_call_platform_override_beats_constructor_default(self): + client = self._client(platform="android") + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + + await client.submit("goal", platform="ios") + + self.assertEqual(self.transport.calls[1][2]["platform"], "ios") + + async def test_run_and_run_task_forward_platform(self): + task_id = "00000000-0000-4000-8000-000000000199" + client = self._client() + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add( + "POST", + "/api/run", + { + "status": "started", + "tasks": [{"session_id": task_id, "status": "pending", "platform": "ios"}], + }, + ) + self.transport.add( + "GET", + f"/api/sessions/{task_id}", + {"session_id": task_id, "status": "completed"}, + ) + task = type( + "T", + (), + { + "goal": "Do it", + "platform": "ios", + "ios_workspace": "/host/ws.xcworkspace", + }, + )() + result = await client.run_task(task, task_id=task_id) + + self.assertTrue(result.done) + body = self.transport.calls[1][2] + self.assertEqual(body["platform"], "ios") + self.assertEqual(body["ios_workspace"], "/host/ws.xcworkspace") + + async def test_platform_defaults_to_android_everywhere(self): + client = self._client() + self.transport.add("POST", "/api/run", self._admitted()) + handle = await client.submit("goal") + self.assertEqual(handle.platform, "android") + + self.transport.add( + "GET", "/api/sessions/task-1", {"session_id": "task-1", "status": "completed"} + ) + result = await client.get_task("task-1") + self.assertEqual(result.platform, "android") + + self.transport.add("GET", "/api/devices", {"devices": [{"serial": "x", "state": "device"}]}) + (device,) = await client.list_devices() + self.assertEqual(device.platform, "android") + + def test_models_read_platform_from_top_level_or_device_info(self): + self.assertEqual( + TaskHandle.from_payload({"task_id": "t", "platform": " iOS "}).platform, + "ios", + ) + self.assertEqual( + TaskResult.from_payload( + { + "task_id": "t", + "status": "completed", + "device_info": {"mobile_platform": "IOS"}, + } + ).platform, + "ios", + ) + self.assertEqual( + Device.from_payload( + { + "serial": "s", + "device_info": '{"mobile_platform": "ios"}', + } + ).platform, + "ios", + ) + self.assertEqual(Device.from_payload({"serial": "s", "platform": "ios"}).platform, "ios") + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/admin_console/test_task_queue_service.py b/tests/unit/admin_console/test_task_queue_service.py index 24a572cd..9be986c0 100644 --- a/tests/unit/admin_console/test_task_queue_service.py +++ b/tests/unit/admin_console/test_task_queue_service.py @@ -1111,3 +1111,83 @@ async def test_enqueue_tasks_debounces_rapid_identical_submissions(): ) assert len(state.queue_items) == 1 assert res2["enqueued_count"] == 0 + + +@pytest.mark.asyncio +async def test_enqueue_tasks_platforms_do_not_dedupe_against_each_other(): + """Identical goal+serial text on Android vs iOS are distinct submissions.""" + state.queue_items = [] + state.active_session_id = None + + with ( + patch("apps.admin_console.services.task_queue_service.session_repo"), + patch( + "apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", + return_value="ticket-x", + ), + patch( + "artemis.runtime.device_pool.device_pool.try_list_devices_async", + new=AsyncMock(return_value=[]), + ), + patch( + "artemis.runtime.device_pool.device_pool.validate_explicit_serial_async", + new=AsyncMock(return_value=None), + ), + patch( + "artemis.runtime.ios_device_pool.ios_device_pool.validate_explicit_serial_async", + new=AsyncMock(return_value=None), + ), + patch.object(TaskQueueService, "ensure_worker_running"), + ): + res_android = await task_queue_service.enqueue_tasks( + ["shared goal"], device_serial="shared-serial", platform="android" + ) + res_ios = await task_queue_service.enqueue_tasks( + ["shared goal"], device_serial="shared-serial", platform="ios" + ) + # Same platform + goal + serial still debounces. + res_ios_again = await task_queue_service.enqueue_tasks( + ["shared goal"], device_serial="shared-serial", platform="ios" + ) + + assert res_android["enqueued_count"] == 1 + assert res_ios["enqueued_count"] == 1 + assert res_ios_again["enqueued_count"] == 0 + platforms = sorted(i["platform"] for i in state.queue_items) + assert platforms == ["android", "ios"] + ios_item = next(i for i in state.queue_items if i["platform"] == "ios") + assert ios_item["device_serial"] == "shared-serial" + + +def test_task_target_ios_skips_a_malformed_adb_endpoint(): + """iOS items never parse their stale/malformed Android endpoint snapshot.""" + from artemis.runtime.adb_endpoint import IosTarget + + target = TaskQueueService._task_target( + { + "platform": "ios", + "device_serial": "AAAA-1111", + "adb_endpoint": {"host": "", "port": "not-a-port"}, + } + ) + assert isinstance(target, IosTarget) + assert target.serial == "AAAA-1111" + + +def test_task_target_android_still_validates_its_endpoint(): + from artemis.runtime.adb_endpoint import AdbTarget + + with pytest.raises((ValueError, TypeError)): + TaskQueueService._task_target( + { + "platform": "android", + "device_serial": "emulator-5554", + "adb_endpoint": {"host": "", "port": "not-a-port"}, + } + ) + target = TaskQueueService._task_target( + {"device_serial": "emulator-5554", "adb_endpoint": AdbEndpoint.local().to_dict()} + if hasattr(AdbEndpoint.local(), "to_dict") + else {"device_serial": "emulator-5554"} + ) + assert isinstance(target, AdbTarget) diff --git a/tests/unit/core/test_device_smoke.py b/tests/unit/core/test_device_smoke.py index 8fd6e367..ef42b244 100644 --- a/tests/unit/core/test_device_smoke.py +++ b/tests/unit/core/test_device_smoke.py @@ -48,10 +48,13 @@ def _install(get_screen_data=None, factory=None): controller.get_screen_data = get_screen_data calls: list[str | None] = [] - def _fake_get_controller(device_serial=None): + def _fake_get_controller(device_serial=None, target_platform=None): calls.append(device_serial) if factory is not None: - return factory(device_serial) + if target_platform is None: + # Preserve the Android call shape: no platform kwarg. + return factory(device_serial) + return factory(device_serial, target_platform=target_platform) return controller monkeypatch.setattr("artemis.mcp.adb_server._get_controller", _fake_get_controller) @@ -358,6 +361,123 @@ def test_fix_for_error_generic_fallback_uses_serial(): assert "" in device_smoke.fix_for_error("timeout", None)[1] +# --------------------------------------------------------------------------- # +# iOS path +# --------------------------------------------------------------------------- # + +IOS_UDID = "AAAA-1111-0000" + + +def _ios_screen_data(): + return SimpleNamespace( + base64=base64.b64encode(FAKE_JPEG).decode("ascii"), + elements='', + width=1170, + height=2532, + ) + + +def _ios_controller(device_id=IOS_UDID): + controller = MagicMock() + controller.ctx = SimpleNamespace( + device=SimpleNamespace(device_id=device_id, mobile_platform="ios") + ) + controller._driver = SimpleNamespace() # not a PhysicalIosDriver -> xcode + return controller + + +@pytest.mark.asyncio +async def test_ios_smoke_runs_observation_helper_and_reports_xcode(patch_controller, monkeypatch): + """iOS observes through the lease-owning helper on the caller's loop.""" + observe = AsyncMock(return_value=_ios_screen_data()) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + captured: dict = {} + + def factory(device_serial, target_platform=None): + captured["device_serial"] = device_serial + captured["target_platform"] = target_platform + return _ios_controller() + + patch_controller(factory=factory) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is True + assert result["error"] is None + assert result["fix"] == [] + assert result["serial"] == IOS_UDID + assert result["hierarchy_backend"] == "xcode" + assert captured == {"device_serial": IOS_UDID, "target_platform": "ios"} + observe.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ios_smoke_ignores_adb_serial_env(patch_controller, monkeypatch): + """ADB_DEVICE_SERIAL must never seed an iOS target.""" + monkeypatch.setenv("ADB_DEVICE_SERIAL", "emulator-5554") + observe = AsyncMock(return_value=_ios_screen_data()) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + + def factory(device_serial, target_platform=None): + return _ios_controller(device_id=device_serial or "booted") + + patch_controller(factory=factory) + + result = await smoke_test_device(platform="ios") + + assert result["ok"] is True + # Serial comes from ctx after canonical resolution — never the ADB serial. + assert result["serial"] == "booted" + assert "emulator-5554" not in str(result) + + +@pytest.mark.asyncio +async def test_ios_smoke_skips_unscoped_android_busy_guard(patch_controller, monkeypatch): + """The helper performs scope-correct admission; an Android owner on the + same serial text must not veto the iOS smoke.""" + monkeypatch.setattr( + DeviceExecutionLock, + "get_active_owners", + classmethod(lambda cls: {IOS_UDID: _owner(IOS_UDID, "android task")}), + ) + observe = AsyncMock(return_value=_ios_screen_data()) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + patch_controller(factory=lambda device_serial, target_platform=None: _ios_controller()) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is True + observe.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ios_smoke_failure_maps_to_ios_fixes(patch_controller, monkeypatch): + observe = AsyncMock(side_effect=RuntimeError("WebDriverAgent did not answer within 45s")) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + patch_controller(factory=lambda device_serial, target_platform=None: _ios_controller()) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is False + assert "WebDriverAgent" in result["error"] + assert any("WebDriverAgent" in step for step in result["fix"]) + assert not any("adb " in step for step in result["fix"]) + + +@pytest.mark.asyncio +async def test_ios_smoke_init_failure_reports_ios_target(patch_controller): + def _boom(device_serial, target_platform=None): + raise Exception("iOS support requires macOS and Xcode 27 or later") + + patch_controller(factory=_boom) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is False + assert "Failed to initialize iOS device controller" in result["error"] + assert any("Xcode" in step for step in result["fix"]) + + def test_smoke_test_is_exported_from_diagnostics_package(): import artemis.core.diagnostics as diagnostics diff --git a/tests/unit/drivers/test_ios_driver.py b/tests/unit/drivers/test_ios_driver.py index faf4491b..e20ba36e 100644 --- a/tests/unit/drivers/test_ios_driver.py +++ b/tests/unit/drivers/test_ios_driver.py @@ -193,6 +193,15 @@ async def test_booted_selector_rejects_ambiguous_or_missing_devices(simulator, b simulator.native.call.assert_not_awaited() +@pytest.mark.asyncio +async def test_booted_selector_tolerates_mixed_case_and_whitespace(simulator): + """The reserved 'booted' token normalizes before comparison.""" + simulator.driver._device_id = " BoOtEd " + await simulator.driver.connect() + assert simulator.driver.device_id == IOS_A + await simulator.driver.disconnect() + + @pytest.mark.asyncio @pytest.mark.parametrize("requested, available", [(WATCH, True), (IOS_A, False)]) async def test_explicit_selection_rejects_non_ios_and_unavailable_devices( @@ -925,3 +934,24 @@ async def test_workspace_must_be_existing_project_directory(simulator, tmp_path) driver = XcodeSimulatorDriver(workspace_path=workspace) await driver.connect() await driver.disconnect() + + +@pytest.mark.asyncio +async def test_duplicate_simulator_names_reject_instead_of_first_matching(simulator): + """Two same-named sims must not silently bind — pin the UDID instead.""" + simulator.driver._device_id = "Office iPhone" + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"] = [ + device(IOS_A) | {"name": "Office iPhone"}, + device(IOS_B, state="Shutdown") | {"name": "Office iPhone"}, + ] + with pytest.raises(ValueError, match="UDID"): + await simulator.driver.connect() + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_exact_simulator_name_resolves_to_its_udid(simulator): + simulator.driver._device_id = "iPhone" + await simulator.driver.connect() + assert simulator.driver.device_id == IOS_A + await simulator.driver.disconnect() diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py index f8d67af4..43c30f65 100644 --- a/tests/unit/drivers/test_ios_physical.py +++ b/tests/unit/drivers/test_ios_physical.py @@ -17,6 +17,7 @@ import json from pathlib import Path import plistlib +import threading from types import SimpleNamespace import pytest @@ -122,7 +123,9 @@ def test_physical_classification_requires_ios_and_hardware(): def test_find_physical_device_matches_udid_case_insensitively(monkeypatch): monkeypatch.setattr( - discovery, "list_core_devices_sync", lambda force_refresh=False: _parsed(PHYSICAL_IPHONE, SIMULATOR) + discovery, + "list_core_devices_sync", + lambda force_refresh=False: _parsed(PHYSICAL_IPHONE, SIMULATOR), ) found = discovery.find_physical_ios_device_sync(IPHONE_UDID.lower()) assert found is not None and found["udid"] == IPHONE_UDID @@ -146,24 +149,27 @@ def driver(monkeypatch): @pytest.fixture(autouse=True) def _clean_wda_env(monkeypatch): for key in ( - 'ARTEMIS_IOS_WDA_URL', - 'ARTEMIS_IOS_WDA_HOST', - 'ARTEMIS_IOS_WDA_XCTESTRUN', - 'ARTEMIS_IOS_WDA_BUNDLE_ID', + "ARTEMIS_IOS_WDA_URL", + "ARTEMIS_IOS_WDA_HOST", + "ARTEMIS_IOS_WDA_XCTESTRUN", + "ARTEMIS_IOS_WDA_BUNDLE_ID", ): monkeypatch.delenv(key, raising=False) @pytest.mark.asyncio -async def test_physical_resolve_rejects_booted_and_empty_serials(driver): - driver._device_id = "booted" +@pytest.mark.parametrize("serial", ["booted", " Booted ", "BOOTED", ""]) +async def test_physical_resolve_rejects_booted_and_empty_serials(driver, serial): + driver._device_id = serial with pytest.raises(ValueError, match="device-serial"): await driver._resolve_device() @pytest.mark.asyncio async def test_physical_resolve_pins_the_udid(driver, monkeypatch): - monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE, SIMULATOR))) + monkeypatch.setattr( + physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE, SIMULATOR)) + ) candidate = await driver._resolve_device() assert candidate["udid"] == IPHONE_UDID assert driver.device_id == IPHONE_UDID @@ -242,11 +248,20 @@ async def fake(*arguments, timeout=30.0): class _FakeWda: """In-memory WebDriverAgent stand-in for driver interaction tests.""" - def __init__(self, window=(100.0, 200.0), tree=None): + def __init__( + self, + window=(100.0, 200.0), + tree=None, + device_name="Jane's iPhone", + is_simulator=False, + ): self.session_id = None self.base_url = "http://fake-wda:8100" self.window = window self.tree = tree if tree is not None else {"type": "Application", "children": []} + self.device_name = device_name + self.is_simulator = is_simulator + self.opened_sessions = 0 self.tapped: list[tuple[float, float, int]] = [] self.swiped: list[tuple[float, float, float, float, int]] = [] self.typed: list[str] = [] @@ -263,7 +278,11 @@ def _make_png(width: int = 300, height: int = 600) -> bytes: Image.new("RGB", (width, height), color="red").save(buffer, format="PNG") return buffer.getvalue() + async def device_info(self, timeout=10.0): + return {"name": self.device_name, "isSimulator": self.is_simulator} + async def open_session(self): + self.opened_sessions += 1 self.session_id = "wda-session" return self.session_id @@ -318,8 +337,14 @@ async def test_launch_records_the_pid_from_devicectl_json(connected_driver, monk async def fake_xcrun(*arguments, timeout=30.0): calls.append(arguments) return json.dumps( - {"result": {"process": {"processIdentifier": 4242, - "executable": "file:///Apps/Example.app/Example"}}} + { + "result": { + "process": { + "processIdentifier": 4242, + "executable": "file:///Apps/Example.app/Example", + } + } + } ).encode() monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) @@ -332,21 +357,177 @@ async def fake_xcrun(*arguments, timeout=30.0): @pytest.mark.asyncio async def test_stop_app_terminates_the_tracked_pid(connected_driver, monkeypatch): + """A cached launch pid wins only after it is re-verified against the app's URL.""" connected_driver._launched_pids["com.example.app"] = 4242 calls = [] + payloads = { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": { + "result": { + "runningProcesses": [ + {"processIdentifier": 7, "executable": "file:///usr/libexec/other"}, + { + "processIdentifier": 4242, + "executable": "file:///private/var/containers/X/Example.app/Example", + }, + ] + } + }, + } async def fake_xcrun(*arguments, timeout=30.0): - calls.append(arguments) - return b"{}" + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) assert await connected_driver.stop_app("com.example.app") - terminate = calls[0] - assert terminate[:5] == ("devicectl", "device", "process", "terminate", "--device") + terminate = next(c for c in calls if "terminate" in c) + assert terminate[:5] == ["devicectl", "device", "process", "terminate", "--device"] assert terminate[terminate.index("--pid") + 1] == "4242" assert connected_driver._launched_pids == {} +@pytest.mark.asyncio +async def test_stop_app_ignores_a_stale_cached_pid(connected_driver, monkeypatch): + """A recycled cached pid now owned by a different app must not be killed.""" + connected_driver._launched_pids["com.example.app"] = 4242 + calls = [] + payloads = { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": { + "result": { + "runningProcesses": [ + # pid 4242 was recycled into an unrelated binary. + { + "processIdentifier": 4242, + "executable": "file:///usr/sbin/otherd", + }, + { + "processIdentifier": 42, + "executable": "file:///var/containers/X/Example.app/Example", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.stop_app("com.example.app") + terminated = [c[c.index("--pid") + 1] for c in calls if "terminate" in c] + assert terminated == ["42"] + + +@pytest.mark.asyncio +async def test_stop_app_never_matches_sibling_app_prefixes(connected_driver, monkeypatch): + """'Example.appOther' must not match the 'Example.app' URL prefix.""" + calls = [] + payloads = { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": { + "result": { + "runningProcesses": [ + { + "processIdentifier": 99, + "executable": "file:///var/containers/X/Example.appOther/Other", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + with pytest.raises(ValueError, match="No running process"): + await connected_driver.stop_app("com.example.app") + assert not any("terminate" in c for c in calls) + + +@pytest.mark.asyncio +async def test_stop_app_rejects_a_missing_app_url(connected_driver, monkeypatch): + """An installed-app entry without a usable URL cannot verify a victim.""" + calls = [] + payloads = { + "apps": {"result": {"apps": [{"bundleIdentifier": "com.example.app", "url": None}]}}, + "processes": { + "result": { + "runningProcesses": [ + { + "processIdentifier": 42, + "executable": "file:///var/containers/X/Example.app/Example", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + with pytest.raises(ValueError, match="not installed|could not be determined"): + await connected_driver.stop_app("com.example.app") + assert not any("terminate" in c for c in calls) + + @pytest.mark.asyncio async def test_stop_app_without_pid_scans_running_processes(connected_driver, monkeypatch): calls = [] @@ -398,8 +579,21 @@ async def test_stop_app_without_process_fails_clearly(connected_driver, monkeypa monkeypatch.setattr( physical_driver, "run_xcrun", - _xcrun_payload_writer({"apps": {"result": {"apps": []}}, - "processes": {"result": {"runningProcesses": []}}}), + _xcrun_payload_writer( + { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": {"result": {"runningProcesses": []}}, + } + ), ) with pytest.raises(ValueError, match="No running process"): await connected_driver.stop_app("com.example.app") @@ -409,9 +603,7 @@ async def test_stop_app_without_process_fails_clearly(connected_driver, monkeypa async def test_install_accepts_signed_app_directories(connected_driver, monkeypatch, tmp_path): app = tmp_path / "Writer.app" app.mkdir() - (app / "Info.plist").write_bytes( - plistlib.dumps({"CFBundleIdentifier": "com.example.writer"}) - ) + (app / "Info.plist").write_bytes(plistlib.dumps({"CFBundleIdentifier": "com.example.writer"})) calls = [] async def fake_xcrun(*arguments, timeout=30.0): @@ -470,9 +662,7 @@ async def test_list_apps_parses_devicectl_json(connected_driver, monkeypatch): async def fake_xcrun(*arguments, timeout=30.0): args = list(arguments) if "--json-output" in args: - Path(args[args.index("--json-output") + 1]).write_text( - json.dumps({"result": payload}) - ) + Path(args[args.index("--json-output") + 1]).write_text(json.dumps({"result": payload})) return b"" monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) @@ -519,10 +709,10 @@ async def fake_ffmpeg(arguments): output.write_bytes(b"mp4") return 0, b"" - async def fake_probe(path): + async def fake_probe(path, timeout_seconds=None): return {"duration": 1.0, "width": 100, "height": 200} - async def fake_manifest(output_dir, paths, offsets): + async def fake_manifest(output_dir, paths, offsets, probe_timeout_seconds=None): manifest = Path(output_dir) / "manifest.json" manifest.write_text("{}") return manifest @@ -773,15 +963,15 @@ async def fake_xcrun(*arguments, timeout=30.0): assert connected_driver._wda is None assert connected_driver._session_key is None assert connected_driver._wda_runner_pid is None - assert any( - "terminate" in args and str(777) in args for args in calls - ) + assert any("terminate" in args and str(777) in args for args in calls) @pytest.mark.asyncio async def test_connect_establishes_wda_session(driver, monkeypatch): client = _FakeWda() - monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE))) + monkeypatch.setattr( + physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE)) + ) monkeypatch.setattr(driver, "_ensure_wda", _async_return(client)) monkeypatch.setattr(driver, "_require_ios_host", _async_return(None)) await driver.connect() @@ -897,3 +1087,529 @@ def test_android_selection_is_unchanged(monkeypatch): monkeypatch.delenv("ARTEMIS_MOCK_DRIVER", raising=False) driver = factory.create_driver(ctx) assert isinstance(driver, AndroidAdbDriver) + + +# --- Xcode 27 devicectl schema and ambiguity -------------------------------- + + +def _modern_devicectl_device( + udid, + name="Device", + os_version="27.0", + platform="iOS", + reality="physical", + state="connected", + pairing="paired", + visibility="default", +): + """One entry in Xcode 27's modern ``properties``-only devicectl shape.""" + return { + "identifier": udid, + "properties": { + "hardware": { + "udid": udid, + "platform": platform, + "reality": reality, + "productType": "iPhone17,2", + }, + "state": {"name": name, "visibilityClass": visibility}, + "software": { + "osVersionNumber": { + "components": [27, 0, 0], + "stringValue": os_version, + } + }, + "connection": {"pairingState": pairing, "state": state}, + }, + } + + +def test_devicectl_devices_read_the_modern_properties_shape(): + modern = _modern_devicectl_device(IPHONE_UDID, name="Test Phone") + devices = discovery.parse_devicectl_devices(_devicectl_payload(modern)) + assert devices == [ + { + "udid": IPHONE_UDID, + "name": "Test Phone", + "os_version": "27.0", + "platform": "iOS", + "reality": "physical", + "product_type": "iPhone17,2", + "connection_state": "connected", + "pairing_state": "paired", + "visibility": "default", + } + ] + + +def test_devicectl_devices_tolerate_null_modern_sections(): + modern = _modern_devicectl_device(IPHONE_UDID) + modern["properties"]["software"] = None + modern["properties"]["state"] = None + devices = discovery.parse_devicectl_devices(_devicectl_payload(modern)) + assert devices[0]["udid"] == IPHONE_UDID + assert devices[0]["os_version"] is None + assert devices[0]["name"] is None + + +def test_devicectl_devices_prefer_legacy_values_when_both_present(): + device = _devicectl_device(IPHONE_UDID, name="Legacy Name", os_version="26.0") + device["properties"] = _modern_devicectl_device( + IPHONE_UDID, name="Modern Name", os_version="27.0" + )["properties"] + parsed = _parsed(device)[0] + assert parsed["name"] == "Legacy Name" + assert parsed["os_version"] == "26.0" + + +def test_find_physical_rejects_ambiguous_names(monkeypatch): + monkeypatch.setattr( + discovery, + "list_core_devices_sync", + lambda force_refresh=False: _parsed( + _devicectl_device(IPHONE_UDID, name="Office iPhone"), + _devicectl_device(IPAD_UDID, name="Office iPhone", platform="iPadOS"), + ), + ) + with pytest.raises(ValueError, match="UDID"): + discovery.find_physical_ios_device_sync("Office iPhone") + # The exact UDID still resolves uniquely. + found = discovery.find_physical_ios_device_sync(IPAD_UDID) + assert found is not None and found["udid"] == IPAD_UDID + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_duplicate_names(driver, monkeypatch): + driver._device_id = "Office iPhone" + monkeypatch.setattr( + physical_driver, + "list_core_devices", + _async_devices( + _parsed( + _devicectl_device(IPHONE_UDID, name="Office iPhone"), + _devicectl_device(IPAD_UDID, name="Office iPhone", platform="iPadOS"), + ) + ), + ) + with pytest.raises(ValueError, match="UDID"): + await driver._resolve_device() + + +# --- WebDriverAgent session safety ----------------------------------------- # + + +class _WdaResponder: + """Scripted urllib stub that records requests and can block on demand.""" + + def __init__(self): + self.requests: list[tuple[str, str]] = [] + self.responses: dict[str, object] = {} + self.blockers: dict[str, threading.Event] = {} + self.entered: dict[str, threading.Event] = {} + + class _Response: + def __init__(self, raw): + self._raw = raw + + def read(self): + return self._raw + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def set(self, suffix, payload): + self.responses[suffix] = payload + + def block_on(self, suffix): + self.blockers[suffix] = threading.Event() + self.entered[suffix] = threading.Event() + + def release(self, suffix): + self.blockers[suffix].set() + + def urlopen(self, request, timeout): + method = request.get_method() + url = request.full_url + for suffix, event in self.entered.items(): + if url.endswith(suffix): + event.set() + self.blockers[suffix].wait(timeout=10) + self.requests.append((method, url)) + for suffix, payload in self.responses.items(): + if url.endswith(suffix): + raw = payload if isinstance(payload, bytes) else json.dumps(payload).encode() + return self._Response(raw) + return self._Response(json.dumps({"value": {}}).encode()) + + +@pytest.fixture +def responder(monkeypatch): + stub = _WdaResponder() + monkeypatch.setattr(wda.urllib.request, "urlopen", stub.urlopen) + return stub + + +def test_normalize_wda_url_keeps_the_port_in_the_authority(): + assert wda.normalize_wda_url("http://localhost/proxy/wda") == "http://localhost:8100/proxy/wda" + assert ( + wda.normalize_wda_url("http://localhost:9000/proxy/wda") + == "http://localhost:9000/proxy/wda" + ) + assert wda.normalize_wda_url("http://[fd20::1]/wda") == "http://[fd20::1]:8100/wda" + + +@pytest.mark.asyncio +async def test_wda_status_surfaces_the_outer_session_id(responder): + responder.set( + "/status", + {"value": {"ready": True, "build": {}}, "sessionId": "foreign-777"}, + ) + status = await WdaClient("http://wda.test:8100").status() + assert status == {"ready": True, "build": {}, "sessionId": "foreign-777"} + + +@pytest.mark.asyncio +async def test_wda_status_rejects_non_json_responses(responder): + """HTML/bytes from a foreign service on the port must not pass as WDA.""" + responder.set("/status", b"proxy error") + client = WdaClient("http://wda.test:8100") + assert await client.status() is None + + +@pytest.mark.asyncio +async def test_open_session_refuses_a_foreign_active_session(responder): + responder.set("/status", {"value": {"ready": True}, "sessionId": "someone-elses"}) + client = WdaClient("http://wda.test:8100") + with pytest.raises(RuntimeError, match="Refusing to replace"): + await client.open_session() + methods = {m for m, _url in responder.requests} + assert methods == {"GET"} # status only — no POST, no DELETE + assert client.session_id is None + + +@pytest.mark.asyncio +async def test_open_session_reuses_its_own_session(responder): + responder.set("/status", {"value": {"ready": True}, "sessionId": None}) + responder.set("/session", {"value": {"sessionId": "owned-1"}}) + client = WdaClient("http://wda.test:8100") + first = await client.open_session() + again = await client.open_session() + assert first == again == "owned-1" + posts = [u for m, u in responder.requests if m == "POST"] + assert len(posts) == 1 + + +@pytest.mark.asyncio +async def test_open_session_creates_after_a_null_session_status(responder): + responder.set("/status", {"value": {"ready": True}, "sessionId": None}) + responder.set("/session", {"value": {"sessionId": "fresh-1"}}) + client = WdaClient("http://wda.test:8100") + assert await client.open_session() == "fresh-1" + assert client.session_id == "fresh-1" + + +@pytest.mark.asyncio +async def test_request_cancellation_drains_the_blocking_call(responder): + """A cancelled input must not leave the urllib call racing the release.""" + client = WdaClient("http://wda.test:8100") + responder.block_on("/session/s1/actions") + responder.set("/session/s1/actions", {"value": {}}) + + task = asyncio.create_task(client._request("POST", "/session/s1/actions", {})) + try: + # Event.wait blocks a worker thread, never the loop — the request + # coroutine must already be inside urlopen before we cancel it. + entered = responder.entered["/session/s1/actions"] + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) # let the cancellation land on the task + assert not task.done() # still draining the blocked HTTP call + finally: + responder.release("/session/s1/actions") + with pytest.raises(asyncio.CancelledError): + await task + # The drained request finished before cancellation propagated. + assert ("POST", "http://wda.test:8100/session/s1/actions") in responder.requests + + +@pytest.mark.asyncio +async def test_open_session_cancellation_deletes_the_orphaned_session(responder): + client = WdaClient("http://wda.test:8100") + responder.set("/status", {"value": {"ready": True}, "sessionId": None}) + responder.block_on("/session") + responder.set("/session", {"value": {"sessionId": "orphan-9"}}) + + task = asyncio.create_task(client.open_session()) + try: + entered = responder.entered["/session"] + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + finally: + responder.release("/session") + with pytest.raises(asyncio.CancelledError): + await task + deletes = [u for m, u in responder.requests if m == "DELETE"] + assert deletes == ["http://wda.test:8100/session/orphan-9"] + assert client.session_id is None + + +@pytest.mark.asyncio +async def test_device_info_rejects_non_dict_responses(responder): + responder.set("/wda/device/info", b"not-json") + client = WdaClient("http://wda.test:8100") + with pytest.raises(RuntimeError, match="non-JSON"): + await client.device_info() + + +@pytest.mark.asyncio +async def test_device_info_returns_the_dict(responder): + responder.set( + "/wda/device/info", + {"value": {"name": "Test Phone", "isSimulator": False, "uuid": "iv"}}, + ) + info = await WdaClient("http://wda.test:8100").device_info() + assert info["name"] == "Test Phone" + assert info["isSimulator"] is False + + +# --- Connect identity verification ----------------------------------------- # + + +def _connect_stubs(driver, monkeypatch, client): + monkeypatch.setattr( + physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE)) + ) + monkeypatch.setattr(driver, "_ensure_wda", _async_return(client)) + monkeypatch.setattr(driver, "_require_ios_host", _async_return(None)) + + +@pytest.mark.asyncio +async def test_connect_verifies_the_wda_endpoint_identity(driver, monkeypatch): + client = _FakeWda(device_name="Jane's iPhone") + _connect_stubs(driver, monkeypatch, client) + await driver.connect() + assert driver._session_key == "wda-session" + assert client.opened_sessions == 1 + + +@pytest.mark.asyncio +async def test_connect_rejects_a_wda_on_the_wrong_device(driver, monkeypatch): + client = _FakeWda(device_name="Someone Else's iPhone") + _connect_stubs(driver, monkeypatch, client) + with pytest.raises(RuntimeError, match="ARTEMIS_IOS_WDA_URL"): + await driver.connect() + assert client.opened_sessions == 0 + assert driver._session_key is None + + +@pytest.mark.asyncio +async def test_connect_rejects_a_simulator_wda(driver, monkeypatch): + client = _FakeWda(device_name="Jane's iPhone", is_simulator=True) + _connect_stubs(driver, monkeypatch, client) + with pytest.raises(RuntimeError, match="ARTEMIS_IOS_WDA_URL"): + await driver.connect() + assert client.opened_sessions == 0 + + +# --- Disconnect cleanup ----------------------------------------------------- # + + +class _HungProcess: + """A subprocess stand-in that ignores terminate() and must be reaped.""" + + def __init__(self): + self.returncode = None + self.terminated = False + self.killed = False + self.communicated = False + + def terminate(self): + self.terminated = True + + def kill(self): + self.killed = True + self.returncode = -9 + + async def wait(self): + await asyncio.Event().wait() # never exits on its own + + async def communicate(self): + self.communicated = True + return b"", b"" + + +@pytest.mark.asyncio +async def test_disconnect_reaps_a_test_process_that_ignores_terminate( + connected_driver, monkeypatch +): + process = _HungProcess() + connected_driver._wda_test_process = process + monkeypatch.setattr(physical_driver.asyncio, "wait_for", _hangs_then_times_out) + await connected_driver.disconnect() + assert process.terminated + assert process.killed and process.communicated + assert connected_driver._wda_test_process is None + + +async def _hangs_then_times_out(awaitable, timeout): + awaitable.close() + raise TimeoutError + + +@pytest.mark.asyncio +async def test_disconnect_reaps_children_even_when_close_session_fails( + connected_driver, monkeypatch +): + async def _boom(): + raise RuntimeError("wda already gone") + + connected_driver._wda.close_session = _boom + process = _HungProcess() + connected_driver._wda_test_process = process + connected_driver._wda_runner_pid = 777 + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_driver.asyncio, "wait_for", _hangs_then_times_out) + await connected_driver.disconnect() + assert process.killed and process.communicated + assert any("terminate" in args and str(777) in args for args in calls) + assert connected_driver._wda is None + + +# --- Physical recorder cancellation and ffconcat ---------------------------- # + + +def test_ffconcat_file_line_escapes_apostrophes(): + path = Path("/tmp/Jane's Frames/frame 1.png") + line = PhysicalIosRecorder._ffconcat_file_line(path) + assert line == "file '/tmp/Jane'\\''s Frames/frame 1.png'" + + +@pytest.mark.asyncio +async def test_run_ffmpeg_cancellation_reaps_and_drops_part_file(tmp_path, monkeypatch): + entered = asyncio.Event() + release = asyncio.Event() + reaped: list[object] = [] + + class _FakeProc: + returncode = None + + async def communicate(self): + entered.set() + await release.wait() + return b"", b"" + + async def fake_exec(*args, **kwargs): + return _FakeProc() + + async def fake_reap(process): + reaped.append(process) + + monkeypatch.setattr(physical_recording.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(physical_recording, "reap_process", fake_reap) + part = tmp_path / "segment_0000.part.mp4" + part.write_bytes(b"partial") + task = asyncio.create_task(physical_recording._run_ffmpeg(["-i", "in", str(part)])) + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert len(reaped) == 1 + assert not part.exists() + + +@pytest.mark.asyncio +async def test_run_ffmpeg_timeout_reaps_and_reports_failure(tmp_path, monkeypatch): + reaped: list[object] = [] + + class _FakeProc: + returncode = None + + async def communicate(self): + await asyncio.Event().wait() + + async def fake_exec(*args, **kwargs): + return _FakeProc() + + async def fake_reap(process): + process.returncode = -9 + reaped.append(process) + + monkeypatch.setattr(physical_recording.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(physical_recording, "reap_process", fake_reap) + monkeypatch.setattr(physical_recording, "FFMPEG_TIMEOUT_SECONDS", 0.05) + code, stderr = await physical_recording._run_ffmpeg(["-i", "in", str(tmp_path / "o.mp4")]) + assert code == -1 + assert b"timed out" in stderr + assert len(reaped) == 1 + + +@pytest.mark.asyncio +async def test_input_fails_clearly_when_wda_is_detached(): + """A stale session key without a WDA client fails before input/observe.""" + driver = PhysicalIosDriver(device_id=IPHONE_UDID) + driver._session_key = "stale-session" + driver._wda = None + for call in ( + driver.tap(10, 10), + driver.press_key("home"), + driver.input_text("hi", clear_existing=False), + ): + with pytest.raises(RuntimeError, match="Connect the physical iOS driver"): + await call + + +class _FalseHomeWda(_FakeWda): + """A WDA backend that always reports Home button failure.""" + + async def press_button(self, name): + self.buttons.append(name) + return False + + +@pytest.mark.asyncio +async def test_home_key_falls_back_to_homescreen_when_button_fails(connected_driver): + connected_driver._wda = _FalseHomeWda() + assert await connected_driver.press_key("home") is True + assert connected_driver._wda.buttons == ["home"] + assert connected_driver._wda.homescreen_calls == 1 + + +@pytest.mark.asyncio +async def test_app_switch_raises_when_home_button_is_unavailable(connected_driver): + connected_driver._wda = _FalseHomeWda() + with pytest.raises(NotImplementedError, match="App switching is unavailable"): + await connected_driver.press_key("app_switch") + # No homescreen fallback may masquerade as a successful app switch. + assert connected_driver._wda.homescreen_calls == 0 + + +@pytest.mark.asyncio +async def test_app_switch_double_presses_home_on_capable_backend(connected_driver): + assert await connected_driver.press_key("app_switch") is True + assert connected_driver._wda.buttons == ["home", "home"] + + +@pytest.mark.parametrize("bad_rect", ["missing", None, "bogus", 42]) +def test_parse_wda_elements_skips_bad_parent_rect_but_keeps_children(bad_rect): + child = { + "type": "XCUIElementTypeButton", + "label": "OK", + "rect": {"x": 1, "y": 1, "width": 10, "height": 10}, + } + tree = {"type": "XCUIElementTypeApplication", "children": [child]} + if bad_rect != "missing": + tree["rect"] = bad_rect + elements = wda.parse_wda_elements(tree, (1.0, 1.0), 300, 600) + assert [e["text"] for e in elements] == ["OK"] diff --git a/tests/unit/drivers/test_ios_recording.py b/tests/unit/drivers/test_ios_recording.py index 552c8408..0865601c 100644 --- a/tests/unit/drivers/test_ios_recording.py +++ b/tests/unit/drivers/test_ios_recording.py @@ -20,6 +20,7 @@ from pathlib import Path import signal import subprocess +import threading import time from types import SimpleNamespace from unittest.mock import AsyncMock @@ -221,6 +222,63 @@ async def test_startup_cancellation_reaps_child(recorder_env, monkeypatch, tmp_p assert signal.SIGINT in proc.signals +@pytest.mark.asyncio +async def test_cancelled_probe_after_marker_reaps_child(recorder_env, monkeypatch, tmp_path): + """Cancelling during the dimension probe must not leak the recorder child.""" + proc = FakeRecorderProcess() # marker arrives instantly + patch_spawn(monkeypatch, [proc]) + probe_entered = threading.Event() + probe_gate = threading.Event() + + async def gated_probe(device_id): + probe_entered.set() + while not probe_gate.is_set(): + await asyncio.sleep(0.005) + return (1206, 2622) + + monkeypatch.setattr(rec, "probe_display_dimensions", gated_probe) + recorder = IosScreenRecorder(UDID) + + task = asyncio.create_task(recorder.start(output_dir=tmp_path)) + try: + assert await asyncio.to_thread(probe_entered.wait, 5) + task.cancel() + await asyncio.sleep(0) + assert not task.done() # still gated inside the probe await + finally: + probe_gate.set() + with pytest.raises(asyncio.CancelledError): + await task + + session = recorder.session + assert session is not None and not session.is_active + assert proc.returncode is not None + assert signal.SIGINT in proc.signals + assert session.stderr_task is None or session.stderr_task.done() + + +@pytest.mark.asyncio +async def test_probe_failure_after_marker_reaps_child(recorder_env, monkeypatch, tmp_path): + """A dimension-probe error after the marker must reap the owned child.""" + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + + async def boom(device_id): + raise RuntimeError("display probe exploded") + + monkeypatch.setattr(rec, "probe_display_dimensions", boom) + recorder = IosScreenRecorder(UDID) + + with pytest.raises(RuntimeError, match="display probe exploded"): + await recorder.start(output_dir=tmp_path) + + session = recorder.session + assert session is not None and not session.is_active + assert proc.returncode is not None + assert signal.SIGINT in proc.signals + assert session.stderr_task is None or session.stderr_task.done() + + @pytest.mark.asyncio async def test_sigint_terminate_kill_escalation(recorder_env, monkeypatch, tmp_path): monkeypatch.setattr(rec, "SIGINT_FLUSH_TIMEOUT_SECONDS", 0.05) @@ -666,3 +724,77 @@ def test_parse_display_dimensions_uses_iosurface_not_default_dims(): assert rec._parse_display_dimensions(rotated) == (2622, 1206) assert rec._parse_display_dimensions("Port:\n Class: Unknown\n") is None assert rec._parse_display_dimensions("") is None + + +@pytest.mark.asyncio +async def test_young_crash_loop_caps_respawns_and_stops(recorder_env, monkeypatch, tmp_path): + """Spawns that succeed then die young must bound the crash loop. + + Every fake proc delivers the 'Recording started' marker (spawn succeeds) + and then exits immediately — the watchdog's young-crash counter must stop + the respawn loop at MAX_CONSECUTIVE_FAILURES instead of spawning forever. + """ + created: list[FakeRecorderProcess] = [] + + async def spawn(*argv, **kwargs): + proc = FakeRecorderProcess() + created.append(proc) + + def die_young(): + if proc.returncode is None: + proc.returncode = 1 + proc._exit.set() + + asyncio.get_running_loop().call_later(0.01, die_young) + return proc + + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and session.is_active: + await asyncio.sleep(0.02) + + assert not session.is_active + assert len(created) <= rec.MAX_CONSECUTIVE_FAILURES + 1 + assert any("recovery limit" in error for error in session.errors) + # No orphan process survives the cap. + assert all(proc.returncode is not None for proc in created) + + +@pytest.mark.asyncio +async def test_finalize_cancellation_reaps_child_and_drops_part(tmp_path, monkeypatch): + """Cancelling finalize must reap the ffmpeg child and remove its .part.""" + source = tmp_path / "segment_0000.mov" + source.write_bytes(b"mov") + entered = asyncio.Event() + release = asyncio.Event() + reaped: list[object] = [] + + class _Proc: + returncode = None + + async def communicate(self): + entered.set() + await release.wait() + return b"", b"" + + async def fake_exec(*args, **kwargs): + return _Proc() + + async def fake_reap(process): + reaped.append(process) + + monkeypatch.setattr(rec.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(rec, "reap_process", fake_reap) + output = tmp_path / "segment_0000.mp4" + task = asyncio.create_task(finalize_mov_to_mp4(source, output, 100, 200, 1.0)) + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert len(reaped) == 1 + assert not output.exists() + assert not (tmp_path / "segment_0000.part.mp4").exists() diff --git a/tests/unit/mcp/test_diagnose_tool.py b/tests/unit/mcp/test_diagnose_tool.py index cd4627f5..75d76798 100644 --- a/tests/unit/mcp/test_diagnose_tool.py +++ b/tests/unit/mcp/test_diagnose_tool.py @@ -1285,3 +1285,148 @@ def test_newer_helper_than_bundle_is_not_a_finding(temp_trace_env): result = _run(_healthy_probes(), helper_status=status) assert result["verdict"] == "ready" assert not any("Accessibility helper" in s for s in result["next_steps"]) + + +# --------------------------------------------------------------------------- # +# iOS device selection +# --------------------------------------------------------------------------- # + +_IOS_SIM_BOOTED = {"udid": "SIM-AAAA", "name": "Sim Booted", "state": "Booted"} +_IOS_SIM_SHUT = {"udid": "SIM-BBBB", "name": "Sim Off", "state": "Shutdown"} +_IOS_PHYS = {"udid": "PHYS-0001", "name": "Office Phone", "state": "device"} + + +def _ios_probe(simulators=None, physical=None, status=ProbeStatus.PASS): + return _probe( + "ios_simulators", + status, + category=ProbeCategory.DEVICE, + metadata={ + "xcode_27_or_newer": True, + "simulators": ([_IOS_SIM_BOOTED, _IOS_SIM_SHUT] if simulators is None else simulators), + "connected_physical_devices": ([_IOS_PHYS] if physical is None else physical), + }, + ) + + +def test_ios_requested_device_matches_udid_case_insensitively(): + selected = diagnose._ios_requested_device(_ios_probe(), "sim-aaaa") + assert selected == { + "serial": "SIM-AAAA", + "name": "Sim Booted", + "state": "device", + "is_emulator": True, + } + + +def test_ios_requested_device_matches_exact_sim_and_physical_names(): + sim = diagnose._ios_requested_device(_ios_probe(), "Sim Off") + assert sim["serial"] == "SIM-BBBB" and sim["is_emulator"] is True + phys = diagnose._ios_requested_device(_ios_probe(), "Office Phone") + assert phys["serial"] == "PHYS-0001" and phys["is_emulator"] is False + # Name matching is exact — case differences do not bind. + assert diagnose._ios_requested_device(_ios_probe(), "sim off") is None + + +def test_ios_requested_device_explicit_booted_needs_exactly_one(): + assert diagnose._ios_requested_device(_ios_probe(), "booted")["serial"] == "SIM-AAAA" + two_booted = [_IOS_SIM_BOOTED, {**_IOS_SIM_SHUT, "state": "Booted"}] + assert diagnose._ios_requested_device(_ios_probe(simulators=two_booted), "booted") is None + none_booted = [dict(_IOS_SIM_SHUT)] + assert diagnose._ios_requested_device(_ios_probe(simulators=none_booted), "booted") is None + + +def test_ios_requested_device_ambiguous_names_return_none(): + twins = [ + {**_IOS_SIM_BOOTED, "name": "Twin"}, + {**_IOS_SIM_SHUT, "name": "Twin"}, + ] + assert diagnose._ios_requested_device(_ios_probe(simulators=twins), "Twin") is None + + +def test_ios_requested_device_never_autopicks_hardware(): + # No request: exactly one booted sim wins; a sole shutdown sim is next. + assert diagnose._ios_requested_device(_ios_probe(), None)["serial"] == "SIM-AAAA" + only_off = diagnose._ios_requested_device(_ios_probe(simulators=[_IOS_SIM_SHUT]), None) + assert only_off["serial"] == "SIM-BBBB" + # Physical-only inventory is never auto-selected. + assert diagnose._ios_requested_device(_ios_probe(simulators=[]), None) is None + + +def test_ios_probe_serial_routes_canonical_udid(monkeypatch): + """_run_device_probe must smoke the canonical UDID, not the raw request.""" + smoke = AsyncMock(return_value=_smoke_ok("SIM-AAAA")) + monkeypatch.setattr(diagnose, "_device_smoke_test", smoke) + result = asyncio.run(diagnose._run_device_probe(None, "sim-aaaa", _ios_probe(), platform="ios")) + smoke.assert_awaited_once_with("SIM-AAAA", platform="ios") + assert result["ok"] is True + + +def test_ios_probe_without_canonical_target_fails_with_udid_guidance(monkeypatch): + smoke = AsyncMock() + monkeypatch.setattr(diagnose, "_device_smoke_test", smoke) + result = asyncio.run(diagnose._run_device_probe(None, "ghost", _ios_probe(), platform="ios")) + assert result["ok"] is False + assert "UDID" in result["error"] + smoke.assert_not_awaited() + # No request and ambiguous inventory -> same guidance shape. + twins = [_IOS_SIM_BOOTED, {**_IOS_SIM_SHUT, "state": "Booted"}] + result = asyncio.run( + diagnose._run_device_probe(None, None, _ios_probe(simulators=twins), platform="ios") + ) + assert result["ok"] is False + assert "device_serial" in result["error"] + smoke.assert_not_awaited() + + +def test_requested_device_ready_ios_uses_the_same_selection(): + results = [_ios_probe()] + assert diagnose._requested_device_ready(results, "SIM-AAAA", platform="ios") + assert diagnose._requested_device_ready(results, "Office Phone", platform="ios") + assert not diagnose._requested_device_ready(results, "ghost", platform="ios") + assert diagnose._requested_device_ready(results, None, platform="ios") + + +def test_ios_diagnose_reports_the_selected_ios_device(temp_trace_env): + """The iOS `device` payload describes the selected iOS target, not ADB's.""" + probes = _healthy_probes() + [_ios_probe()] + result = _run(probes, platform="ios") + device = result["device"] + assert device == { + "serial": "SIM-AAAA", + "model": "Sim Booted", + "platform": "ios", + "is_emulator": True, + } + + +def test_ios_diagnose_smoke_uses_selected_udid(temp_trace_env): + probes = _healthy_probes() + [_ios_probe()] + smoke = AsyncMock(return_value=_smoke_ok("SIM-AAAA")) + result = _run(probes, platform="ios", probe_device=True, smoke=smoke) + smoke.assert_awaited_once_with("SIM-AAAA", platform="ios") + assert result["device_probe"]["ok"] is True + + +def test_android_diagnose_device_shape_unchanged(temp_trace_env): + """Regression: Android responses keep the compact active-device payload.""" + device = DeviceInfo( + serial="pixel-1", + model="Pixel 8", + android_version="15", + is_locked=False, + installed_packages=["a"], + screen_resolution="1080x2400", + ) + result = _run(_healthy_probes(), report=_report(_healthy_probes(), active_device=device)) + device_out = result["device"] + # The accessibility_helper attachment is part of the compact payload. + assert isinstance(device_out.pop("accessibility_helper"), dict) + assert device_out == { + "serial": "pixel-1", + "state": "device", + "model": "Pixel 8", + "android_version": "15", + "is_locked": False, + "is_emulator": False, + } diff --git a/tests/unit/mcp/test_mcp_tools.py b/tests/unit/mcp/test_mcp_tools.py index 0064ed9a..f1eed944 100644 --- a/tests/unit/mcp/test_mcp_tools.py +++ b/tests/unit/mcp/test_mcp_tools.py @@ -402,6 +402,64 @@ async def test_mobile_get_device_state_passes_device_serial(): get_ctrl_mock.assert_called_once_with(device_serial="device-serial-abc") +@pytest.mark.asyncio +async def test_mobile_get_device_state_rejects_invalid_platform_before_factory(): + """An invalid platform token must never reach the controller factory.""" + with ( + patch("mcp_server.tools.device_state._get_controller") as get_ctrl_mock, + patch( + "mcp_server.tools.device_state.observe_ios_controller", new_callable=AsyncMock + ) as observe_mock, + ): + result = await mobile_get_device_state(view_type="hierarchy", platform="windows-phone") + + assert result.startswith("Error:") + assert "platform" in result.lower() + get_ctrl_mock.assert_not_called() + observe_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_mobile_get_device_state_ios_routes_through_observation_helper(): + """A case/whitespace-tolerant iOS token builds an iOS controller and + observes it through the lease-owning helper, not a raw connect.""" + controller = MagicMock() + controller.ctx.device.mobile_platform = "ios" + controller.ctx.device.device_width = 1206 + controller.ctx.device.device_height = 2622 + screen = SimpleNamespace( + base64="aVBORw==", + elements=[ + { + "class": "XCUIElementTypeButton", + "text": "Continue", + "bounds": "[100,500][400,600]", + } + ], + width=1206, + height=2622, + ) + + with ( + patch( + "mcp_server.tools.device_state._get_controller", return_value=controller + ) as get_ctrl_mock, + patch( + "mcp_server.tools.device_state.observe_ios_controller", + new_callable=AsyncMock, + return_value=screen, + ) as observe_mock, + patch("mcp_server.tools.device_state.is_ocr_configured", return_value=False), + ): + result = await mobile_get_device_state(view_type="hierarchy", platform=" iOS ") + + assert "Continue" in result + assert not result.startswith("Error:") + get_ctrl_mock.assert_called_once_with(device_serial=None, target_platform="ios") + observe_mock.assert_awaited_once_with(controller) + controller.get_screen_data.assert_not_called() + + @pytest.mark.asyncio async def test_mobile_inspect_trace_includes_device_serial(temp_trace_env): import sqlite3 diff --git a/tests/unit/runtime/test_device_lock.py b/tests/unit/runtime/test_device_lock.py index 51cae530..33d491b8 100644 --- a/tests/unit/runtime/test_device_lock.py +++ b/tests/unit/runtime/test_device_lock.py @@ -329,9 +329,7 @@ def test_active_owner_is_discoverable_and_can_be_annotated(monkeypatch): def test_annotation_preserves_lock_scope(monkeypatch): """Annotating a scoped (e.g. iOS) owner must not strip its lock_scope.""" monkeypatch.setenv("ARTEMIS_TASK_INGRESS", "daemon") - owner_lock = DeviceExecutionLock( - "00008100-000000000000001E", "iOS task", lock_scope="ios" - ) + owner_lock = DeviceExecutionLock("00008100-000000000000001E", "iOS task", lock_scope="ios") owner_lock.acquire() try: assert DeviceExecutionLock.annotate_active_owner( diff --git a/tests/unit/runtime/test_ios_observation_lifecycle.py b/tests/unit/runtime/test_ios_observation_lifecycle.py new file mode 100644 index 00000000..4edffc8b --- /dev/null +++ b/tests/unit/runtime/test_ios_observation_lifecycle.py @@ -0,0 +1,266 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lifecycle tests for the shared one-shot iOS observation helper. + +``observe_ios_controller`` must acquire the real iOS-scoped lease before any +native setup, and must always disconnect + release — including on failure and +cancellation. All native calls are stubbed; only the lock registry is real +(isolated per test via ``get_temp_dir``). +""" + +import asyncio +import base64 +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.device_lock import DeviceBusyError, DeviceExecutionLock +from artemis.runtime.ios_observation import observe_ios_controller + +UDID = "AAAA-1111-0000" + + +@pytest.fixture(autouse=True) +def isolated_lock_dir(tmp_path, monkeypatch): + monkeypatch.setattr("artemis.runtime.device_lock.get_temp_dir", lambda *a, **k: tmp_path) + + +def _screen_data(): + return SimpleNamespace( + base64=base64.b64encode(b"\x89PNG" + bytes(600)).decode("ascii"), + elements='', + width=1170, + height=2532, + ) + + +def _controller(events, *, connect_error=None, capture_error=None): + """A lazy iOS controller whose driver records its call order.""" + + async def resolve_device(): + events.append("resolve") + return UDID + + async def connect(): + events.append("connect") + if connect_error is not None: + raise connect_error + + async def disconnect(): + events.append("disconnect") + + async def capture(): + events.append("capture") + if capture_error is not None: + raise capture_error + return _screen_data() + + driver = SimpleNamespace(resolve_device=resolve_device, connect=connect, disconnect=disconnect) + device = SimpleNamespace(device_id="booted", device_width=0, device_height=0) + return SimpleNamespace( + _driver=driver, ctx=SimpleNamespace(device=device), get_screen_data=capture + ) + + +def _ios_owner(device_id: str = UDID): + owners = DeviceExecutionLock.get_active_owners() + return next( + (o for o in owners.values() if o.device_id == device_id and o.lock_scope == IOS_LOCK_SCOPE), + None, + ) + + +@pytest.mark.asyncio +async def test_observation_orders_resolve_lease_connect_capture_cleanup(): + events: list[str] = [] + controller = _controller(events) + + data = await observe_ios_controller(controller) + + assert events == ["resolve", "connect", "capture", "disconnect"] + assert data.width == 1170 + # Real metrics and the resolved UDID land on the context. + assert controller.ctx.device.device_id == UDID + assert controller.ctx.device.device_width == 1170 + assert controller.ctx.device.device_height == 2532 + # The lease is released after disconnect. + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_observation_holds_ios_lease_during_capture(): + events: list[str] = [] + controller = _controller(events) + held: list[bool] = [] + + original_capture = controller.get_screen_data + + async def capture(): + held.append(_ios_owner() is not None) + return await original_capture() + + controller.get_screen_data = capture + await observe_ios_controller(controller) + assert held == [True] + + +@pytest.mark.asyncio +async def test_held_ios_lease_blocks_observation_before_connect(): + events: list[str] = [] + controller = _controller(events) + held = DeviceExecutionLock( + UDID, + "running iOS task", + ingress="mcp", + concurrency_mode="per_device", + lock_scope=IOS_LOCK_SCOPE, + ) + held.acquire(blocking=False) + try: + with pytest.raises(DeviceBusyError): + await observe_ios_controller(controller) + finally: + held.release() + assert "connect" not in events + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_android_lock_on_same_text_does_not_block_ios(): + """Identical serial text in the Android scope is a different device.""" + events: list[str] = [] + controller = _controller(events) + android_lock = DeviceExecutionLock(UDID, "android task") + android_lock.acquire(blocking=False) + try: + await observe_ios_controller(controller) + finally: + android_lock.release() + assert events == ["resolve", "connect", "capture", "disconnect"] + + +@pytest.mark.asyncio +async def test_capture_failure_disconnects_and_releases(): + events: list[str] = [] + controller = _controller(events, capture_error=RuntimeError("capture blew up")) + + with pytest.raises(RuntimeError, match="capture blew up"): + await observe_ios_controller(controller) + assert events == ["resolve", "connect", "capture", "disconnect"] + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_connect_failure_still_disconnects_and_releases(): + """A half-open native session is closed when connect was attempted.""" + events: list[str] = [] + controller = _controller(events, connect_error=RuntimeError("bridge down")) + + with pytest.raises(RuntimeError, match="bridge down"): + await observe_ios_controller(controller) + assert events == ["resolve", "connect", "disconnect"] + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_disconnect_error_does_not_mask_capture_failure(): + events: list[str] = [] + controller = _controller(events, capture_error=RuntimeError("native lost")) + + async def disconnect(): + events.append("disconnect") + raise RuntimeError("disconnect failed") + + controller._driver.disconnect = disconnect + with pytest.raises(RuntimeError, match="native lost"): + await observe_ios_controller(controller) + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_cancellation_still_disconnects_and_releases(): + events: list[str] = [] + controller = _controller(events) + started = asyncio.Event() + + async def capture(): + events.append("capture") + started.set() + await asyncio.Event().wait() # hang until cancelled + + controller.get_screen_data = capture + task = asyncio.create_task(observe_ios_controller(controller)) + await asyncio.wait_for(started.wait(), timeout=5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert events == ["resolve", "connect", "capture", "disconnect"] + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_busy_acquire_failure_never_connects(): + """A failed non-blocking acquire must not touch the native session.""" + events: list[str] = [] + controller = _controller(events) + held = DeviceExecutionLock( + UDID, "owner", concurrency_mode="per_device", lock_scope=IOS_LOCK_SCOPE + ) + held.acquire(blocking=False) + try: + with pytest.raises(DeviceBusyError): + await observe_ios_controller(controller) + finally: + held.release() + assert events == ["resolve"] + assert controller.ctx.device.device_id == UDID + + +@pytest.mark.asyncio +async def test_cancelled_acquire_is_drained_and_never_connects(monkeypatch): + """Cancelling while lease.acquire runs must wait for the gated thread. + + The acquisition eventually lands the real isolated lock; the observer + must not propagate until the thread drains, never connect, and leave no + owned lease behind. + """ + events: list[str] = [] + controller = _controller(events) + gate = threading.Event() + entered = threading.Event() + real_acquire = DeviceExecutionLock.acquire + + def gated_acquire(self, blocking=True): + entered.set() + gate.wait(timeout=10) + return real_acquire(self, blocking=blocking) + + monkeypatch.setattr(DeviceExecutionLock, "acquire", gated_acquire) + task = asyncio.create_task(observe_ios_controller(controller)) + try: + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) # let the cancellation land + assert not task.done() # still draining the gated acquire thread + finally: + gate.set() + with pytest.raises(asyncio.CancelledError): + await task + # Cleanup released the landed lease; connect was never attempted. + assert events == ["resolve"] + assert _ios_owner() is None diff --git a/tests/unit/sdk/test_agent_config_builder_tuning.py b/tests/unit/sdk/test_agent_config_builder_tuning.py index 6b808c9d..7aa6e08a 100644 --- a/tests/unit/sdk/test_agent_config_builder_tuning.py +++ b/tests/unit/sdk/test_agent_config_builder_tuning.py @@ -13,19 +13,19 @@ def test_with_verification_level_off_disables_the_checker(): - cfg = AgentConfigBuilder().with_verification_level("off").build() + cfg = AgentConfigBuilder().with_verification_level("off").build(validate_profiles=False) assert cfg.disable_checker is True def test_with_verification_level_final_keeps_factory_layering(): - cfg = AgentConfigBuilder().with_verification_level("final").build() + cfg = AgentConfigBuilder().with_verification_level("final").build(validate_profiles=False) assert cfg.disable_checker is False assert cfg.disable_midway_checks is True assert cfg.disable_final_check is False def test_with_verification_level_checkpoints_enables_midway_checks(): - cfg = AgentConfigBuilder().with_verification_level("checkpoints").build() + cfg = AgentConfigBuilder().with_verification_level("checkpoints").build(validate_profiles=False) assert cfg.disable_checker is False assert cfg.disable_midway_checks is False assert cfg.disable_final_check is False @@ -34,7 +34,7 @@ def test_with_verification_level_checkpoints_enables_midway_checks(): def test_with_verification_level_strict_applies_the_full_preset(): - cfg = AgentConfigBuilder().with_verification_level("strict").build() + cfg = AgentConfigBuilder().with_verification_level("strict").build(validate_profiles=False) preset = VERIFICATION_LEVEL_PRESETS["strict"] assert cfg.disable_checker is False assert cfg.disable_midway_checks is False @@ -46,7 +46,7 @@ def test_with_verification_level_strict_applies_the_full_preset(): def test_with_verification_level_is_case_and_whitespace_insensitive(): - cfg = AgentConfigBuilder().with_verification_level(" STRICT ").build() + cfg = AgentConfigBuilder().with_verification_level(" STRICT ").build(validate_profiles=False) assert cfg.assert_failure_policy == "halt" @@ -55,12 +55,12 @@ def test_with_verification_level_rejects_unknown_preset(): with pytest.raises(ValueError, match="Unknown verification level"): builder.with_verification_level("paranoid") # The builder is untouched by a rejected call. - assert builder.build().disable_checker is False + assert builder.build(validate_profiles=False).disable_checker is False def test_with_verification_level_matches_the_shared_preset_table(): for level in VERIFICATION_LEVEL_PRESETS: - cfg = AgentConfigBuilder().with_verification_level(level).build() + cfg = AgentConfigBuilder().with_verification_level(level).build(validate_profiles=False) overrides = checker_overrides_for_level(level) assert cfg.disable_checker is (not overrides["enabled"]) if "midway_checks" in overrides: @@ -68,7 +68,12 @@ def test_with_verification_level_matches_the_shared_preset_table(): def test_with_verification_level_is_chainable_and_explicit_checker_wins(): - cfg = AgentConfigBuilder().with_verification_level("strict").with_checker(enabled=False).build() + cfg = ( + AgentConfigBuilder() + .with_verification_level("strict") + .with_checker(enabled=False) + .build(validate_profiles=False) + ) assert cfg.disable_checker is True # The preset's other fields survive the later master switch. assert cfg.assert_failure_policy == "halt" @@ -78,7 +83,7 @@ def test_with_pro_config_carries_both_knobs(): cfg = ( AgentConfigBuilder() .with_pro_config(verification_level="checkpoints", explorer_mode="ultra") - .build() + .build(validate_profiles=False) ) assert cfg.disable_checker is False assert cfg.disable_midway_checks is False @@ -88,13 +93,17 @@ def test_with_pro_config_carries_both_knobs(): def test_with_pro_config_checker_switch_wins_over_verification_level(): - cfg = AgentConfigBuilder().with_pro_config(verification_level="strict", checker=False).build() + cfg = ( + AgentConfigBuilder() + .with_pro_config(verification_level="strict", checker=False) + .build(validate_profiles=False) + ) assert cfg.disable_checker is True def test_with_explorer_pro_mode_is_the_explorer_mode_knob(monkeypatch): monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False) - cfg = AgentConfigBuilder().with_explorer(pro_mode="pro").build() + cfg = AgentConfigBuilder().with_explorer(pro_mode="pro").build(validate_profiles=False) assert cfg.explorer.pro_mode == "pro" assert cfg.get_explorer_version(agent_name=None, profile="pro") == "pro" # The shipped per-agent override is empty, so the Pro agents follow the knob. @@ -104,7 +113,7 @@ def test_with_explorer_pro_mode_is_the_explorer_mode_knob(monkeypatch): def test_with_flash_config_explorer_mode_is_the_flash_profile_knob(monkeypatch): monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False) - cfg = AgentConfigBuilder().with_flash_config(explorer_mode="pro").build() + cfg = AgentConfigBuilder().with_flash_config(explorer_mode="pro").build(validate_profiles=False) assert cfg.flash.explorer_mode == "pro" assert cfg.explorer.flash_mode == "pro" assert cfg.get_explorer_version(agent_name="flash") == "pro" @@ -117,14 +126,14 @@ def test_with_explorer_versions_is_an_advanced_per_agent_override(monkeypatch): cfg = ( AgentConfigBuilder() .with_explorer(pro_mode="flash", versions={"validator": "ultra"}) - .build() + .build(validate_profiles=False) ) assert cfg.get_explorer_version(agent_name="validator") == "ultra" assert cfg.get_explorer_version(agent_name="operator") == "flash" def test_ios_workspace_path_defaults_to_none_and_survives_build(): - cfg = AgentConfigBuilder().build() + cfg = AgentConfigBuilder().build(validate_profiles=False) assert cfg.ios_workspace_path is None @@ -132,9 +141,9 @@ def test_with_ios_workspace_accepts_str_and_path_and_clears(tmp_path): project = tmp_path / "My App.xcodeproj" project.mkdir() builder = AgentConfigBuilder() - cfg = builder.with_ios_workspace(str(project)).build() + cfg = builder.with_ios_workspace(str(project)).build(validate_profiles=False) assert cfg.ios_workspace_path == project - cfg = builder.with_ios_workspace(None).build() + cfg = builder.with_ios_workspace(None).build(validate_profiles=False) assert cfg.ios_workspace_path is None @@ -155,3 +164,30 @@ def test_for_ios_simulator_workspace_kwarg_and_omission(tmp_path): .build(validate_profiles=False) ) assert cfg.ios_workspace_path == project + + +def test_for_ios_device_and_alias_preserve_device_and_workspace(tmp_path): + """The canonical name and the simulator-era alias carry identical config.""" + from artemis.context import DevicePlatform + + project = tmp_path / "App.xcworkspace" + project.mkdir() + udid = "AAAA-1111-0000" + canonical = ( + AgentConfigBuilder() + .for_ios_device(udid, workspace_path=project) + .build(validate_profiles=False) + ) + alias = ( + AgentConfigBuilder() + .for_ios_simulator(udid, workspace_path=project) + .build(validate_profiles=False) + ) + for cfg in (canonical, alias): + assert cfg.device_platform == DevicePlatform.IOS + assert cfg.device_id == udid + assert cfg.ios_workspace_path == project + # The default iOS target stays the lazy 'booted' selector. + default = AgentConfigBuilder().for_ios_device().build(validate_profiles=False) + assert default.device_platform == DevicePlatform.IOS + assert default.device_id == "booted" diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py index 25afe13c..79cea42f 100644 --- a/tests/unit/test_ios_integration.py +++ b/tests/unit/test_ios_integration.py @@ -110,7 +110,9 @@ def test_factory_selects_ios_without_creating_adb(native_driver, monkeypatch): monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) context = ios_context() assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted", width=1170, height=2532, workspace_path=None) + native_driver.constructor.assert_called_once_with( + device_id="booted", width=1170, height=2532, workspace_path=None + ) assert context.adb_client is None @@ -125,13 +127,17 @@ def test_factory_forwards_configured_ios_workspace(native_driver, tmp_path): .build(validate_profiles=False) ) assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted", width=1170, height=2532, workspace_path=project) + native_driver.constructor.assert_called_once_with( + device_id="booted", width=1170, height=2532, workspace_path=project + ) def test_factory_tolerates_missing_agent_config(native_driver): context = ios_context() assert create_driver(context) is native_driver - native_driver.constructor.assert_called_once_with(device_id="booted", width=1170, height=2532, workspace_path=None) + native_driver.constructor.assert_called_once_with( + device_id="booted", width=1170, height=2532, workspace_path=None + ) def test_macos_host_keeps_android_default(monkeypatch): @@ -422,9 +428,7 @@ def test_cli_approval_error_exits_2_with_guidance_panel(monkeypatch): ) ), ) - result = CliRunner().invoke( - app, ["run", "--platform", "ios", "--standalone", "Open Settings"] - ) + result = CliRunner().invoke(app, ["run", "--platform", "ios", "--standalone", "Open Settings"]) assert result.exit_code == 2, result.output assert "Xcode Approval Required" in result.output assert "Always Allow" in result.output diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py index f7923bd5..b0e14d62 100644 --- a/tests/unit/test_ios_parity.py +++ b/tests/unit/test_ios_parity.py @@ -934,3 +934,166 @@ def test_ios_press_key_vocabulary_passes_the_operator_gate(): ios_supported = {"enter", "home", "power", "volume_up", "volume_down", "app_switch"} assert ios_supported <= {key.lower() for key in SUPPORTED_PRESS_KEYS} + + +# --------------------------------------------------------------------------- # +# lazy MCP controller selection +# --------------------------------------------------------------------------- # + + +def test_lazy_mcp_ios_controller_uses_ios_context_and_isolated_cache(monkeypatch): + """The lazy MCP path builds an iOS DeviceContext without a native session.""" + from types import SimpleNamespace + + from artemis.context import DevicePlatform + from artemis.mcp import adb_server + + captured = {} + + def fake_controller(ctx): + captured["ctx"] = ctx + return SimpleNamespace(ctx=ctx) + + monkeypatch.setattr(adb_server, "_CONTROLLERS", {}) + monkeypatch.setattr(adb_server, "_GLOBAL_CONTROLLER", None) + monkeypatch.setattr(adb_server, "UnifiedMobileController", fake_controller) + monkeypatch.delenv("ARTEMIS_DEVICE_ID", raising=False) + monkeypatch.delenv("ADB_DEVICE_SERIAL", raising=False) + + controller = adb_server._get_controller(target_platform="ios") + assert captured["ctx"].device.mobile_platform == DevicePlatform.IOS + assert captured["ctx"].device.device_id == "booted" + assert adb_server._CONTROLLERS == {"ios:booted": controller} + + again = adb_server._get_controller(target_platform="ios") + assert again is controller + + # iOS lookups never populate the Android global slot or un-namespaced keys. + assert adb_server._GLOBAL_CONTROLLER is None + assert all(key.startswith("ios:") for key in adb_server._CONTROLLERS) + + +def test_lazy_mcp_ios_controller_never_reads_adb_device_serial(monkeypatch): + """ADB_DEVICE_SERIAL is Android-only: it must not seed an iOS target.""" + from types import SimpleNamespace + + from artemis.context import DevicePlatform + from artemis.mcp import adb_server + + captured = {} + + def fake_controller(ctx): + captured["ctx"] = ctx + return SimpleNamespace(ctx=ctx) + + monkeypatch.setattr(adb_server, "_CONTROLLERS", {}) + monkeypatch.setattr(adb_server, "_GLOBAL_CONTROLLER", None) + monkeypatch.setattr(adb_server, "UnifiedMobileController", fake_controller) + monkeypatch.delenv("ARTEMIS_DEVICE_ID", raising=False) + monkeypatch.setenv("ADB_DEVICE_SERIAL", "android-only") + + controller = adb_server._get_controller(target_platform="ios") + assert captured["ctx"].device.mobile_platform == DevicePlatform.IOS + assert captured["ctx"].device.device_id == "booted" + assert set(adb_server._CONTROLLERS) == {"ios:booted"} + assert adb_server._GLOBAL_CONTROLLER is None + + +# --------------------------------------------------------------------------- # +# iOS pool: "booted" selector and name ambiguity +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_pins_the_unique_booted_sim(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert await pool.select_device_async(preferred_serial="booted") == "AAAA-1111" + assert await pool.validate_explicit_serial_async("booted") is None + assert pool.validate_explicit_serial("booted") is None + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_tolerates_case_and_whitespace(monkeypatch): + """' Booted ' / 'BOOTED' normalize to the reserved selector, not a name.""" + pool = _pool_with_devices(monkeypatch) + assert await pool.select_device_async(preferred_serial=" Booted ") == "AAAA-1111" + assert await pool.validate_explicit_serial_async(" BOOTED ") is None + assert pool.validate_explicit_serial("Booted") is None + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_fails_open_on_ambiguity(monkeypatch): + """Zero or multiple booted sims keep the literal so the driver's guidance fires.""" + two_booted = [dict(SIM_LIST[0]), {**SIM_LIST[1], "state": "Booted"}] + pool = _pool_with_devices(monkeypatch, two_booted) + assert await pool.select_device_async(preferred_serial="booted") == "booted" + rejection = await pool.validate_explicit_serial_async("booted") + assert rejection is not None and "exactly one booted" in rejection + assert "UDID" in rejection + + no_booted = [{**SIM_LIST[0], "state": "Shutdown"}, dict(SIM_LIST[1])] + pool = _pool_with_devices(monkeypatch, no_booted) + assert await pool.select_device_async(preferred_serial="booted") == "booted" + rejection = await pool.validate_explicit_serial_async("booted") + assert rejection is not None and "0 found" in rejection + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_fails_open_on_enumeration_error(monkeypatch): + pool = IosDevicePool() + import importlib + + module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=None)) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=None)) + assert await pool.select_device_async(preferred_serial="booted") == "booted" + assert await pool.validate_explicit_serial_async("booted") is None + + +def test_ios_pool_rejects_duplicate_device_names(monkeypatch): + """Two usable devices sharing a name must be rejected, not first-matched.""" + twins = [ + {**SIM_LIST[0], "name": "Office iPhone"}, + {**SIM_LIST[1], "name": "Office iPhone", "state": "Booted"}, + ] + pool = _pool_with_devices(monkeypatch, twins) + rejection = pool.validate_explicit_serial("Office iPhone") + assert rejection is not None and "UDID" in rejection + # UDID pins resolve fine. + assert pool.validate_explicit_serial("AAAA-1111") is None + + +# --------------------------------------------------------------------------- # +# Admin capabilities endpoint (thin-SDK iOS wire contract) +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_admin_capabilities_endpoint_is_static_and_advertises_ios(monkeypatch): + """GET /api/v1/capabilities advertises 'platform.ios' without probing.""" + import importlib + + tasks_router = importlib.import_module("apps.admin_console.routers.tasks") + pool_module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr( + pool_module, + "list_core_devices", + AsyncMock(side_effect=AssertionError("device discovery must not run")), + ) + monkeypatch.setattr( + pool_module, + "list_ios_simulators", + AsyncMock(side_effect=AssertionError("device discovery must not run")), + ) + + response = await tasks_router.get_capabilities() + + assert response["api_version"] == "1" + assert "platform.ios" in response["features"] + assert { + "tasks.submit", + "tasks.get", + "tasks.stop", + "devices.list", + "system.readiness", + }.issubset(response["features"]) diff --git a/tests/unit/utils/test_video_manifest.py b/tests/unit/utils/test_video_manifest.py index f0e0009f..ca75365f 100644 --- a/tests/unit/utils/test_video_manifest.py +++ b/tests/unit/utils/test_video_manifest.py @@ -130,3 +130,97 @@ async def test_manifest_returns_none_without_valid_segments(tmp_path): ): assert await write_recording_manifest(tmp_path, [tmp_path / "nope.mp4"]) is None assert not (tmp_path / "recording.json").exists() + + +@pytest.mark.asyncio +async def test_manifest_forwards_probe_timeout_only_when_supplied(tmp_path): + """iOS callers bound the probe; default callers keep the old signature.""" + import artemis.utils.video as video + + seg = tmp_path / "recording.mp4" + seg.write_bytes(b"mp4") + seen: list = [] + + async def probe(path, timeout_seconds=None): + seen.append(timeout_seconds) + return {"duration": 1.0, "width": 1080, "height": 1920} + + with patch("artemis.utils.video.probe_video_segment", AsyncMock(side_effect=probe)): + await write_recording_manifest(tmp_path, [seg], probe_timeout_seconds=30.0) + assert seen == [30.0] + + +@pytest.mark.asyncio +async def test_probe_timeout_kills_and_drains_the_ffprobe_child(tmp_path, monkeypatch): + """A bounded probe must reap its owned ffprobe child on timeout.""" + import asyncio + + import artemis.utils.video as video + + class _Proc: + def __init__(self): + self.returncode = None + self.killed = False + self.drained = False + + def kill(self): + self.killed = True + self.returncode = -9 + + async def communicate(self): + if self.killed: + self.drained = True + return b"", b"" + await asyncio.Event().wait() # never answers in time + return b"", b"" + + proc = _Proc() + + async def fake_exec(*args, **kwargs): + return proc + + monkeypatch.setattr(video.asyncio, "create_subprocess_exec", fake_exec) + seg = tmp_path / "seg.mp4" + seg.write_bytes(b"mp4") + with pytest.raises(TimeoutError): + await video.probe_video_segment(seg, timeout_seconds=0.05) + assert proc.killed and proc.drained + + +@pytest.mark.asyncio +async def test_probe_cancellation_reaps_the_child_before_propagating(tmp_path, monkeypatch): + import asyncio + + import artemis.utils.video as video + + class _Proc: + def __init__(self): + self.returncode = None + self.killed = False + self.drained = False + + def kill(self): + self.killed = True + self.returncode = -9 + + async def communicate(self): + if self.killed: + self.drained = True + return b"", b"" + await asyncio.Event().wait() + return b"", b"" + + proc = _Proc() + + async def fake_exec(*args, **kwargs): + return proc + + monkeypatch.setattr(video.asyncio, "create_subprocess_exec", fake_exec) + seg = tmp_path / "seg.mp4" + seg.write_bytes(b"mp4") + task = asyncio.create_task(video.probe_video_segment(seg, timeout_seconds=30.0)) + await asyncio.sleep(0.02) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert proc.killed and proc.drained From 19e04cc64bb35213f3b7207867b88b9ed280a477 Mon Sep 17 00:00:00 2001 From: James Lee Date: Wed, 7 Oct 2026 20:31:26 +0900 Subject: [PATCH 30/32] fix: physical WDA session attach on real devices Live acceptance on iPhone 15 Pro surfaced two defects that blocked every normally-named physical device: - WDA device_info reports the product family name ("iPhone"), not the personalized devicectl name ("Jane's iPhone"). The identity guard now rejects only a conflicting *specific* name instead of any inequality. - A driver-launched WDA runner auto-creates a session on startup; the foreign-session refusal then deadlocked connect(). open_session gains adopt_existing, enabled only when the driver owns the runner process; discovered/user-provisioned endpoints keep the refusal. Verified live: WDA session, 319-element hierarchy, tap/swipe/home, WDA and devicectl screenshots, pid-verified terminate, MP4 recording, clean disconnect with zero orphaned processes. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/drivers/ios/physical_driver.py | 37 ++++++++++++++++----- artemis/drivers/ios/wda.py | 16 +++++++-- tests/unit/drivers/test_ios_physical.py | 44 ++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py index 1566a8cc..8ab5d428 100644 --- a/artemis/drivers/ios/physical_driver.py +++ b/artemis/drivers/ios/physical_driver.py @@ -179,15 +179,28 @@ async def connect(self) -> None: try: self._wda = await self._ensure_wda() # The reachable WDA endpoint must belong to THIS device before - # we open a session or send input: names corroborate identity - # (uuid is identifierForVendor, not the UDID). + # we open a session or send input. WDA reports the product + # family name ("iPhone"), not the personalized devicectl name + # ("Dana's iPhone"), and uuid is identifierForVendor — so a + # *specific* conflicting name is the wrong-device signal; + # generic family names are accepted. info = await self._wda.device_info() - expected_name = candidate.get("name") or "" - if ( - info.get("isSimulator") is not False - or not expected_name - or info.get("name") != expected_name - ): + expected_name = (candidate.get("name") or "").strip() + wda_name = str(info.get("name") or "").strip() + generic_names = { + "iphone", + "ipad", + "ipod touch", + "apple watch", + "apple tv", + } + name_conflict = ( + expected_name + and wda_name + and wda_name.lower() not in generic_names + and wda_name != expected_name + ) + if info.get("isSimulator") is not False or name_conflict: raise RuntimeError( f"The WebDriverAgent at {self._wda.base_url} does not report " f"the selected physical device {self._device_id} " @@ -196,7 +209,13 @@ async def connect(self) -> None: "Point ARTEMIS_IOS_WDA_URL at a WDA server running on the " "selected device." ) - self._session_key = await self._wda.open_session() + # A session auto-created by a runner WE launched is ours to + # adopt; an endpoint discovered via env/probe keeps the + # foreign-session refusal. + runner_owned = ( + self._wda_runner_pid is not None or self._wda_test_process is not None + ) + self._session_key = await self._wda.open_session(adopt_existing=runner_owned) await self.get_screen_data(skip_settling=True) connected = True finally: diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py index ede29c9e..27515b00 100644 --- a/artemis/drivers/ios/wda.py +++ b/artemis/drivers/ios/wda.py @@ -167,7 +167,7 @@ async def device_info(self, timeout: float = 10.0) -> dict[str, Any]: ) return value - async def _create_owned_session(self) -> str: + async def _create_owned_session(self, adopt_existing: bool) -> str: """Status preflight, POST /session, and ID assignment as one unit.""" if self._session_id: return self._session_id @@ -176,6 +176,11 @@ async def _create_owned_session(self) -> str: raise WdaUnavailableError(f"WebDriverAgent at {self._base} did not answer /status.") active = status.get("sessionId") if isinstance(active, str) and active: + if adopt_existing: + # The driver launched this runner itself, so its auto-created + # session is ours to reuse — not a foreign client to protect. + self._session_id = active + return active raise RuntimeError( "Refusing to replace the active WebDriverAgent session " f"({active}), which this client does not own — POST /session " @@ -201,14 +206,19 @@ async def _create_owned_session(self) -> str: self._session_id = session_id return session_id - async def open_session(self) -> str: + async def open_session(self, adopt_existing: bool = False) -> str: """Create a WDA session, refusing to take over a foreign one. + ``adopt_existing=True`` reuses the active session reported by + ``/status``; only safe when the caller owns the WDA runner process + (auto-launched runners create a session on startup). Foreign or + user-provisioned endpoints must keep the default refusal. + Cancellation-safe: when the caller is cancelled while creation is in flight, the request is drained and any session it produced is closed before the CancelledError propagates. """ - create_task = asyncio.ensure_future(self._create_owned_session()) + create_task = asyncio.ensure_future(self._create_owned_session(adopt_existing)) try: return await asyncio.shield(create_task) except asyncio.CancelledError: diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py index 43c30f65..0546c80e 100644 --- a/tests/unit/drivers/test_ios_physical.py +++ b/tests/unit/drivers/test_ios_physical.py @@ -281,7 +281,8 @@ def _make_png(width: int = 300, height: int = 600) -> bytes: async def device_info(self, timeout=10.0): return {"name": self.device_name, "isSimulator": self.is_simulator} - async def open_session(self): + async def open_session(self, adopt_existing=False): + self.adopt_requested = adopt_existing self.opened_sessions += 1 self.session_id = "wda-session" return self.session_id @@ -1290,6 +1291,17 @@ async def test_open_session_refuses_a_foreign_active_session(responder): assert client.session_id is None +@pytest.mark.asyncio +async def test_open_session_adopts_active_session_when_allowed(responder): + """A driver-launched runner's auto-session is adopted, not replaced.""" + responder.set("/status", {"value": {"ready": True}, "sessionId": "auto-77"}) + client = WdaClient("http://wda.test:8100") + assert await client.open_session(adopt_existing=True) == "auto-77" + assert client.session_id == "auto-77" + methods = {m for m, _url in responder.requests} + assert methods == {"GET"} # adopted via /status — no POST /session + + @pytest.mark.asyncio async def test_open_session_reuses_its_own_session(responder): responder.set("/status", {"value": {"ready": True}, "sessionId": None}) @@ -1416,6 +1428,36 @@ async def test_connect_rejects_a_simulator_wda(driver, monkeypatch): assert client.opened_sessions == 0 +@pytest.mark.asyncio +async def test_connect_accepts_generic_wda_family_name(driver, monkeypatch): + # WDA reports the product family ("iPhone"), not the personalized + # devicectl name ("Jane's iPhone") — that is the same device. + client = _FakeWda(device_name="iPhone") + _connect_stubs(driver, monkeypatch, client) + await driver.connect() + assert client.opened_sessions == 1 + await driver.disconnect() + + +@pytest.mark.asyncio +async def test_connect_adopts_session_when_we_launched_the_runner(driver, monkeypatch): + client = _FakeWda() + _connect_stubs(driver, monkeypatch, client) + driver._wda_runner_pid = 4242 # _ensure_wda launched this runner + await driver.connect() + assert client.adopt_requested is True + await driver.disconnect() + + +@pytest.mark.asyncio +async def test_connect_refuses_adoption_for_discovered_endpoints(driver, monkeypatch): + client = _FakeWda() + _connect_stubs(driver, monkeypatch, client) + await driver.connect() + assert client.adopt_requested is False + await driver.disconnect() + + # --- Disconnect cleanup ----------------------------------------------------- # From 65c3412ffaceb51fac5ee6d15f4b43162f9b4e11 Mon Sep 17 00:00:00 2001 From: James Lee Date: Thu, 8 Oct 2026 03:54:21 +0900 Subject: [PATCH 31/32] docs: clarify Xcode agent grant expiry and locked-device WDA behavior Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docs/ios.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/ios.md b/docs/ios.md index eab43cfb..0ddbb1b1 100644 --- a/docs/ios.md +++ b/docs/ios.md @@ -201,6 +201,19 @@ folder, or an expiring grant can require approval again. The Always/persistent choice belongs to you and Xcode; Artemis requests only scoped approval for its interpreter and the folder you select and never enables global access. +Grant duration is Xcode's decision, not Artemis's: Xcode binds each agent +approval to the requesting binary's code signature. A binary signed with a +real signing identity can hold a persistent **Always Allow** grant, but an +unsigned or adhoc-signed interpreter — the common case for `python3` from uv, +Homebrew, or a virtual environment — receives a grant that expires after +roughly 24 hours, and upgrading or replacing that interpreter binary requires +re-approval either way. `xcrun mcp-server status` lists every grant with its +expiry. On headless or CI machines an administrator can run +`sudo xcrun mcp-server enable` to keep the service reachable while Xcode is +closed; per-agent approval still applies unless the administrator also passes +`--unsafe-always-allow-all-agents`, which lets any local process drive +reachable projects and should stay disabled outside CI. + ## Embedded Python SDK Configure iOS through the embedded SDK's builder: @@ -385,10 +398,12 @@ the timeline rather than stretching recorded frames. `*WebDriverAgent*` was found to launch. Build and install the WDA runner (one-time steps above), or point `ARTEMIS_IOS_WDA_URL` at a forwarded or LAN-reachable server. -- **WDA runner launches but never answers:** the runner may be crashing on - launch — check provisioning (`get-task-allow`, matching certificate) and - whether a free-team profile expired; also try forwarding port 8100 - (`iproxy 8100 8100`) and setting `ARTEMIS_IOS_WDA_URL`. +- **WDA runner launches but never answers:** unlock the device first — UI + Automation, and therefore WDA's HTTP server, cannot start while it is + locked. Otherwise the runner may be crashing on launch — check provisioning + (`get-task-allow`, matching certificate) and whether a free-team profile + expired; also try forwarding port 8100 (`iproxy 8100 8100`) and setting + `ARTEMIS_IOS_WDA_URL`. - **A queued iOS task runs on the wrong surface:** confirm the submission carried `platform: "ios"` (CLI `--platform ios`, web request `platform`, or the MCP `platform` argument); tasks default to Android. From b199e0e37e9889e7f9cfe18b7b93423752c1203e Mon Sep 17 00:00:00 2001 From: James Lee Date: Fri, 9 Oct 2026 04:19:06 +0900 Subject: [PATCH 32/32] fix: honor custom provider endpoints and label image payloads correctly Three compatibility bugs surfaced while driving a real iOS task through the admin console against a strict OpenAI-compatible local endpoint: - strip_json_comments stripped "//" inside string literals, corrupting URL values such as "api_base": "http://127.0.0.1:8080/v1". Replaced the regexes with a string/escape-aware scanner that removes comments only outside string literals and preserves newlines for stable error positions. - _resolve_endpoint dropped api_base/api_key/max_tokens/timeout because the LLM schema had no fields for them, so custom providers silently fell back to the environment OpenAI base URL. Added the fields and pass them through to ModelEndpoint (fallback configs resolve independently). - Screenshot bytes are PNG on both platforms yet every OpenAI-style payload labeled them data:image/jpeg; strict validators reject the mislabeled request. Added artemis.utils.image_mime sniffing helpers (PNG/JPEG magic, PNG project-default fallback) and applied them to all 21 data-URI/Part/ImageContent sites. Covered by new tests for the JSONC scanner, endpoint resolution, and mime detection; deterministic suite passes (2925 tests). Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- artemis/agents/checker/checker.py | 3 +- artemis/agents/diagnoser/diagnoser.py | 3 +- artemis/agents/explorer/native_runner.py | 7 +- artemis/agents/explorer/universal_runner.py | 7 +- artemis/agents/flash/runner.py | 5 +- artemis/agents/flash/summarizer.py | 5 +- .../agents/image_processor/image_processor.py | 7 +- .../agents/object_detector/object_detector.py | 4 +- artemis/agents/operator/prompts.py | 3 +- artemis/agents/outputter/outputter.py | 3 +- artemis/agents/planner/planner.py | 3 +- .../agents/validator/precondition_pixel.py | 5 +- artemis/agents/validator/tool_declarations.py | 3 +- .../video_analyzer/native_coordinator.py | 6 +- .../agents/video_analyzer/universal_engine.py | 3 +- artemis/mcp/action_server.py | 3 +- artemis/services/llm.py | 3 + artemis/tools/committee_tool.py | 5 +- artemis/tools/explorer_tool.py | 3 +- artemis/tools/history/screenshot.py | 3 +- artemis/utils/image_mime.py | 42 +++++++++ tests/unit/agents/test_explorer_tiers.py | 2 +- tests/unit/agents/test_outputter.py | 2 +- tests/unit/services/test_llm_endpoint.py | 94 +++++++++++++++++++ tests/unit/test_jsonc_comments.py | 86 +++++++++++++++++ tests/unit/tools/test_explorer_tool.py | 2 +- tests/unit/utils/test_image_mime.py | 60 ++++++++++++ third_party/mobile_use/config/llm.py | 4 + third_party/mobile_use/utils/file.py | 45 ++++++++- 29 files changed, 382 insertions(+), 39 deletions(-) create mode 100644 artemis/utils/image_mime.py create mode 100644 tests/unit/services/test_llm_endpoint.py create mode 100644 tests/unit/test_jsonc_comments.py create mode 100644 tests/unit/utils/test_image_mime.py diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index 3c081177..93cc2b41 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -67,6 +67,7 @@ ) from artemis.utils.visualization import format_minimal_list_with_elements from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -844,7 +845,7 @@ async def run_final_check( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index f77113fc..b87e7e7f 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -54,6 +54,7 @@ get_recent_subgoal_hashes, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -232,7 +233,7 @@ def _build_initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) diff --git a/artemis/agents/explorer/native_runner.py b/artemis/agents/explorer/native_runner.py index 4b29bd1b..afe98442 100644 --- a/artemis/agents/explorer/native_runner.py +++ b/artemis/agents/explorer/native_runner.py @@ -28,6 +28,7 @@ from collections.abc import Awaitable, Callable import json import os +from pathlib import Path import time from typing import TYPE_CHECKING, Any @@ -45,6 +46,7 @@ ) from artemis.services.llm import _record_llm_event, _record_llm_retry from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -188,12 +190,13 @@ async def get_image_part(file_path: str) -> types.Part: return types.Part( file_data=types.FileData( file_uri=file_ref.uri, - mime_type=file_ref.mime_type or "image/jpeg", + mime_type=file_ref.mime_type + or image_mime_type(Path(file_path).read_bytes()), ) ) with open(file_path, "rb") as f: img_bytes = f.read() - return types.Part.from_bytes(data=img_bytes, mime_type="image/jpeg") + return types.Part.from_bytes(data=img_bytes, mime_type=image_mime_type(img_bytes)) return get_image_part diff --git a/artemis/agents/explorer/universal_runner.py b/artemis/agents/explorer/universal_runner.py index c5b5bc89..48f0dbe2 100644 --- a/artemis/agents/explorer/universal_runner.py +++ b/artemis/agents/explorer/universal_runner.py @@ -36,6 +36,7 @@ from artemis.graph.state import State from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -172,7 +173,7 @@ def _build_universal_messages( }, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] @@ -266,9 +267,7 @@ def _build_tool_images_message(self, images: list[tuple[str, str]]) -> HumanMess continue if tool_name not in tool_names: tool_names.append(tool_name) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if not blocks: return None header = { diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index a875f3ce..61cdb852 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -95,6 +95,7 @@ parse_swipe_parameters, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -373,9 +374,7 @@ def _build_tail( if img_bytes: img_b64 = base64.b64encode(img_bytes).decode("utf-8") blocks.append({"type": "text", "text": "--- Current Screenshot ---"}) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if xml_list: blocks.append({"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n{xml_list}"}) ephemeral: list[int] = [] diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 5b8a3cf9..c577717d 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -41,6 +41,7 @@ from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -430,7 +431,7 @@ async def _attempt(self, key: JobKey) -> bool: } ) content_blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_pre}"}} + {"type": "image_url", "image_url": {"url": image_data_uri(b64_pre)}} ) if post_bytes: @@ -449,7 +450,7 @@ async def _attempt(self, key: JobKey) -> bool: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_post}"}, + "image_url": {"url": image_data_uri(b64_post)}, } ) diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 4ce52698..2ee22724 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -35,6 +35,7 @@ from artemis.services.llm import get_llm from artemis.utils.python_executor import PythonExecutor from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -118,7 +119,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: {"type": "text", "text": "Here is the target image. Begin writing your code."}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] ), @@ -186,9 +187,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: image_blocks.append( { "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{b64_data}" - }, + "image_url": {"url": image_data_uri(b64_data)}, } ) image_blocks.append({"type": "text", "text": f"\n{label}\n"}) diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index b2acb4f5..dae8f8e8 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -26,6 +26,7 @@ from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -107,7 +108,7 @@ async def _run_object_detection( image_bytes: bytes | str | Path | None = None, queries: list[str] | None = None, templates: list[str] | None = None, - mime_type: str = "image/jpeg", + mime_type: str | None = None, global_timeout: float = 30.0, image_path: str | Path | None = None, ) -> dict: @@ -120,6 +121,7 @@ async def _run_object_detection( image_data = Path(target_img).read_bytes() else: image_data = target_img + mime_type = mime_type or image_mime_type(image_data) queries = queries or [] templates = templates or ["Point to the following objects: {labels_str}"] diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index d7eb3dc2..d74659d0 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -40,6 +40,7 @@ from artemis.agents.prompt_assembly import render_tool_enum, resolve_available from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER +from artemis.utils.image_mime import image_data_uri @lru_cache(maxsize=1) @@ -392,7 +393,7 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte builder.add_human_content( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index 9b75b427..88259b63 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -46,6 +46,7 @@ from artemis.utils.task_tree import get_active_subgoal_hashes from third_party.mobile_use.utils.logger import get_logger from pydantic import BaseModel +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -274,7 +275,7 @@ def _initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 02be5c7c..e3374505 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -54,6 +54,7 @@ from artemis.utils.cython_compat import CyFunctionDetector from third_party.mobile_use.utils.decorators import agent_lifecycle_logging from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -279,7 +280,7 @@ async def __call__(self, state: State): human_message_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) messages = [ diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index 80a542fb..5cf32a4c 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -36,6 +36,7 @@ from artemis.services.llm import acomplete_structured from artemis.utils import visualization from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -120,12 +121,12 @@ def _build_messages( {"type": "text", "text": "[Image 1 (Reference)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{orig_b64}"}, + "image_url": {"url": image_data_uri(orig_b64)}, }, {"type": "text", "text": "[Image 2 (Current State)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{live_b64}"}, + "image_url": {"url": image_data_uri(live_b64)}, }, ] if state: diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index fba8dd75..b2d30b46 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -45,6 +45,7 @@ READ_NOTE_DOCSTRING, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -79,7 +80,7 @@ def to_langchain_tool_message(self) -> ToolMessage: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, + "image_url": {"url": image_data_uri(b64_img)}, } ) return ToolMessage( diff --git a/artemis/agents/video_analyzer/native_coordinator.py b/artemis/agents/video_analyzer/native_coordinator.py index 6cfab85d..b0994f89 100644 --- a/artemis/agents/video_analyzer/native_coordinator.py +++ b/artemis/agents/video_analyzer/native_coordinator.py @@ -34,6 +34,7 @@ from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.llm.google import normalize_usage from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -171,10 +172,11 @@ def _append_ledger_diff(analyzer, contents: list, last_ledger_index: int) -> int image_parts.append( types.Part.from_text(text=f"PROOF: {Path(screenshot_file).name}") ) + image_bytes = Path(screenshot_file).read_bytes() image_parts.append( types.Part.from_bytes( - data=Path(screenshot_file).read_bytes(), - mime_type="image/jpeg", + data=image_bytes, + mime_type=image_mime_type(image_bytes), ) ) lines.append(line) diff --git a/artemis/agents/video_analyzer/universal_engine.py b/artemis/agents/video_analyzer/universal_engine.py index 96ade3fe..4eaf8fe2 100644 --- a/artemis/agents/video_analyzer/universal_engine.py +++ b/artemis/agents/video_analyzer/universal_engine.py @@ -42,6 +42,7 @@ ) from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -159,7 +160,7 @@ async def _build_universal_user_blocks( user_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_str}"}, + "image_url": {"url": image_data_uri(b64_str)}, } ) diff --git a/artemis/mcp/action_server.py b/artemis/mcp/action_server.py index be32b95c..f810d132 100644 --- a/artemis/mcp/action_server.py +++ b/artemis/mcp/action_server.py @@ -46,6 +46,7 @@ from artemis.mcp.actuators.base import Actuator from artemis.mcp.observation import observe as observe_impl from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -112,7 +113,7 @@ async def observe_screen( ImageContent( type="image", data=_b64.b64encode(img_bytes).decode("utf-8"), - mimeType="image/jpeg", + mimeType=image_mime_type(img_bytes), ) ) return CallToolResult( diff --git a/artemis/services/llm.py b/artemis/services/llm.py index ec91e6fa..2a46b362 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -929,6 +929,9 @@ def _get_val(obj, attr, expected_type): return ModelEndpoint( provider=ModelProvider.from_string(provider_val), model_name=str(model_val), + api_base=_get_val(cfg, "api_base", str), + api_key=_get_val(cfg, "api_key", str), + max_tokens=_get_val(cfg, "max_tokens", int), temperature=_get_val(cfg, "temperature", (int, float)) or 0.0, timeout_seconds=_get_val(cfg, "timeout", (int, float)) or 60.0, thinking_budget=_get_val(cfg, "thinking_budget", int), diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 28735c90..e5dca5b5 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -55,6 +55,7 @@ get_recent_subgoal_hashes, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -283,7 +284,7 @@ async def run_agent_turn(llm, system_prompt, tools, agent_name): human_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) @@ -377,7 +378,7 @@ async def run_tool(tc): final_human_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/tools/explorer_tool.py b/artemis/tools/explorer_tool.py index 14a95b94..93fc7a72 100644 --- a/artemis/tools/explorer_tool.py +++ b/artemis/tools/explorer_tool.py @@ -62,6 +62,7 @@ from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.visualization import draw_dots from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -480,7 +481,7 @@ def render_operator_blocks( return text return [ {"type": "text", "text": text}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}, + {"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}, ] diff --git a/artemis/tools/history/screenshot.py b/artemis/tools/history/screenshot.py index 94f55d34..3548e7e6 100644 --- a/artemis/tools/history/screenshot.py +++ b/artemis/tools/history/screenshot.py @@ -32,6 +32,7 @@ from artemis.core.tool_failure import ToolFailure from artemis.utils.visualization import overlay_action_on_screenshot from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -65,7 +66,7 @@ def to_content_blocks(self) -> list[dict[str, Any]] | str: encoded = base64.b64encode(self.image_bytes).decode("utf-8") return [ {"type": "text", "text": self.description}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded}"}}, + {"type": "image_url", "image_url": {"url": image_data_uri(encoded)}}, ] diff --git a/artemis/utils/image_mime.py b/artemis/utils/image_mime.py new file mode 100644 index 00000000..20706ec0 --- /dev/null +++ b/artemis/utils/image_mime.py @@ -0,0 +1,42 @@ +"""Image media-type detection for LLM message payloads. + +Screenshots flow through the codebase as raw bytes or base64 strings and are +embedded in OpenAI-style ``image_url`` data URIs and provider ``Part`` objects. +Historically every site hardcoded ``image/jpeg`` while the bytes were actually +PNG, which strict OpenAI-compatible endpoints reject. Sniff the magic bytes +instead of trusting the label. +""" + +import base64 + +_PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +_JPEG_MAGIC = b"\xff\xd8" + + +def image_mime_type(data: bytes | str) -> str: + """Return ``image/png`` or ``image/jpeg`` for raw or base64 image bytes.""" + if isinstance(data, str): + # The base64 alphabet can't hold the magic bytes; decode just enough + # to sniff the header. + if data.startswith("data:"): + data = data.rsplit(",", 1)[-1] + chunk = data[:20] + chunk += "=" * (-len(chunk) % 4) + data = base64.b64decode(chunk) + if data[:2] == _JPEG_MAGIC: + return "image/jpeg" + if data[:8] == _PNG_MAGIC: + return "image/png" + # Every screenshot path in the project emits PNG; keep PNG as the honest + # default for unrecognized-but-image bytes. + return "image/png" + + +def image_data_uri(data: bytes | str) -> str: + """Return a ``data:image/;base64,`` URI with the correct type.""" + if isinstance(data, bytes): + mime = image_mime_type(data) + data = base64.b64encode(data).decode("utf-8") + else: + mime = image_mime_type(data) + return f"data:{mime};base64,{data}" diff --git a/tests/unit/agents/test_explorer_tiers.py b/tests/unit/agents/test_explorer_tiers.py index e8c400d3..1f84c6ca 100644 --- a/tests/unit/agents/test_explorer_tiers.py +++ b/tests/unit/agents/test_explorer_tiers.py @@ -251,7 +251,7 @@ async def test_universal_dispatch_appends_image_message(tmp_path): image_blocks = human.content[1:] assert len(image_blocks) == 2 # the missing file is skipped assert all(b["type"] == "image_url" for b in image_blocks) - assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert image_blocks[0]["image_url"]["url"].startswith("data:image/png;base64,") @pytest.mark.asyncio diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index 8def19fa..6cf135a7 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -151,7 +151,7 @@ def __init__(self, messages, initial_goal, operator_raw_data): assert "Step 1: Opened browser" in human_content[0]["text"] assert "Plan: Find capital" in human_content[0]["text"] assert human_content[1]["type"] == "image_url" - assert human_content[1]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data" + assert human_content[1]["image_url"]["url"] == "data:image/png;base64,mock_b64_data" @patch("artemis.agents.outputter.outputter.get_llm") diff --git a/tests/unit/services/test_llm_endpoint.py b/tests/unit/services/test_llm_endpoint.py new file mode 100644 index 00000000..82821999 --- /dev/null +++ b/tests/unit/services/test_llm_endpoint.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for endpoint field pass-through in ``_resolve_endpoint``. + +Custom/self-hosted providers declare ``api_base``/``api_key``/``max_tokens``/ +``timeout`` in JSONC config. Previously the LLM schema dropped them as extra +fields and ``_resolve_endpoint`` never read them, so a custom provider always +silently fell back to the environment OpenAI base URL. +""" + +from types import SimpleNamespace + +from artemis.services.llm import _resolve_endpoint +from third_party.mobile_use.config.llm import LLM, LLMWithFallback + + +def _ctx(llm_cfg): + return SimpleNamespace( + llm_config=SimpleNamespace( + get_agent=lambda _name: llm_cfg, + get_utils=lambda _name: llm_cfg, + ) + ) + + +def test_llm_model_accepts_endpoint_fields(): + llm = LLM( + provider="custom", + model="gemma4-e4b", + api_base="http://127.0.0.1:8080/v1", + api_key="k", + max_tokens=4096, + timeout=120.0, + ) + assert llm.api_base == "http://127.0.0.1:8080/v1" + assert llm.api_key == "k" + assert llm.max_tokens == 4096 + assert llm.timeout == 120.0 + + +def test_resolve_endpoint_passes_custom_fields(): + llm = LLM( + provider="custom", + model="gemma4-e4b", + api_base="http://127.0.0.1:8080/v1", + api_key="k", + max_tokens=4096, + timeout=120.0, + ) + ep = _resolve_endpoint(_ctx(llm), "planner") + assert ep.api_base == "http://127.0.0.1:8080/v1" + assert ep.api_key == "k" + assert ep.max_tokens == 4096 + assert ep.timeout_seconds == 120.0 + + +def test_resolve_endpoint_defaults_when_fields_absent(): + llm = LLM(provider="google", model="gemini-2.5-flash") + ep = _resolve_endpoint(_ctx(llm), "planner") + assert ep.api_base is None + assert ep.api_key is None + assert ep.max_tokens is None + assert ep.timeout_seconds == 60.0 + + +def test_resolve_endpoint_fallback_uses_fallback_endpoint_fields(): + llm = LLMWithFallback( + provider="google", + model="gemini-2.5-flash", + timeout=45.0, + fallback=LLM( + provider="custom", + model="gemma4-e4b", + api_base="http://127.0.0.1:8080/v1", + timeout=15.0, + ), + ) + primary = _resolve_endpoint(_ctx(llm), "planner") + assert primary.timeout_seconds == 45.0 + fallback = _resolve_endpoint(_ctx(llm), "planner", use_fallback=True) + assert fallback.api_base == "http://127.0.0.1:8080/v1" + assert fallback.timeout_seconds == 15.0 diff --git a/tests/unit/test_jsonc_comments.py b/tests/unit/test_jsonc_comments.py new file mode 100644 index 00000000..7b02a709 --- /dev/null +++ b/tests/unit/test_jsonc_comments.py @@ -0,0 +1,86 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for ``strip_json_comments``. + +The previous regex implementation treated ``//`` inside string literals as a +comment start, corrupting URL values like ``"api_base": "http://127.0.0.1:8080"``. +These tests pin comment removal to positions outside string literals. +""" + +import io +import json + +from third_party.mobile_use.utils.file import load_jsonc, strip_json_comments + + +def test_line_comment_removed_outside_string(): + text = '{\n // a comment\n "a": 1\n}' + assert json.loads(strip_json_comments(text)) == {"a": 1} + + +def test_url_scheme_not_treated_as_comment(): + text = '{"api_base": "http://127.0.0.1:8080/v1"}' + assert json.loads(strip_json_comments(text)) == {"api_base": "http://127.0.0.1:8080/v1"} + + +def test_https_and_trailing_comment_markers_in_string(): + text = '{"a": "https://x//y", "b": "keep // this"}' + assert json.loads(strip_json_comments(text)) == { + "a": "https://x//y", + "b": "keep // this", + } + + +def test_block_comment_removed_outside_string(): + text = '{\n /* multi\n line */\n "a": 1\n}' + assert json.loads(strip_json_comments(text)) == {"a": 1} + + +def test_block_comment_markers_inside_string_preserved(): + text = '{"a": "not a /* comment */", "b": 2}' + assert json.loads(strip_json_comments(text)) == { + "a": "not a /* comment */", + "b": 2, + } + + +def test_escaped_quote_does_not_end_string(): + # \" inside the string must not toggle string state, so the real "//" + # comment after the closing quote is still removed. + text = '{"a": "he said \\"http://x\\""} // real comment\n' + assert json.loads(strip_json_comments(text)) == {"a": 'he said "http://x"'} + + +def test_escaped_backslash_before_quote(): + # After \\ the quote is not escaped, so the string ends there and the + # tail is a comment. + text = '{"a": "c:\\\\"} // tail\n' + assert json.loads(strip_json_comments(text)) == {"a": "c:\\"} + + +def test_comment_only_line_leaves_newline_for_stable_line_numbers(): + text = '{\n// comment\n"a": 1\n}' + stripped = strip_json_comments(text) + assert stripped.count("\n") == 3 + + +def test_unterminated_comment_consumes_to_eof(): + text = '{"a": 1} // never ends' + assert json.loads(strip_json_comments(text)) == {"a": 1} + + +def test_load_jsonc_roundtrip(tmp_path): + cfg = '{"llm": {"api_base": "http://127.0.0.1:8080/v1"}} // env' + assert load_jsonc(io.StringIO(cfg)) == {"llm": {"api_base": "http://127.0.0.1:8080/v1"}} diff --git a/tests/unit/tools/test_explorer_tool.py b/tests/unit/tools/test_explorer_tool.py index 5e363b3f..fa7ed6ca 100644 --- a/tests/unit/tools/test_explorer_tool.py +++ b/tests/unit/tools/test_explorer_tool.py @@ -355,7 +355,7 @@ def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs): assert "target=3, a bare integer" in result[0]["text"] assert "target=[x, y]" not in result[0]["text"] assert result[1]["type"] == "image_url" - assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert result[1]["image_url"]["url"].startswith("data:image/png;base64,") def test_render_operator_blocks_numbers_annotations_sequentially(tmp_path): diff --git a/tests/unit/utils/test_image_mime.py b/tests/unit/utils/test_image_mime.py new file mode 100644 index 00000000..0fa74693 --- /dev/null +++ b/tests/unit/utils/test_image_mime.py @@ -0,0 +1,60 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for screenshot media-type detection and data-URI construction.""" + +import base64 + +from artemis.utils.image_mime import image_data_uri, image_mime_type + +PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 20 +JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"\x00" * 20 + + +def test_png_bytes_detected(): + assert image_mime_type(PNG_BYTES) == "image/png" + + +def test_jpeg_bytes_detected(): + assert image_mime_type(JPEG_BYTES) == "image/jpeg" + + +def test_unknown_bytes_default_to_png(): + # Screenshot paths emit PNG; an unrecognized header keeps the honest + # project default rather than the historical hardcoded jpeg. + assert image_mime_type(b"\x00\x01\x02\x03") == "image/png" + + +def test_base64_string_detected(): + b64 = base64.b64encode(JPEG_BYTES).decode() + assert image_mime_type(b64) == "image/jpeg" + b64 = base64.b64encode(PNG_BYTES).decode() + assert image_mime_type(b64) == "image/png" + + +def test_data_uri_uses_detected_type(): + uri = image_data_uri(PNG_BYTES) + b64 = base64.b64encode(PNG_BYTES).decode() + assert uri == f"data:image/png;base64,{b64}" + + +def test_data_uri_for_jpeg_bytes(): + uri = image_data_uri(JPEG_BYTES) + b64 = base64.b64encode(JPEG_BYTES).decode() + assert uri == f"data:image/jpeg;base64,{b64}" + + +def test_data_uri_from_base64_input(): + b64 = base64.b64encode(PNG_BYTES).decode() + assert image_data_uri(b64) == f"data:image/png;base64,{b64}" diff --git a/third_party/mobile_use/config/llm.py b/third_party/mobile_use/config/llm.py index be9f509a..f93c5bc8 100644 --- a/third_party/mobile_use/config/llm.py +++ b/third_party/mobile_use/config/llm.py @@ -63,6 +63,10 @@ class LLM(BaseModel): reasoning_effort: Literal["none", "low", "medium", "high"] | None = None include_thoughts: bool | None = None enable_grounding: bool | None = None + api_base: str | None = None + api_key: str | None = None + max_tokens: int | None = None + timeout: float | None = None def validate_provider(self, name: str) -> None: """Ensure the required API key or credentials exist in settings for this provider.""" diff --git a/third_party/mobile_use/utils/file.py b/third_party/mobile_use/utils/file.py index e7930014..055adc10 100644 --- a/third_party/mobile_use/utils/file.py +++ b/third_party/mobile_use/utils/file.py @@ -17,14 +17,51 @@ # See third_party/mobile_use/METADATA for the upstream source and local modifications. import json -import re from typing import IO def strip_json_comments(text: str) -> str: - text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return text + """Remove ``//`` and ``/* */`` comments while preserving string literals. + + The naive regex version stripped ``//`` sequences inside quoted strings, + which made URL values (e.g. ``"api_base": "http://127.0.0.1:8080/v1"``) + unparseable. This scan tracks string/escape state and only treats the + comment markers that appear outside strings as comments. + """ + out: list[str] = [] + i, n = 0, len(text) + in_string = False + while i < n: + char = text[i] + if in_string: + out.append(char) + if char == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + if char == '"': + in_string = False + i += 1 + continue + if char == '"': + in_string = True + out.append(char) + i += 1 + continue + if char == "/" and i + 1 < n: + if text[i + 1] == "/": + while i < n and text[i] != "\n": + i += 1 + continue + if text[i + 1] == "*": + i += 2 + while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + i += 2 + continue + out.append(char) + i += 1 + return "".join(out) def load_jsonc(file: IO) -> dict: