diff --git a/README.md b/README.md index 8683bcce..2d6d4724 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,21 @@ cd artemis > **Tip**: Opens `http://localhost:8000` in your default browser with a device connection wizard, live screen mirroring, prompt sandbox, and execution replays. You can also run directly from CLI: `uv run artemis run "Open Settings, find Battery and tell me current level" --profile flash`. +### iOS Devices (macOS, Xcode 27+) + +CLI, embedded Python SDK, web console, and daemon tasks can target iOS +simulators through Xcode's native MCP device-interaction tools, and paired +physical iPhones/iPads through `devicectl` plus a WebDriverAgent runner +installed on the device. See the +[iOS setup and usage guide](./docs/ios.md) for prerequisites, Xcode access +approval, WDA setup, and current limits: + +```bash +bash scripts/setup_ios_env.sh +uv run artemis run "Open Settings and inspect the General screen" \ + --platform ios --standalone --device-serial --profile flash +``` +
@@ -300,7 +315,7 @@ ARTEMIS supports two execution profiles tailored for different automation requir ## Roadmap - [ ] **Android Studio Integration**: Native IDE plugin and workflow integration to enable in-editor debugging, test recording, and automated device control directly within Android Studio. -- [ ] **iOS Platform Expansion**: Extending multimodal perception and mobile automation to iOS devices and simulators. +- [x] **iOS Platform Expansion**: [iOS support](./docs/ios.md) is available with Xcode 27+ across the CLI, SDK, web console, daemon, MCP, replay, and streaming, covering simulators and paired physical devices. - [ ] **On-Device Lightweight VLMs**: Local execution with lightweight edge vision models for low-latency, privacy-first automation. - [ ] **Real-time Duplex Voice Interaction**: Voice-driven task dispatch with real-time conversational control and interruption handling. diff --git a/README_CN.md b/README_CN.md index 67f461c5..2594d444 100644 --- a/README_CN.md +++ b/README_CN.md @@ -106,6 +106,17 @@ cd artemis > **提示**:启动后将自动在默认浏览器中打开 Web 控制台(`http://localhost:8000`),提供设备连接向导、实时投屏、任务演练与状态回放面板。你也可以通过命令行直接运行:`uv run artemis run "打开系统设置,找到电池选项并告诉我当前电量" --profile flash`。 +### iOS 设备(macOS,Xcode 27+) + +CLI、嵌入式 Python SDK、Web 控制台与守护进程任务都可以通过 Xcode 原生 MCP 设备交互工具控制 iOS 模拟器;真机(iPhone/iPad)则通过 `devicectl` 管理生命周期、并在设备上安装 WebDriverAgent 负责界面操作。 +安装要求、Xcode 权限审批、WDA 配置和当前限制见 [iOS 使用指南(英文)](./docs/ios.md): + +```bash +bash scripts/setup_ios_env.sh +uv run artemis run "打开系统设置并查看通用页面" \ + --platform ios --standalone --device-serial --profile flash +``` +
@@ -298,7 +309,7 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求: ## 路线图 - [ ] **Android Studio 深度集成**:推出官方 IDE 插件与协同工作流,支持在 Android Studio 内直接进行自动化测试、设备交互与断点调试。 -- [ ] **iOS 跨平台支持**:将视觉感知与自动化执行引擎拓展至 iOS 真机与模拟器。 +- [x] **iOS 跨平台支持**:[iOS 支持](./docs/ios.md)已支持 Xcode 27+,覆盖 CLI、SDK、Web 控制台、守护进程、MCP、回放与投屏,含模拟器与已配对真机。 - [ ] **端侧轻量化模型**:支持离线运行的轻量级 Edge VLM,实现低延迟与隐私安全的本地自动化。 - [ ] **实时语音双工交互**:支持自然语音下发任务与实时打断(Barge-in)控制。 diff --git a/apps/admin_console/index.html b/apps/admin_console/index.html index 0d60c4c0..fc0fc2df 100644 --- a/apps/admin_console/index.html +++ b/apps/admin_console/index.html @@ -1442,29 +1442,6 @@

No devices detected'; - return; - } - - devices.forEach(d => { - const opt = document.createElement('option'); - opt.value = d.serial; - opt.innerText = `${d.serial} (${d.status})`; - select.appendChild(opt); - }); - } catch (err) { - console.error("Failed to fetch devices:", err); - } - } - let replayTools = []; async function fetchReplayTools() { diff --git a/apps/admin_console/replay_manager.py b/apps/admin_console/replay_manager.py index c708629d..50af4250 100644 --- a/apps/admin_console/replay_manager.py +++ b/apps/admin_console/replay_manager.py @@ -113,21 +113,15 @@ class ReplayManager: def __init__( self, workspace_root: Path = WORKSPACE_ROOT, - device_id: str = None, original_db_path: Path = None, - init_device: bool = False, ): - """Initializes the ReplayManager and autodetects the connected Android device. + """Initializes the ReplayManager. Args: workspace_root (Path, optional): The absolute path to the workspace root. Defaults to WORKSPACE_ROOT. - device_id (str, optional): The serial ID of the target Android - device. If not specified, autodetects the first connected device. original_db_path (Path, optional): Path to the master SQLite database. Defaults to DB_PATH. - init_device (bool, optional): Whether to initialize the ADB device - connection on startup. """ self.workspace_root = workspace_root self.traces_path = TRACES_PATH @@ -137,62 +131,6 @@ def __init__( self.db_path = Path(original_db_path) if original_db_path else DB_PATH self.images_dir = IMAGES_DIR - # Device connection and autodetection - self.device_id = None - self.w = None - self.h = None - self.adb = None - self.ui_client = None - - if init_device: - try: - self._init_device(device_id) - except Exception as e: - print( - "Warning: Connected device could not be initialized during" - f" ReplayManager startup: {e}" - ) - - def _init_device(self, device_id: str = None): - """Initializes connection to the target device and queries screen metrics.""" - from adbutils import AdbClient - - try: - from third_party.mobile_use.clients.ui_automator_client import UIAutomatorClient - except ImportError: - raise ImportError( - "Failed to import UIAutomatorClient. Ensure artemis package is installed in path." - ) - - self.adb = AdbClient(host="localhost", port=5037) - try: - devices = self.adb.device_list() - except Exception as adb_err: - raise ConnectionError( - f"Failed to query device list from ADB server: {adb_err}" - ) from adb_err - - if not devices: - raise ConnectionError( - "No active ADB devices connected. Please connect an Android device via ADB." - ) - - if device_id: - matched_device = next((d.serial for d in devices if d.serial == device_id), None) - if not matched_device: - raise ConnectionError( - f"Requested device '{device_id}' is not connected." - f" Connected devices: {[d.serial for d in devices]}" - ) - self.device_id = matched_device - else: - self.device_id = devices[0].serial - - self.ui_client = UIAutomatorClient(device_id=self.device_id) - ui_data = self.ui_client.get_screen_data() - self.w, self.h = ui_data.width, ui_data.height - print(f"Connected to device: {self.device_id} ({self.w}x{self.h})") - def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path: """Chunks traces, screenshots, and metadata for a specific execution session. @@ -396,18 +334,6 @@ def chunk_session_traces(self, session_id: str, output_dir: Path = None) -> Path print("Chunking trace completion status: success") return output_dir - def list_devices(self) -> list[dict]: - """Dynamically queries the ADB server for connected Android devices.""" - try: - from adbutils import AdbClient - - adb = AdbClient(host="localhost", port=5037) - devices = adb.device_list() - return [{"serial": d.serial, "status": "online"} for d in devices] - except Exception as e: - print(f"Warning: Failed to query device list from ADB: {e}") - return [] - def load_session_goal( self, session_id: str, step_dir: Path, original_db_path: str = None ) -> str: @@ -1604,12 +1530,69 @@ def _preemptive_clip_video( except Exception as e: print(f"Warning: Failed to update sandbox SQLite database: {e}") + def _session_device_info(self, session_id: str) -> dict: + """The recorded session's ``device_info`` blob (``{}`` when absent).""" + import sqlite3 + + try: + conn = sqlite3.connect(self.db_path) + try: + row = conn.execute( + "SELECT device_info FROM sessions WHERE session_id = ?", + (session_id,), + ).fetchone() + finally: + conn.close() + except sqlite3.Error: + return {} + if not row or not row[0]: + return {} + try: + info = json.loads(row[0]) + except (ValueError, TypeError): + # Malformed device_info: simulate with the defaults. + return {} + return info if isinstance(info, dict) else {} + + @staticmethod + def _replay_device_context(device_info: dict, override_device_id: str | None): + """Build the simulated ``DeviceContext`` for a replayed session. + + The recorded session decides the platform: an iOS session replays + through the native iOS driver, never through ADB. The frontend's + device pick only retargets iOS replays (the picker was always + decorative for Android and stays that way); the driver still + validates the UDID — simctl for simulators, devicectl for physical + devices — at connect time. + """ + from artemis.context import DeviceContext, DevicePlatform + + sim_platform = str( + device_info.get("mobile_platform") or DevicePlatform.ANDROID.value + ).lower() + mobile_platform = ( + DevicePlatform.IOS + if sim_platform == DevicePlatform.IOS.value + else DevicePlatform.ANDROID + ) + sim_device_id = device_info.get("device_id", "replay-device") + if mobile_platform == DevicePlatform.IOS and override_device_id: + sim_device_id = override_device_id + return DeviceContext( + host_platform=("DARWIN" if mobile_platform == DevicePlatform.IOS else "LINUX"), + mobile_platform=mobile_platform, + device_id=sim_device_id, + device_width=device_info.get("device_width", 1080), + device_height=device_info.get("device_height", 2400), + ) + def create_ctx( self, session_id: str, step_number: int, agent_name: str = "explorer", replay_id: str = None, + override_device_id: str = None, ) -> ArtemisContext: """Creates a sandboxed ArtemisContext and DataEngine for the given session and step.""" self._ensure_session_chunked(session_id) @@ -1682,37 +1665,13 @@ def create_ctx( except ImportError as import_err: raise ImportError(f"Failed to import Artemis core modules: {import_err}") - import sqlite3 - - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - cursor.execute( - "SELECT device_info FROM sessions WHERE session_id = ?", - (session_id,), - ) - row = cursor.fetchone() - conn.close() - - device_info = {} - if row and row[0]: - try: - device_info = json.loads(row[0]) - except (ValueError, TypeError): - # Malformed device_info: simulate with the defaults below. - pass - - sim_device_id = device_info.get("device_id", "replay-device") - sim_w = device_info.get("device_width", 1080) - sim_h = device_info.get("device_height", 2400) - - device_context = DeviceContext( - host_platform="LINUX", - mobile_platform=DevicePlatform.ANDROID, - device_id=sim_device_id, - device_width=sim_w, - device_height=sim_h, + device_info = self._session_device_info(session_id) + device_context = self._replay_device_context(device_info, override_device_id) + sim_device_id = device_context.device_id + print( + f"Replay simulated device: {sim_device_id} " + f"({device_context.device_width}x{device_context.device_height})" ) - print(f"Replay simulated device: {sim_device_id} ({sim_w}x{sim_h})") from artemis.config import get_default_llm_config @@ -1765,6 +1724,7 @@ def instantiate_state( pre_image_meta = json.load(f) initial_goal = self.load_session_goal(session_id, step_dir, str(self.db_path)) + device_info = self._session_device_info(session_id) # State has no fields for the app info or device date (extra="forbid"). ui_hier, decisions, _app_info, _dev_date = self.extract_state_prepopulation_data( @@ -1781,7 +1741,10 @@ def instantiate_state( latest_screenshot=str(step_dir / "pre.jpg") if (step_dir / "pre.jpg").exists() else str(step_dir / "post.jpg"), - operator_raw_data={"width": self.w, "height": self.h}, + operator_raw_data={ + "width": device_info.get("device_width"), + "height": device_info.get("device_height"), + }, current_step_id=step_data["step_id"], subagent_calls=step_data.get("subagent_calls") or [], latest_ui_hierarchy=ui_hier, @@ -1872,6 +1835,7 @@ async def replay_step_tool( step_number=step_number, agent_name=agent_name, replay_id=replay_id, + override_device_id=override_device_id, ) state = self.instantiate_state( diff --git a/apps/admin_console/routers/replay.py b/apps/admin_console/routers/replay.py index 2a83b6e2..d6de82c9 100644 --- a/apps/admin_console/routers/replay.py +++ b/apps/admin_console/routers/replay.py @@ -30,12 +30,6 @@ router = APIRouter(tags=["replay"]) -@router.get("/api/devices") -async def list_devices(): - """Dynamically queries the ADB server for connected Android devices.""" - return replay_manager.list_devices() - - @router.get("/api/replay/tools") async def get_replay_tools(): """Returns the list of registered tools available for replay.""" diff --git a/apps/admin_console/routers/stream.py b/apps/admin_console/routers/stream.py index 5bc3c688..6cdd2f58 100644 --- a/apps/admin_console/routers/stream.py +++ b/apps/admin_console/routers/stream.py @@ -45,12 +45,13 @@ async def stream_device_live(): @router.get("/api/stream/device-state") async def get_device_stream_state(): - """Returns whether an ADB device is connected and live streaming is available.""" - serial = await device_stream_service.get_device_serial() + """Returns whether a device is connected and live streaming is available.""" + target = await device_stream_service.get_stream_target() return JSONResponse( { - "connected": serial is not None, - "serial": serial, - "live_stream_url": "/api/stream/device-live" if serial else None, + "connected": target is not None, + "serial": target["serial"] if target else None, + "platform": target["platform"] if target else None, + "live_stream_url": "/api/stream/device-live" if target else None, } ) diff --git a/apps/admin_console/routers/system.py b/apps/admin_console/routers/system.py index c8ce1834..b6d6685b 100644 --- a/apps/admin_console/routers/system.py +++ b/apps/admin_console/routers/system.py @@ -28,6 +28,7 @@ adb_server_connection, ) from artemis.core.diagnostics.schema import SystemReadinessReport +from artemis.runtime.adb_endpoint import normalize_device_platform router = APIRouter(prefix="/api/system", tags=["system"]) @@ -93,9 +94,13 @@ def _require_local_lifecycle_request(request: Request) -> None: class SelectDeviceRequest(BaseModel): - """Payload to select an active target Android device.""" + """Payload to select an active target device.""" - serial: str = Field(description="Serial number or identifier of the Android device to select") + serial: str = Field(description="Serial number or identifier of the device to select") + platform: str = Field( + default="android", + description="Device platform: 'android' (default) or 'ios' (device UDID)", + ) @router.get("/readiness", response_model=SystemReadinessReport) @@ -106,11 +111,28 @@ async def get_system_readiness(force: bool = False) -> SystemReadinessReport: @router.post("/devices/select") async def select_active_device(request: SelectDeviceRequest): - """Select the active Android device or emulator for subsequent automated tasks.""" + """Select the active Android or iOS device for subsequent automated tasks.""" serial = request.serial.strip() if not serial: raise HTTPException(status_code=400, detail="Device serial cannot be empty.") + try: + platform = normalize_device_platform(request.platform) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) + if platform == "ios": + # iOS has no ADB probe to retarget; validate the UDID and let the + # task payload's device_serial + platform carry the binding. + from artemis.runtime import ios_device_pool + + rejection = await ios_device_pool.validate_explicit_serial_async(serial) + if rejection: + raise HTTPException(status_code=400, detail=rejection) + return {"status": "success", "selected_serial": serial, "platform": "ios"} + readiness_engine.set_probe_target_serial(serial) # Return updated readiness report = await readiness_engine.run_all(force_refresh=True) diff --git a/apps/admin_console/routers/tasks.py b/apps/admin_console/routers/tasks.py index 26fca461..d655581e 100644 --- a/apps/admin_console/routers/tasks.py +++ b/apps/admin_console/routers/tasks.py @@ -19,7 +19,12 @@ from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse from artemis.core.diagnostics import readiness_engine -from artemis.runtime import DeviceExecutionLock, device_pool +from artemis.runtime import DeviceExecutionLock, device_pool, ios_device_pool +from artemis.runtime.adb_endpoint import ( + IOS_LOCK_SCOPE, + device_pool_for, + normalize_device_platform, +) try: from admin_console.core.state import state @@ -66,6 +71,26 @@ async def get_task_catalog(): } +@router.get("/api/v1/capabilities") +async def get_capabilities(): + """Static wire-feature advertisement — no device or service probing. + + ``platform.ios`` means the API accepts iOS submissions; it does not + imply Xcode approval or attached hardware on this host. + """ + return { + "api_version": "1", + "features": [ + "tasks.submit", + "tasks.get", + "tasks.stop", + "devices.list", + "system.readiness", + "platform.ios", + ], + } + + @router.post("/api/run") async def run_task(request: RunRequest): incoming_goals = [] @@ -113,6 +138,19 @@ async def run_task(request: RunRequest): "total_queued": len(state.queue_tasks), } + try: + platform = normalize_device_platform(request.platform) + except ValueError: + raise HTTPException( + status_code=400, + detail=f"Unsupported platform '{request.platform}'. Expected 'android' or 'ios'.", + ) + if platform == "ios" and request.locked_app_package: + raise HTTPException( + status_code=400, + detail="locked_app_package is unavailable for iOS tasks.", + ) + # Reject an explicit unknown/offline target before running the more # expensive readiness probe. Besides producing a stable SDK response, # this avoids probing the currently active device for a serial that can @@ -121,7 +159,9 @@ async def run_task(request: RunRequest): # fail downstream with a clear error instead. if request.device_serial: try: - rejection = await device_pool.validate_explicit_serial_async(request.device_serial) + rejection = await device_pool_for(platform).validate_explicit_serial_async( + request.device_serial + ) except Exception: rejection = None if rejection: @@ -140,30 +180,35 @@ async def run_task(request: RunRequest): # With no explicit serial the probe itself resolves a live target (it # prefers the diagnostics target preference, then any unlocked ready # device); the verified serial is bound below. + # iOS skips the Android screen-lock probe entirely: the Xcode driver + # validates and boots the pinned simulator at execution time. target_serial = request.device_serial - device_probe = await readiness_engine.run_device_submission_probe(target_serial=target_serial) - if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: - locked_serial = ( - device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" - ) - detail = ( - f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace( - " ", " " - ).strip() - if device_probe.summary == "Device Locked" - else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace( - " ", " " - ).strip() + if platform != "ios": + device_probe = await readiness_engine.run_device_submission_probe( + target_serial=target_serial ) - raise HTTPException(status_code=409, detail=detail) + if device_probe and device_probe.summary in {"Device Locked", "Lock State Unknown"}: + locked_serial = ( + device_probe.metadata.get("active_device", {}).get("serial") or target_serial or "" + ) + detail = ( + f"Android device {locked_serial} is locked. Unlock it and enter the home screen before running a task.".replace( + " ", " " + ).strip() + if device_probe.summary == "Device Locked" + else f"Android device {locked_serial} lock state could not be verified. Keep it unlocked on the home screen and try again.".replace( + " ", " " + ).strip() + ) + raise HTTPException(status_code=409, detail=detail) - if device_probe and device_probe.metadata.get("active_device"): - verified_serial = device_probe.metadata["active_device"].get("serial") - # Only auto-selected targets may be re-bound to the probed device. An - # explicitly requested serial is never silently replaced -- if it is - # invalid, enqueue_tasks rejects the submission with a clear error. - if verified_serial and not request.device_serial: - target_serial = verified_serial + if device_probe and device_probe.metadata.get("active_device"): + verified_serial = device_probe.metadata["active_device"].get("serial") + # Only auto-selected targets may be re-bound to the probed device. An + # explicitly requested serial is never silently replaced -- if it is + # invalid, enqueue_tasks rejects the submission with a clear error. + if verified_serial and not request.device_serial: + target_serial = verified_serial return await task_queue_service.enqueue_tasks( incoming_goals, @@ -178,6 +223,8 @@ async def run_task(request: RunRequest): ingress=request.ingress or "frontend", session_id=request.session_id, conversation_id=request.conversation_id, + platform=platform, + ios_workspace=request.ios_workspace, ) @@ -199,9 +246,11 @@ async def get_run_defaults(): @router.get("/api/devices") async def list_devices(): - """List all connected Android devices with their busy / idle status.""" - devices = await device_pool.list_devices_async() - return {"devices": [d.to_dict() for d in devices]} + """List all connected devices (Android + iOS) with lock status.""" + android_devices, ios_devices = await asyncio.gather( + device_pool.list_devices_async(), ios_device_pool.list_devices_async() + ) + return {"devices": [d.to_dict() for d in (*android_devices, *ios_devices)]} @router.post("/api/stop") @@ -276,6 +325,9 @@ async def get_status(): "pid": owner.pid, "ingress": owner.ingress, "acquired_at": owner.acquired_at, + "platform": "ios" + if getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE + else "android", } for owner in active_owners.values() ] diff --git a/apps/admin_console/schemas/task_schema.py b/apps/admin_console/schemas/task_schema.py index 144a9720..e0ee9cbe 100644 --- a/apps/admin_console/schemas/task_schema.py +++ b/apps/admin_console/schemas/task_schema.py @@ -29,6 +29,10 @@ class RunRequest(BaseModel): locked_app_package: str | None = None app_path: str | None = None device_serial: str | None = None + # Target platform: "android" (default) or "ios". For iOS, device_serial is a + # simulator or physical UDID and ios_workspace optionally scopes the Xcode approval grant. + platform: str | None = None + ios_workspace: str | None = None ingress: str | None = "frontend" session_id: str | None = None conversation_id: str | None = None diff --git a/apps/admin_console/services/device_stream_service.py b/apps/admin_console/services/device_stream_service.py index 6455a3e7..199edf6e 100644 --- a/apps/admin_console/services/device_stream_service.py +++ b/apps/admin_console/services/device_stream_service.py @@ -15,7 +15,9 @@ """Device Live Screen Streaming Service. Provides real-time, low-latency device screen frames over HTTP MJPEG and WebSocket. -Operates concurrently with ADB agent actions with zero interference. +Android frames come from ``adb exec-out screencap``; iOS frames come +from the native ``xcrun simctl io screenshot`` capture. The target is +picked per frame so the stream follows whichever platform is under automation. """ import asyncio @@ -23,7 +25,11 @@ import subprocess import time from collections.abc import AsyncGenerator +from pathlib import Path +from artemis.config.paths import get_temp_dir +from artemis.runtime import DeviceExecutionLock, ios_device_pool +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.toolchain import find_adb logger = logging.getLogger("artemis.stream_service") @@ -39,8 +45,41 @@ def __init__(self): self._last_frame_time: float = 0.0 self._is_capturing = False self._capture_task: asyncio.Task | None = None + self._last_target: dict[str, str] | None = None + + async def get_stream_target(self) -> dict[str, str] | None: + """Pick the device the stream should follow right now. + + An actively locked iOS device wins (it is the device under + automation), then a connected Android device, then the single + unambiguous booted simulator. Returns ``{"platform", "serial"}``. + """ + try: + for owner in DeviceExecutionLock.get_active_owners().values(): + if ( + owner + and getattr(owner, "lock_scope", None) == IOS_LOCK_SCOPE + and owner.device_id + ): + return {"platform": "ios", "serial": str(owner.device_id)} + except Exception as exc: + logger.debug(f"[StreamService] iOS lock-owner scan failed: {exc}") + + serial = await self._android_serial() + if serial: + return {"platform": "android", "serial": serial} + + udid = await ios_device_pool.select_device_async() + if udid: + return {"platform": "ios", "serial": udid} + return None async def get_device_serial(self) -> str | None: + """Serial of the currently streamable device, regardless of platform.""" + target = await self.get_stream_target() + return target["serial"] if target else None + + async def _android_serial(self) -> str | None: """Find the currently connected active ADB device serial.""" try: adb_bin = find_adb() @@ -60,27 +99,95 @@ async def get_device_serial(self) -> str | None: logger.warning(f"Error checking adb devices: {e}") return None + async def _capture_android(self, serial: str | None) -> bytes | None: + adb_bin = find_adb() + cmd = ( + [adb_bin, "-s", serial, "exec-out", "screencap", "-p"] + if serial + else [adb_bin, "exec-out", "screencap", "-p"] + ) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout if proc.returncode == 0 and len(stdout) > 1000 else None + + async def _capture_ios(self, udid: str) -> bytes | None: + """One PNG frame via simctl (simulator) or devicectl (physical). + + Both tools only write files, so frames stream through one reused temp + path per UDID. + """ + from artemis.drivers.ios.discovery import find_physical_ios_device_sync + + frame_path = Path(get_temp_dir("streams")) / f"ios_stream_{udid}.png" + physical = await asyncio.to_thread(find_physical_ios_device_sync, udid) + if physical is not None: + cmd = [ + "xcrun", + "devicectl", + "device", + "capture", + "screenshot", + "--device", + udid, + "--destination", + str(frame_path), + ] + else: + cmd = [ + "xcrun", + "simctl", + "io", + udid, + "screenshot", + "--type=png", + str(frame_path), + ] + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + await proc.communicate() + except asyncio.CancelledError: + if proc.returncode is None: + try: + proc.kill() + except ProcessLookupError: + pass + await proc.communicate() + raise + if proc.returncode == 0 and frame_path.exists(): + data = frame_path.read_bytes() + return data if len(data) > 1000 else None + finally: + frame_path.unlink(missing_ok=True) + return None + async def _capture_loop(self): """Background frame capture loop that runs while listeners > 0.""" logger.info("[StreamService] Starting live screen capture loop...") while self._active_listeners > 0: try: start_t = time.time() - serial = await self.get_device_serial() - adb_bin = find_adb() - cmd = ( - [adb_bin, "-s", serial, "exec-out", "screencap", "-p"] - if serial - else [adb_bin, "exec-out", "screencap", "-p"] - ) - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, _ = await proc.communicate() - if proc.returncode == 0 and len(stdout) > 1000: - self._latest_frame = stdout + target = await self.get_stream_target() + if target != self._last_target: + # Never carry frames across a device/platform switch. + self._latest_frame = None + self._last_target = target + frame = None + if target is not None: + if target["platform"] == "ios": + frame = await self._capture_ios(target["serial"]) + else: + frame = await self._capture_android(target["serial"]) + if frame is not None: + self._latest_frame = frame self._last_frame_time = time.time() elapsed = time.time() - start_t diff --git a/apps/admin_console/services/task_queue_service.py b/apps/admin_console/services/task_queue_service.py index b99b4c91..93e7613b 100644 --- a/apps/admin_console/services/task_queue_service.py +++ b/apps/admin_console/services/task_queue_service.py @@ -52,6 +52,13 @@ request_cancel, trace_store, ) +from artemis.runtime.adb_endpoint import ( + IOS_LOCK_SCOPE, + IosTarget, + device_pool_for, + normalize_device_platform, + target_for_platform, +) logger = logging.getLogger(__name__) @@ -216,15 +223,16 @@ def _enforce_sync() -> None: ).start() @staticmethod - def _task_target(task_item: dict[str, Any]) -> AdbTarget: + def _task_target(task_item: dict[str, Any]) -> AdbTarget | IosTarget: + serial = str(task_item.get("device_serial") or "") or None + if normalize_device_platform(task_item.get("platform"), strict=False) == "ios": + # iOS targets never read a stale/malformed Android endpoint blob. + return IosTarget(serial=serial) endpoint_data = task_item.get("adb_endpoint") endpoint = ( - AdbEndpoint.from_mapping(endpoint_data) - if isinstance(endpoint_data, dict) - else current_adb_endpoint() + AdbEndpoint.from_mapping(endpoint_data) if isinstance(endpoint_data, dict) else None ) - serial = task_item.get("device_serial") - return AdbTarget(endpoint=endpoint, serial=str(serial) if serial else None) + return target_for_platform(task_item.get("platform"), serial=serial, endpoint=endpoint) @classmethod def _broadcast_event(cls, event_type: str, data: Any): @@ -498,7 +506,7 @@ def _build_worker_invocation( sess_id: Any, goal: str, profile: str, - target: AdbTarget, + target: AdbTarget | IosTarget, ) -> tuple[list[str], dict[str, str]]: """Assemble the worker subprocess command line and environment.""" expected_output = task_item.get("expected_output") @@ -525,7 +533,7 @@ def _build_worker_invocation( env["ARTEMIS_SESSION_ID"] = str(sess_id) env["ARTEMIS_TASK_INGRESS"] = str(task_item.get("ingress", "frontend")) env["ARTEMIS_TASK_WORKER"] = "1" - target.endpoint.apply_to_environment(env) + target.apply_to_environment(env) env[DeviceExecutionLock.LOCK_SCOPE_ENV] = target.lock_scope queue_ticket = task_item.get("queue_ticket") if queue_ticket: @@ -556,7 +564,14 @@ def _build_worker_invocation( if app_path: cmd.extend(["--app-path", str(app_path)]) device_serial = task_item.get("device_serial") - if device_serial: + if isinstance(target, IosTarget): + cmd.extend(["--platform", "ios"]) + ios_workspace = task_item.get("ios_workspace") + if ios_workspace: + cmd.extend(["--ios-workspace", str(ios_workspace)]) + if device_serial: + cmd.extend(["--device-serial", str(device_serial)]) + elif device_serial: cmd.extend(["--device-serial", str(device_serial)]) env["ADB_DEVICE_SERIAL"] = str(device_serial) return cmd, env @@ -569,7 +584,7 @@ def _register_worker_run( sess_id: Any, goal: str, profile: str, - target: AdbTarget, + target: AdbTarget | IosTarget, proc: asyncio.subprocess.Process, ) -> None: """Record the spawned worker in shared state and hand it the device reservation.""" @@ -580,7 +595,9 @@ def _register_worker_run( "process": proc, "device_id": str(device_serial) if device_serial else None, "lock_key": target.lock_key if device_serial else None, - "adb_endpoint": target.endpoint.to_dict(), + "platform": target.platform, + "target": target.to_dict(), + "adb_endpoint": (target.endpoint.to_dict() if isinstance(target, AdbTarget) else None), "goal": goal, "profile": profile, } @@ -909,6 +926,8 @@ def _find_duplicate_submission( device_serial: str | None, endpoint: AdbEndpoint, now: float, + *, + platform: str = "android", ) -> dict[str, Any] | None: """Return the short-circuit response for a duplicate submission, if any.""" # 1. Deduplication by session_id: if session_id is already running or queued, do not re-enqueue @@ -947,8 +966,14 @@ def _find_duplicate_submission( if isinstance(item, dict) and item.get("status") == "pending" and item.get("goal") == first_goal + and normalize_device_platform(item.get("platform"), strict=False) == platform and (not device_serial or item.get("device_serial") == device_serial) - and item.get("adb_endpoint", {}).get("identity") == endpoint.identity + # iOS items carry no ADB endpoint snapshot; matching the + # platform + serial is the whole identity there. + and ( + platform == "ios" + or (item.get("adb_endpoint") or {}).get("identity") == endpoint.identity + ) and (now - float(item.get("created_at", 0))) < 1.0 ), None, @@ -963,7 +988,9 @@ def _find_duplicate_submission( return None @classmethod - async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str, Any] | None: + async def _reject_unavailable_device( + cls, device_serial: str | None, platform: str = "android" + ) -> dict[str, Any] | None: """Return the rejection response for an unattached explicit serial, if any.""" # Strict device binding: reject an explicitly requested serial that is not # attached and authorized, instead of silently running on another device. @@ -971,9 +998,9 @@ async def _reject_unavailable_device(cls, device_serial: str | None) -> dict[str # the task can proceed and fail downstream with a clear no-device error. if device_serial: try: - from artemis.runtime import device_pool - - rejection = await device_pool.validate_explicit_serial_async(device_serial) + rejection = await device_pool_for(platform).validate_explicit_serial_async( + device_serial + ) except Exception: rejection = None if rejection: @@ -1004,6 +1031,8 @@ def _create_queue_item( conversation_id: str | None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ) -> dict[str, Any]: """Reserve a device slot and build one pending queue item for a goal.""" sess_id = single_session_id if single_session_id else str(uuid.uuid4()) @@ -1015,7 +1044,7 @@ def _create_queue_item( device_id=assigned_serial or "pending", session_id=sess_id, ingress=ingress, - lock_scope=endpoint.identity, + lock_scope=IOS_LOCK_SCOPE if platform == "ios" else endpoint.identity, ) return { "session_id": sess_id, @@ -1029,6 +1058,8 @@ def _create_queue_item( "app_path": app_path, "device_serial": assigned_serial, "adb_endpoint": endpoint.to_dict(), + "platform": platform, + "ios_workspace": ios_workspace, "ingress": ingress, "conversation_id": conversation_id, "status": "pending", @@ -1052,13 +1083,20 @@ async def enqueue_tasks( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, ) -> dict[str, Any]: """Enqueues one or more goals and wakes up the background worker. ``verification_level`` and ``explorer_mode`` are Pro-profile tuning knobs forwarded to the worker as ``--verification-level`` / ``--explorer-pro-mode``; they are normalised here so the queue item and the CLI see one spelling. + + ``platform="ios"`` enqueues a native iOS task: the serial is a simulator + or paired physical UDID, the device lock is scoped under ``ios``, and the worker + runs with ``--platform ios`` instead of ADB bindings. """ + platform = normalize_device_platform(platform, strict=False) verification_level = ( str(verification_level).strip().lower() or None if verification_level else None ) @@ -1070,22 +1108,19 @@ async def enqueue_tasks( endpoint = current_adb_endpoint() duplicate_response = cls._find_duplicate_submission( - goals, session_id, device_serial, endpoint, now + goals, session_id, device_serial, endpoint, now, platform=platform ) if duplicate_response is not None: return duplicate_response - rejection_response = await cls._reject_unavailable_device(device_serial) + rejection_response = await cls._reject_unavailable_device(device_serial, platform) if rejection_response is not None: return rejection_response single_session_id = session_id if (session_id and len(goals) == 1) else None if not device_serial: - # Device enumeration may block on ADB. - from artemis.runtime import device_pool - try: - device_serial = await device_pool.select_device_async() + device_serial = await device_pool_for(platform).select_device_async() except Exception: device_serial = None for i, goal in enumerate(goals): @@ -1105,6 +1140,8 @@ async def enqueue_tasks( conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, ) state.queue_items.append(task_item) enqueued_tasks.append(task_item) diff --git a/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html b/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html index 44ec4120..c7bd726b 100644 --- a/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +++ b/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html @@ -35,7 +35,7 @@

👣 Real-time Agent Stream (Legacy Card View)

} diff --git a/apps/showcase_ui/src/app/core/models/session.model.ts b/apps/showcase_ui/src/app/core/models/session.model.ts index ef322d1c..8733ac3d 100644 --- a/apps/showcase_ui/src/app/core/models/session.model.ts +++ b/apps/showcase_ui/src/app/core/models/session.model.ts @@ -30,6 +30,7 @@ export interface TaskQueueItem { start_time?: number; device_serial?: string | null; device_id?: string | null; + platform?: 'android' | 'ios' | string | null; } export interface Session { @@ -44,6 +45,7 @@ export interface Session { device_serial?: string | null; device_id?: string | null; device_info?: any; + platform?: 'android' | 'ios' | string | null; } export interface AgentStatusResponse { diff --git a/apps/showcase_ui/src/app/core/models/system.model.ts b/apps/showcase_ui/src/app/core/models/system.model.ts index cf15406f..cfa52957 100644 --- a/apps/showcase_ui/src/app/core/models/system.model.ts +++ b/apps/showcase_ui/src/app/core/models/system.model.ts @@ -34,6 +34,8 @@ export interface DeviceInfo { is_locked: boolean | null; is_emulator: boolean; installed_packages?: string[]; + /** 'android' (default) or 'ios' (iOS Simulator; serial is the UDID). */ + platform?: 'android' | 'ios'; } export interface AdbServerEndpoint { diff --git a/apps/showcase_ui/src/app/pages/home/home.component.html b/apps/showcase_ui/src/app/pages/home/home.component.html index 060a01d2..571e0c82 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.html +++ b/apps/showcase_ui/src/app/pages/home/home.component.html @@ -672,11 +672,12 @@

AI Model Setup

-

Android Device & Emulator Connection

+

Device & Emulator Connection

-

Local emulator, attached phone, wireless device, or another ADB server

+

Android device or emulator via ADB, or an iOS Simulator / paired iPhone or iPad (Xcode 27+)

+ @if (deviceGuidePlatform() === 'android') {

{{ isRemoteAdbServer() ? 'Refresh Devices' : 'Restart ADB' }} } + }
+ +
+ + +
+ + @if (deviceGuidePlatform() === 'android') { + - @if (isDeviceReady()) { + @if (androidDeviceReady()) {
@@ -750,22 +782,22 @@

Android Device & Emulator Connection

- @if (connectedDevices().length > 1) { + @if (androidDevices().length > 1) {
Select Active Device:
- @for (dev of connectedDevices(); track dev.serial) { + @for (dev of androidDevices(); track dev.serial) { @@ -777,7 +809,7 @@

Android Device & Emulator Connection

} - @if (isEmulatorLaunching() || (!isDeviceReady() && adbProbe()?.summary === 'Device Booting')) { + @if (isEmulatorLaunching() || (!androidDeviceReady() && adbProbe()?.summary === 'Device Booting')) {
@@ -1561,22 +1593,22 @@

No Android Virtual Devices (AVD) Detected

- @if (connectedDevices().length > 1) { + @if (androidDevices().length > 1) {
Select Active Device:
- @for (dev of connectedDevices(); track dev.serial) { + @for (dev of androidDevices(); track dev.serial) { @@ -1674,6 +1706,90 @@

No Android Virtual Devices (AVD) Detected

} + } @else { + + +
+ + + @if (selectedIosDevice(); as sim) { +
+
+
+ + {{ sim.model?.includes('iPad') ? 'tablet_mac' : 'phone_iphone' }} + +
+
+
+ {{ sim.model || sim.serial }} +
+
+ {{ sim.is_emulator ? 'iOS Simulator' : 'Physical iOS Device' }} + · + {{ sim.serial }} + @if (sim.product) { + · + {{ sim.product }} + } +
+
+
+ + {{ sim.state === 'device' ? (sim.is_emulator ? 'Booted' : 'Connected') : (sim.is_emulator ? 'Shutdown — boots on run' : sim.state) }} +
+
+
+ } + + + @if (iosDevices().length > 0) { +
+
+ phone_iphone +
+ iOS Devices + + {{ iosDevices().length }} available — select a simulator or paired iPhone/iPad to run iOS tasks + +
+
+
+ @for (dev of iosDevices(); track dev.serial) { + + } +
+
+ } @else { +
+ phone_iphone +
+
No iOS Devices Available
+
+ {{ iosProbe()?.description || 'iOS automation requires macOS with Xcode 27+ and an available simulator or a paired iPhone/iPad. Physical devices also need WebDriverAgent on the device.' }} +
+
+
+ } + +
+ + } +
@@ -1686,7 +1802,7 @@

No Android Virtual Devices (AVD) Detected

System Ready for Autonomous Tasks

-

All dependencies, AI model credentials, and Android device are verified and ready to run.

+

All dependencies, AI model credentials, and a target device are verified and ready to run.

@@ -1808,6 +1924,23 @@

System Ready for Autonomous Tasks

Pro
+ + + @if (selectedTarget(); as tgt) { + + }
diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index c1a5e68d..14c3c4db 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -1761,6 +1761,10 @@ $text-light: #94a3b8; margin-left: auto; flex-shrink: 0; + &.status-idle { + color: #b45309; + } + .status-dot { width: 7px; height: 7px; @@ -1768,6 +1772,12 @@ $text-light: #94a3b8; background: #10b981; box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); animation: statusDotPulse 2s infinite cubic-bezier(0.4, 0, 0.6, 1); + + &.dot-warn { + background: #f59e0b; + box-shadow: none; + animation: none; + } } } @@ -1811,6 +1821,26 @@ $text-light: #94a3b8; font-size: 14px; } + .chip-name { + color: $text-main; + } + + .sim-state-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: #94a3b8; + flex-shrink: 0; + + &.booted { + background: #10b981; + } + + &.booting { + background: #f59e0b; + } + } + .active-check { color: $emerald; font-size: 10.5px; @@ -2394,6 +2424,120 @@ $text-light: #94a3b8; display: flex; flex-direction: column; gap: 12px; + } + + .platform-tabs { + display: inline-flex; + align-items: center; + gap: 4px; + align-self: flex-start; + background: rgba(241, 245, 249, 0.85); + padding: 3px; + border-radius: 10px; + margin-bottom: 12px; + + .platform-tab { + display: inline-flex; + align-items: center; + gap: 6px; + height: 28px; + padding: 0 12px; + border: none; + border-radius: 8px; + background: transparent; + font-family: inherit; + font-size: 12px; + font-weight: 600; + color: $text-muted; + cursor: pointer; + transition: all 0.15s ease; + + .plat-icon { + font-size: 15px; + } + + .plat-count { + font-size: 10px; + font-weight: 700; + padding: 1px 6px; + border-radius: 8px; + background: rgba(100, 116, 139, 0.12); + color: inherit; + } + + &.active { + background: #ffffff; + color: $text-main; + box-shadow: 0 1px 3px rgba(15, 23, 42, 0.1); + } + } + } + + .ios-empty-state { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 16px; + border: 1px dashed $card-border-hover; + border-radius: 8px; + background: #fafbff; + + .ios-empty-icon { + font-size: 22px; + color: #818cf8; + } + + .ios-empty-title { + font-size: 12.5px; + font-weight: 700; + color: $text-main; + margin-bottom: 3px; + } + + .ios-empty-desc { + font-size: 11.5px; + color: $text-secondary; + line-height: 1.5; + } + } + + .ios-sim-panel { + display: flex; + flex-direction: column; + gap: 10px; + padding: 12px; + border: 1px solid #e0e7ff; + border-radius: 8px; + background: #f8faff; + + .ios-sim-header { + display: flex; + align-items: center; + gap: 10px; + + .ios-sim-icon { + font-size: 20px; + color: #4f46e5; + } + + .ios-sim-header-text { + display: flex; + flex-direction: column; + } + + .ios-sim-title { + font-size: 12.5px; + font-weight: 700; + color: $text-main; + } + + .ios-sim-desc { + font-size: 11px; + color: $text-secondary; + } + } + + } .connection-methods-nav { display: grid; @@ -4056,6 +4200,55 @@ $text-light: #94a3b8; .bar-left { display: flex; align-items: center; + gap: 8px; + + .target-device-chip { + display: inline-flex; + align-items: center; + gap: 6px; + height: 30px; + padding: 0 10px; + background: #ffffff; + border: 1px solid $card-border; + border-radius: 15px; + font-family: inherit; + font-size: 11.5px; + font-weight: 600; + color: $text-secondary; + cursor: pointer; + transition: all 0.15s ease; + + .target-icon { + font-size: 15px; + } + + .target-name { + max-width: 160px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .target-plat { + font-size: 9px; + font-weight: 700; + padding: 1px 5px; + border-radius: 4px; + background: #f1f5f9; + color: $text-muted; + letter-spacing: 0.3px; + } + + &.is-ios { + border-color: #c7d2fe; + color: #4338ca; + + .target-plat { + background: #eef2ff; + color: #4f46e5; + } + } + } .model-toggle-capsule { display: inline-flex; @@ -4993,7 +5186,6 @@ $text-light: #94a3b8; } } -} // ============================================================================= // KEYFRAME ANIMATIONS diff --git a/apps/showcase_ui/src/app/pages/home/home.component.ts b/apps/showcase_ui/src/app/pages/home/home.component.ts index e189b9ce..b4d98f3d 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.ts +++ b/apps/showcase_ui/src/app/pages/home/home.component.ts @@ -371,6 +371,21 @@ export class HomeComponent implements OnInit, OnDestroy { // Device information public activeDevice = computed(() => this.systemService.activeDevice()); public connectedDevices = computed(() => this.systemService.connectedDevices()); + public selectedDeviceSerial = computed(() => this.systemService.selectedDeviceSerial()); + public iosDevices = computed(() => this.systemService.iosDevices()); + public selectedIosDevice = computed(() => this.systemService.selectedIosDevice()); + public iosProbe = computed(() => this.systemService.iosProbe()); + public androidDevices = computed( + () => this.connectedDevices().filter(d => d.platform !== 'ios') + ); + // Raw ADB state for the Android tab — isDeviceReady also counts an iOS pick. + public androidDeviceReady = computed(() => this.adbProbe()?.status === 'pass'); + public selectedTarget = computed( + () => this.systemService.selectedIosDevice() ?? this.activeDevice() + ); + // Device guide platform split: Android (ADB) vs iOS Simulator + public deviceGuidePlatform = signal<'android' | 'ios'>('android'); + private deviceGuidePlatformAutoSet = false; public installedAvds = computed(() => this.systemService.installedAvds()); public emulatorPath = computed(() => this.systemService.emulatorPath()); public isEmulatorInPath = computed(() => this.systemService.isEmulatorInPath()); @@ -533,6 +548,19 @@ export class HomeComponent implements OnInit, OnDestroy { this.ocrKeyInput.set(ocrKey); } }); + // Default the device guide to whichever platform actually has devices. + effect(() => { + if (this.deviceGuidePlatformAutoSet) return; + if (this.systemService.selectedIosDevice()) { + this.deviceGuidePlatform.set('ios'); + this.deviceGuidePlatformAutoSet = true; + } else if (this.iosDevices().length > 0 && this.androidDevices().length === 0) { + this.deviceGuidePlatform.set('ios'); + this.deviceGuidePlatformAutoSet = true; + } else if (this.androidDevices().length > 0) { + this.deviceGuidePlatformAutoSet = true; + } + }); } ngOnInit(): void { @@ -1008,8 +1036,9 @@ export class HomeComponent implements OnInit, OnDestroy { this.systemService.dismissEmulatorStatus().subscribe(); } - public selectTargetDevice(serial: string): void { - this.systemService.selectDevice(serial).subscribe(); + public selectTargetDevice(dev: DeviceInfo): void { + this.deviceGuidePlatform.set(dev.platform === 'ios' ? 'ios' : 'android'); + this.systemService.selectDevice(dev.serial, dev.platform ?? 'android').subscribe(); } public getEmulatorCommand(avdName: string): string { diff --git a/apps/showcase_ui/src/app/services/agent.service.spec.ts b/apps/showcase_ui/src/app/services/agent.service.spec.ts index c17e334d..980d065b 100644 --- a/apps/showcase_ui/src/app/services/agent.service.spec.ts +++ b/apps/showcase_ui/src/app/services/agent.service.spec.ts @@ -180,6 +180,53 @@ describe('AgentService live LLM retry timeline', () => { expect(selectSpy).toHaveBeenCalledWith('new-session', false); }); + it('sends platform and device_serial when an iOS simulator is selected', () => { + const service = createServiceWithoutPolling(); + let postedPayload: any = null; + (service as any).http = { + post: (_url: string, payload: any) => { + postedPayload = payload; + return of({ tasks: [] }); + } + }; + (service as any).systemService = { + selectedIosDevice: () => ({ serial: 'UDID-1', platform: 'ios' }) + }; + service.agentStatus = signal('idle'); + service.runningSessionId = signal(null); + service.userPinnedSessionId = signal(null); + (service as any).sessions = signal([]); + spyOn(service, 'selectSession'); + + service.runTask('test goal').subscribe(); + + expect(postedPayload.platform).toBe('ios'); + expect(postedPayload.device_serial).toBe('UDID-1'); + }); + + it('keeps the Android payload unchanged when no iOS simulator is selected', () => { + const service = createServiceWithoutPolling(); + let postedPayload: any = null; + (service as any).http = { + post: (_url: string, payload: any) => { + postedPayload = payload; + return of({ tasks: [] }); + } + }; + (service as any).systemService = { selectedIosDevice: () => null }; + service.agentStatus = signal('idle'); + service.runningSessionId = signal(null); + service.userPinnedSessionId = signal(null); + (service as any).sessions = signal([]); + spyOn(service, 'selectSession'); + + service.runTask('test goal').subscribe(); + + expect(postedPayload.goal).toBe('test goal'); + expect('platform' in postedPayload).toBeFalse(); + expect('device_serial' in postedPayload).toBeFalse(); + }); + it('keeps the paused state when the backend says there is nothing to resume', () => { const service = createServiceWithoutPolling(); (service as any).http = { post: () => of({ status: 'not_paused' }) }; diff --git a/apps/showcase_ui/src/app/services/agent.service.ts b/apps/showcase_ui/src/app/services/agent.service.ts index b063c99b..2ee14c60 100644 --- a/apps/showcase_ui/src/app/services/agent.service.ts +++ b/apps/showcase_ui/src/app/services/agent.service.ts @@ -23,6 +23,7 @@ import { ProTuningDefaults, ProTuningOptions } from '../core/models/pro-tuning.m import { StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE, PersistedCheckerStream, StreamSegment } from '../core/models/stream.model'; import { extractStepReplayFrames } from '../utils/action-formatter.util'; import { persistedStreamToSegments } from '../utils/stream-aggregator.util'; +import { SystemService } from './system.service'; export type { Session, ModelInfo, TaskQueueItem, AgentStatusResponse, StepItemData, StepReplayFrame, LLMStreamResetEventData, StreamResetNotice }; const SESSION_CACHE_KEY = 'artemis.sessions.v1'; @@ -70,6 +71,7 @@ interface SessionVideoResponse { export class AgentService { private http = inject(HttpClient); private zone = inject(NgZone); + private systemService = inject(SystemService); private activePauseCardKey: string | null = null; // Signals to expose state to components @@ -158,7 +160,8 @@ export class AgentService { start_time: at.acquired_at ? (new Date(at.acquired_at).getTime() / 1000) : (Date.now() / 1000), status: 'running', model_info: this.activeModel() || undefined, - device_serial: at.device_id || null + device_serial: at.device_id || null, + platform: at.platform || null }; sessionMap.set(sid, newSession); this.activeSessionTracking.set(sid, newSession); @@ -430,6 +433,13 @@ export class AgentService { }; this.pendingStartupProgress.set([submittedEvent]); const payload: any = { goal, profile }; + const iosDevice = this.systemService?.selectedIosDevice?.() ?? null; + if (iosDevice) { + // iOS submissions carry the simulator UDID + platform explicitly; + // Android keeps resolving through the ADB probe target. + payload.platform = 'ios'; + payload.device_serial = iosDevice.serial; + } if (expectedOutput && expectedOutput.trim()) { payload.expected_output = expectedOutput.trim(); } @@ -1546,7 +1556,8 @@ export class AgentService { initial_goal: item.goal || '', start_time: item.start_time || item.created_at || (Date.now() / 1000 + index), status: item.status || 'pending', - device_serial: item.device_serial || item.device_id || null + device_serial: item.device_serial || item.device_id || null, + platform: item.platform || null }; } return { diff --git a/apps/showcase_ui/src/app/services/system.service.spec.ts b/apps/showcase_ui/src/app/services/system.service.spec.ts index bd2c6d51..f75ac47f 100644 --- a/apps/showcase_ui/src/app/services/system.service.spec.ts +++ b/apps/showcase_ui/src/app/services/system.service.spec.ts @@ -145,6 +145,331 @@ describe('SystemService readiness polling', () => { expect(service.adbServerStatus()).toBeNull(); }); + it('merges iOS simulators into the device list with platform tags', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(60), + probes: [ + { + id: 'android_adb', + category: 'device', + title: 'Android Device', + status: 'pass', + is_blocker: true, + summary: 'Device Ready', + description: '', + metadata: { + devices: [ + { + serial: 'emulator-5554', + state: 'device', + model: 'Pixel_8', + product: 'sdk', + android_version: '14', + screen_resolution: '1080x2400', + is_locked: false, + is_emulator: true + } + ] + }, + actions: [] + }, + { + id: 'ios_simulators', + category: 'device', + title: 'iOS Simulator', + status: 'pass', + is_blocker: false, + summary: '1 Booted', + description: '', + metadata: { + simulators: [ + { + udid: 'E1D9F1D1-04E5-4E95-801F-830B854FD3E2', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + }, + { + udid: 'AAAA1111-2222-3333-4444-555566667777', + name: 'iPhone 17', + state: 'Shutdown', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-26-0', + isAvailable: true + } + ] + }, + actions: [] + } + ] + }); + + const devices = service.connectedDevices(); + expect(devices.length).toBe(3); + expect(devices[0].platform).toBe('android'); + const booted = devices.find(d => d.serial === 'E1D9F1D1-04E5-4E95-801F-830B854FD3E2'); + expect(booted?.platform).toBe('ios'); + expect(booted?.model).toBe('iPhone 18 Pro'); + expect(booted?.state).toBe('device'); + expect(booted?.product).toBe('iOS 27.0'); + expect(booted?.is_emulator).toBeTrue(); + const shutdown = devices.find(d => d.serial === 'AAAA1111-2222-3333-4444-555566667777'); + expect(shutdown?.state).toBe('Shutdown'); + }); + + it('selects an iOS simulator by posting its platform and tracks it locally', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(70), + probes: [ + { + id: 'ios_simulators', + category: 'device', + title: 'iOS Simulator', + status: 'pass', + is_blocker: false, + summary: '1 Booted', + description: '', + metadata: { + simulators: [ + { + udid: 'UDID-1', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + } + ] + }, + actions: [] + } + ] + }); + + service.selectDevice('UDID-1', 'ios').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'UDID-1', platform: 'ios' }); + request.flush({ status: 'success', selected_serial: 'UDID-1', platform: 'ios' }); + + expect(service.selectedIosDevice()?.serial).toBe('UDID-1'); + expect(service.selectedDeviceSerial()).toBe('UDID-1'); + expect(service.isDeviceReady()).toBeTrue(); + }); + + it('clears the iOS selection when an Android device is selected instead', () => { + service.selectedIosDevice.set({ + serial: 'UDID-1', + state: 'device', + model: 'iPhone', + product: 'iOS 27.0', + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: true, + platform: 'ios' + }); + + service.selectDevice('emulator-5554', 'android').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'emulator-5554', platform: 'android' }); + request.flush({ status: 'success', selected_serial: 'emulator-5554', report: report(80) }); + + expect(service.selectedIosDevice()).toBeNull(); + }); + + const iosProbeWith = (metadata: any) => ({ + id: 'ios_simulators', + category: 'device', + title: 'iOS Devices', + status: 'pass', + is_blocker: false, + summary: 'iOS ready', + description: '', + metadata, + actions: [] + }); + + const envProbe = (id: string, status = 'pass') => ({ + id, + category: 'runtime', + title: id, + status, + is_blocker: true, + summary: status, + description: '', + metadata: {}, + actions: [] + }); + + it('merges paired physical iOS devices alongside simulators', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(90), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { + udid: 'SIM-1', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-27-0', + isAvailable: true + } + ], + connected_physical_devices: [ + { + udid: 'PHYS-1', + name: 'Office iPhone', + os_version: '27.1', + product_type: 'iPhone17,2' + }, + { name: 'No UDID — skipped', os_version: '27.0' } + ] + }) + ] + }); + + const devices = service.connectedDevices(); + const physical = devices.find(d => d.serial === 'PHYS-1'); + expect(devices.filter(d => d.platform === 'ios').length).toBe(2); + expect(physical?.platform).toBe('ios'); + expect(physical?.is_emulator).toBeFalse(); + expect(physical?.state).toBe('device'); + expect(physical?.product).toBe('iOS 27.1'); + expect(physical?.android_version).toBeNull(); + }); + + it('selects a physical iOS device by serial and posts its platform', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(91), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + connected_physical_devices: [ + { udid: 'PHYS-9', name: 'Office iPhone', os_version: '27.0' } + ] + }) + ] + }); + + service.selectDevice('PHYS-9', 'ios').subscribe(); + const request = http.expectOne('/api/system/devices/select'); + expect(request.request.body).toEqual({ serial: 'PHYS-9', platform: 'ios' }); + request.flush({ status: 'success', selected_serial: 'PHYS-9', platform: 'ios' }); + + expect(service.selectedIosDevice()?.serial).toBe('PHYS-9'); + expect(service.selectedIosDevice()?.is_emulator).toBeFalse(); + expect(service.isDeviceReady()).toBeTrue(); + }); + + it('reverts the optimistic iOS selection when the server rejects it', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(92), + probes: [ + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { + udid: 'SIM-2', + name: 'iPhone 18 Pro', + state: 'Booted', + runtime: 'r', + isAvailable: true + } + ] + }) + ] + }); + + service.selectDevice('SIM-2', 'ios').subscribe({ error: () => {} }); + expect(service.selectedIosDevice()?.serial).toBe('SIM-2'); // optimistic + http + .expectOne('/api/system/devices/select') + .flush('boom', { status: 500, statusText: 'Internal Server Error' }); + expect(service.selectedIosDevice()).toBeNull(); + }); + + it('gates iOS readiness on macOS + Xcode 27, not the ADB toolchain', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(93), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + { ...envProbe('android_adb', 'fail'), metadata: { installed: false } }, + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { udid: 'SIM-3', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + expect(service.isEnvironmentReady()).toBeFalse(); // no iOS selection yet + + service.selectDevice('SIM-3', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + + expect(service.isEnvironmentReady()).toBeTrue(); + }); + + it('keeps an iOS selection not-ready when Xcode is too old', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(94), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + iosProbeWith({ + xcode_27_or_newer: false, + simulators: [ + { udid: 'SIM-4', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + + service.selectDevice('SIM-4', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + + expect(service.isEnvironmentReady()).toBeFalse(); + }); + + it('restores ADB gates when an Android device is selected', () => { + service.fetchReadiness().subscribe(); + http.expectOne('/api/system/readiness').flush({ + ...report(95), + os_type: 'darwin', + probes: [ + envProbe('python_runtime'), + envProbe('system_config'), + { ...envProbe('android_adb', 'fail'), metadata: { installed: false } }, + { ...envProbe('toolchain'), is_blocker: false }, + iosProbeWith({ + xcode_27_or_newer: true, + simulators: [ + { udid: 'SIM-5', name: 'Sim', state: 'Booted', runtime: 'r', isAvailable: true } + ] + }) + ] + }); + + service.selectDevice('SIM-5', 'ios').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + expect(service.isEnvironmentReady()).toBeTrue(); + + service.selectDevice('emulator-5554', 'android').subscribe(); + http.expectOne('/api/system/devices/select').flush({ status: 'success' }); + expect(service.selectedIosDevice()).toBeNull(); + expect(service.isEnvironmentReady()).toBeFalse(); + }); + it('restores the standard local ADB server explicitly', () => { service.useLocalAdbServer(true).subscribe(); diff --git a/apps/showcase_ui/src/app/services/system.service.ts b/apps/showcase_ui/src/app/services/system.service.ts index b69697d9..af1e916a 100644 --- a/apps/showcase_ui/src/app/services/system.service.ts +++ b/apps/showcase_ui/src/app/services/system.service.ts @@ -56,6 +56,7 @@ export class SystemService { public configProbe = computed(() => this.probes().find(p => p.id === 'system_config') || null); public toolchainProbe = computed(() => this.probes().find(p => p.id === 'toolchain') || null); public adbProbe = computed(() => this.probes().find(p => p.id === 'android_adb') || null); + public iosProbe = computed(() => this.probes().find(p => p.id === 'ios_simulators') || null); public llmProbe = computed(() => this.probes().find(p => p.id === 'gemini_api_key' || p.id === 'llm_api_key') || null); public geminiProbe = computed(() => this.llmProbe()); public ocrProbe = computed(() => this.probes().find(p => p.id === 'vision_ocr_key' || p.id === 'ocr_api_key') || null); @@ -64,10 +65,16 @@ export class SystemService { public isEnvironmentReady = computed(() => { const py = this.pythonProbe(); const cfg = this.configProbe(); - const adb = this.adbProbe(); - const tc = this.toolchainProbe(); const pyOk = py?.status === 'pass'; const cfgOk = cfg?.status === 'pass'; + if (this.selectedIosDevice() != null) { + // An iOS target replaces the ADB toolchain gates: it needs macOS with + // Xcode 27+, never an ADB installation. + const iosOk = this.iosProbe()?.metadata?.['xcode_27_or_newer'] === true; + return pyOk && cfgOk && this.osType() === 'darwin' && iosOk; + } + const adb = this.adbProbe(); + const tc = this.toolchainProbe(); const adbInstalled = adb?.metadata?.['installed'] ?? (adb?.status !== 'fail'); const tcOk = tc?.status === 'pass' || tc?.is_blocker === false; return pyOk && cfgOk && adbInstalled && tcOk; @@ -85,7 +92,8 @@ export class SystemService { public isDeviceReady = computed(() => { const adb = this.adbProbe(); - return adb?.status === 'pass'; + // A selected iOS simulator satisfies the device requirement on its own. + return adb?.status === 'pass' || this.selectedIosDevice() != null; }); // Step-level counting matching the 3-step onboarding guide @@ -122,15 +130,64 @@ export class SystemService { } - // Device list from metadata + // Device list from metadata: Android devices plus available iOS simulators. public connectedDevices = computed(() => { - const meta = this.adbProbe()?.metadata; - if (meta && Array.isArray(meta['devices'])) { - return meta['devices'] as DeviceInfo[]; + const devices: DeviceInfo[] = []; + const adbMeta = this.adbProbe()?.metadata; + if (adbMeta && Array.isArray(adbMeta['devices'])) { + for (const d of adbMeta['devices'] as DeviceInfo[]) { + devices.push({ ...d, platform: d.platform ?? 'android' }); + } } - return []; + const iosMeta = this.iosProbe()?.metadata; + if (iosMeta && Array.isArray(iosMeta['simulators'])) { + for (const s of iosMeta['simulators'] as any[]) { + if (!s?.udid) continue; + devices.push({ + serial: s.udid, + state: s.state === 'Booted' ? 'device' : (s.state || 'unknown'), + model: s.name ?? null, + product: String(s.runtime ?? '') + .replace('com.apple.CoreSimulator.SimRuntime.', '') + .replace(/^([A-Za-z]+)-(.+)$/, (_m: string, name: string, ver: string) => `${name} ${ver.replace(/-/g, '.')}`) || null, + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: true, + platform: 'ios', + }); + } + } + if (iosMeta && Array.isArray(iosMeta['connected_physical_devices'])) { + for (const p of iosMeta['connected_physical_devices'] as any[]) { + if (!p?.udid) continue; + devices.push({ + serial: p.udid, + state: 'device', + model: p.name ?? null, + product: p.os_version ? `iOS ${p.os_version}` : 'iOS physical device', + android_version: null, + screen_resolution: null, + is_locked: null, + is_emulator: false, + platform: 'ios', + }); + } + } + return devices; }); + public iosDevices = computed( + () => this.connectedDevices().filter(d => d.platform === 'ios') + ); + + // iOS selection lives client-side: iOS has no ADB probe to retarget, so + // the chosen simulator is carried in the run payload instead. + public selectedIosDevice = signal(null); + public selectedDeviceSerial = computed( + () => this.selectedIosDevice()?.serial ?? this.activeDevice()?.serial ?? null + ); + // Installed local AVD emulators from metadata public installedAvds = computed(() => { const meta = this.adbProbe()?.metadata; @@ -496,13 +553,34 @@ export class SystemService { } /** - * Select a specific connected device serial as active target + * Select a specific connected device or iOS simulator as active target */ - public selectDevice(serial: string): Observable { + public selectDevice(serial: string, platform: 'android' | 'ios' = 'android'): Observable { + // iOS selection is client-side state (the run payload carries the binding); + // apply it optimistically since UDID validation can take seconds and the + // error path below reverts it if the server rejects the pick. + if (platform === 'ios') { + const dev = + this.connectedDevices().find( + d => d.serial === serial && d.platform === 'ios' + ) ?? null; + this.selectedIosDevice.set(dev); + } this.isLoading.set(true); - return this.http.post('/api/system/devices/select', { serial }).pipe( + return this.http.post('/api/system/devices/select', { serial, platform }).pipe( tap({ next: (res) => { + if (platform === 'ios') { + this.selectedIosDevice.set( + res?.status === 'success' + ? this.connectedDevices().find( + d => d.serial === serial && d.platform === 'ios' + ) ?? this.selectedIosDevice() + : null + ); + } else if (platform === 'android') { + this.selectedIosDevice.set(null); + } if (res?.report) { this.applyReadinessReport(res.report); } @@ -510,6 +588,9 @@ export class SystemService { }, error: (err) => { console.error('Failed to select active device:', err); + if (platform === 'ios') { + this.selectedIosDevice.set(null); + } this.isLoading.set(false); } }) diff --git a/apps/showcase_ui/tsconfig.json b/apps/showcase_ui/tsconfig.json index 5525117c..3e01b56f 100644 --- a/apps/showcase_ui/tsconfig.json +++ b/apps/showcase_ui/tsconfig.json @@ -4,6 +4,7 @@ "compileOnSave": false, "compilerOptions": { "outDir": "./dist/out-tsc", + "rootDir": "./src", "strict": true, "noImplicitOverride": true, "noPropertyAccessFromIndexSignature": true, diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index 52dff2ca..93cc2b41 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -67,6 +67,7 @@ ) from artemis.utils.visualization import format_minimal_list_with_elements from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -255,6 +256,8 @@ def _load_prompts() -> dict[str, str]: def probes_enabled(ctx: ArtemisContext) -> bool: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + return False setup = getattr(ctx, "execution_setup", None) return not (setup and getattr(setup, "disable_device_probes", False)) @@ -842,7 +845,7 @@ async def run_final_check( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 13d80317..b87e7e7f 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -54,6 +54,7 @@ get_recent_subgoal_hashes, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -72,6 +73,8 @@ def __init__(self, ctx: ArtemisContext): self._llm = None def _is_device_online(self) -> bool: + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": + return getattr(self.ctx, "_active_driver", None) is not None try: if self.ctx.adb_client is None: return False @@ -111,6 +114,9 @@ def _build_traced_tools(self) -> list: *get_history_tools(self.ctx), get_submit_answer_tool(self.ctx), ] + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) == "ios": + # logcat-backed analysis can never succeed on iOS. + all_tools = [t for t in all_tools if t.name not in ("run_adb_command", "analyze_logs")] if not self.is_device_online: logger.info( "Diagnoser running in OFFLINE environment: stripping adb short" @@ -227,7 +233,7 @@ def _build_initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) diff --git a/artemis/agents/explorer/native_runner.py b/artemis/agents/explorer/native_runner.py index 4b29bd1b..afe98442 100644 --- a/artemis/agents/explorer/native_runner.py +++ b/artemis/agents/explorer/native_runner.py @@ -28,6 +28,7 @@ from collections.abc import Awaitable, Callable import json import os +from pathlib import Path import time from typing import TYPE_CHECKING, Any @@ -45,6 +46,7 @@ ) from artemis.services.llm import _record_llm_event, _record_llm_retry from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -188,12 +190,13 @@ async def get_image_part(file_path: str) -> types.Part: return types.Part( file_data=types.FileData( file_uri=file_ref.uri, - mime_type=file_ref.mime_type or "image/jpeg", + mime_type=file_ref.mime_type + or image_mime_type(Path(file_path).read_bytes()), ) ) with open(file_path, "rb") as f: img_bytes = f.read() - return types.Part.from_bytes(data=img_bytes, mime_type="image/jpeg") + return types.Part.from_bytes(data=img_bytes, mime_type=image_mime_type(img_bytes)) return get_image_part diff --git a/artemis/agents/explorer/screen_index.py b/artemis/agents/explorer/screen_index.py index 837082a2..a91b0b2f 100644 --- a/artemis/agents/explorer/screen_index.py +++ b/artemis/agents/explorer/screen_index.py @@ -24,7 +24,7 @@ import re from typing import Any, Literal -from artemis.utils.visualization import parse_bounds +from artemis.utils.visualization import get_hit_point, parse_bounds Bounds = tuple[int, int, int, int] ElementSource = Literal["xml", "ocr"] @@ -84,6 +84,8 @@ class ScreenElement: @property def center(self) -> tuple[int, int]: + if (point := get_hit_point(self.node)) is not None: + return point left, top, right, bottom = self.bounds return (left + right) // 2, (top + bottom) // 2 @@ -173,8 +175,12 @@ def add(element: ScreenElement) -> None: for node in fused_xml or []: if not isinstance(node, dict): continue + if "hit_point" in node and get_hit_point(node, width, height) is None: + node = {key: value for key, value in node.items() if key != "hit_point"} class_name = node.get("class") or node.get("className") - resource_id = node.get("resource-id") or node.get("resourceId") + resource_id = ( + node.get("resource-id") or node.get("resourceId") or node.get("resource_id") + ) interactive = any(_is_truthy_attr(node.get(key)) for key in INTERACTION_KEYS) for ocr in node.get("ocr_elements") or []: diff --git a/artemis/agents/explorer/universal_runner.py b/artemis/agents/explorer/universal_runner.py index c5b5bc89..48f0dbe2 100644 --- a/artemis/agents/explorer/universal_runner.py +++ b/artemis/agents/explorer/universal_runner.py @@ -36,6 +36,7 @@ from artemis.graph.state import State from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -172,7 +173,7 @@ def _build_universal_messages( }, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] @@ -266,9 +267,7 @@ def _build_tool_images_message(self, images: list[tuple[str, str]]) -> HumanMess continue if tool_name not in tool_names: tool_names.append(tool_name) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if not blocks: return None header = { diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 3ea3f686..61cdb852 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -95,6 +95,7 @@ parse_swipe_parameters, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -296,7 +297,11 @@ def _render_system_prompt(self, tools_declaration: list) -> str: prompt_path = Path(__file__).parent / "flash_runner.md" prompt_template = prompt_path.read_text(encoding="utf-8") available_tools = frozenset(t.name for t in tools_declaration) - return Template(prompt_template).render(goal=self.goal, available_tools=available_tools) + from artemis.agents.platform_guidance import device_action_guidance + + return device_action_guidance(self.ctx) + Template(prompt_template).render( + goal=self.goal, available_tools=available_tools + ) # ------------------------------------------------------------------ # Per-turn helpers (observe / think) @@ -369,9 +374,7 @@ def _build_tail( if img_bytes: img_b64 = base64.b64encode(img_bytes).decode("utf-8") blocks.append({"type": "text", "text": "--- Current Screenshot ---"}) - blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}} - ) + blocks.append({"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}) if xml_list: blocks.append({"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n{xml_list}"}) ephemeral: list[int] = [] diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 5b8a3cf9..c577717d 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -41,6 +41,7 @@ from artemis.utils.task_tree import format_actions_clean from artemis.utils.visualization import draw_action_overlay_on_image from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -430,7 +431,7 @@ async def _attempt(self, key: JobKey) -> bool: } ) content_blocks.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_pre}"}} + {"type": "image_url", "image_url": {"url": image_data_uri(b64_pre)}} ) if post_bytes: @@ -449,7 +450,7 @@ async def _attempt(self, key: JobKey) -> bool: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_post}"}, + "image_url": {"url": image_data_uri(b64_post)}, } ) diff --git a/artemis/agents/image_processor/image_processor.py b/artemis/agents/image_processor/image_processor.py index 4ce52698..2ee22724 100644 --- a/artemis/agents/image_processor/image_processor.py +++ b/artemis/agents/image_processor/image_processor.py @@ -35,6 +35,7 @@ from artemis.services.llm import get_llm from artemis.utils.python_executor import PythonExecutor from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -118,7 +119,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: {"type": "text", "text": "Here is the target image. Begin writing your code."}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}, + "image_url": {"url": image_data_uri(img_b64)}, }, ] ), @@ -186,9 +187,7 @@ async def run(self, instruction: str, target_image_path: str) -> dict: image_blocks.append( { "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{b64_data}" - }, + "image_url": {"url": image_data_uri(b64_data)}, } ) image_blocks.append({"type": "text", "text": f"\n{label}\n"}) diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index b2acb4f5..dae8f8e8 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -26,6 +26,7 @@ from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -107,7 +108,7 @@ async def _run_object_detection( image_bytes: bytes | str | Path | None = None, queries: list[str] | None = None, templates: list[str] | None = None, - mime_type: str = "image/jpeg", + mime_type: str | None = None, global_timeout: float = 30.0, image_path: str | Path | None = None, ) -> dict: @@ -120,6 +121,7 @@ async def _run_object_detection( image_data = Path(target_img).read_bytes() else: image_data = target_img + mime_type = mime_type or image_mime_type(image_data) queries = queries or [] templates = templates or ["Point to the following objects: {labels_str}"] diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index d2453627..4514a2b2 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -72,7 +72,15 @@ } # Bare key names press_key accepts (case-insensitive, optional KEYCODE_ prefix). -SUPPORTED_PRESS_KEYS = ("ENTER", "BACK", "HOME", "APP_SWITCH") +SUPPORTED_PRESS_KEYS = ( + "ENTER", + "BACK", + "HOME", + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN", +) from artemis.agents.operator.prompts import ( OPERATOR_MAX_TOOL_ITERATIONS, @@ -919,7 +927,9 @@ async def __call__(self, state: State): # The analyzer is always available: the "output truncated, use # analyze_task_output" hint arrives mid-turn, and tools are bound once per # turn, so a conditional mount would be one turn late. - if not any(t.name == "analyze_task_output" for t in all_tools): + if getattr(getattr(self.ctx, "device", None), "mobile_platform", None) != "ios" and not any( + t.name == "analyze_task_output" for t in all_tools + ): all_tools.append(analyze_task_output_wrapper.tool_fn_getter(self.ctx)) traced_tools = [trace_langchain_tool(t, self.ctx) for t in all_tools] diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index e800de23..d74659d0 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -40,6 +40,7 @@ from artemis.agents.prompt_assembly import render_tool_enum, resolve_available from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER +from artemis.utils.image_mime import image_data_uri @lru_cache(maxsize=1) @@ -203,6 +204,8 @@ def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]: # video_recording_tools_enabled); the prompt must not advertise it when the # tool is not actually available this run. available = set(OPERATOR_PROMPT_TOOLSET) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + available.difference_update({"run_adb_command", "manage_task", "analyze_task_output"}) setup = getattr(ctx, "execution_setup", None) if not (setup and getattr(setup, "video_recording_tools_enabled", False)): available.discard("video_analyzer") @@ -316,6 +319,9 @@ def render_transcript_static_system( available = resolve_operator_prompt_tools(ctx) static_template = apply_operator_prompt_contract(static_template, available_tools=available) + from artemis.agents.platform_guidance import device_action_guidance + + static_template = device_action_guidance(ctx) + static_template return Template(static_template).render( initial_goal=state.initial_goal, subgoals_status="", @@ -354,6 +360,9 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte available = resolve_operator_prompt_tools(ctx) prompt_template = apply_operator_prompt_contract(prompt_template, available_tools=available) + from artemis.agents.platform_guidance import device_action_guidance + + prompt_template = device_action_guidance(ctx) + prompt_template plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.") @@ -384,7 +393,7 @@ async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisConte builder.add_human_content( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"}, + "image_url": {"url": image_data_uri(latest_screenshot_b64)}, } ) builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index 9b75b427..88259b63 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -46,6 +46,7 @@ from artemis.utils.task_tree import get_active_subgoal_hashes from third_party.mobile_use.utils.logger import get_logger from pydantic import BaseModel +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -274,7 +275,7 @@ def _initial_messages( content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 02be5c7c..e3374505 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -54,6 +54,7 @@ from artemis.utils.cython_compat import CyFunctionDetector from third_party.mobile_use.utils.decorators import agent_lifecycle_logging from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -279,7 +280,7 @@ async def __call__(self, state: State): human_message_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) messages = [ diff --git a/artemis/agents/platform_guidance.py b/artemis/agents/platform_guidance.py new file mode 100644 index 00000000..8418cece --- /dev/null +++ b/artemis/agents/platform_guidance.py @@ -0,0 +1,29 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared platform constraints for the Flash and Pro device-action prompts.""" + + +def device_action_guidance(ctx) -> str: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) != "ios": + return "" + return ( + "Target platform: iOS. For input_text, explicitly set clear_exist=false; " + "type into an empty field or at its existing cursor. Whole-field clearing, Android " + "keycodes, Back/Delete keys, ADB commands, and app locking are " + "unavailable. Navigate with visible iOS controls; press_key supports enter, home, " + "power, volume_up, volume_down, and app_switch. manage_app accepts installed iOS " + "display names or bundle identifiers. Action coordinates use the screenshot's " + "normalized 0-1000 space.\n\n" + ) diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index 80a542fb..5cf32a4c 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -36,6 +36,7 @@ from artemis.services.llm import acomplete_structured from artemis.utils import visualization from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -120,12 +121,12 @@ def _build_messages( {"type": "text", "text": "[Image 1 (Reference)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{orig_b64}"}, + "image_url": {"url": image_data_uri(orig_b64)}, }, {"type": "text", "text": "[Image 2 (Current State)]"}, { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{live_b64}"}, + "image_url": {"url": image_data_uri(live_b64)}, }, ] if state: diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index fba8dd75..b2d30b46 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -45,6 +45,7 @@ READ_NOTE_DOCSTRING, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -79,7 +80,7 @@ def to_langchain_tool_message(self) -> ToolMessage: content_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_img}"}, + "image_url": {"url": image_data_uri(b64_img)}, } ) return ToolMessage( diff --git a/artemis/agents/video_analyzer/native_coordinator.py b/artemis/agents/video_analyzer/native_coordinator.py index 6cfab85d..b0994f89 100644 --- a/artemis/agents/video_analyzer/native_coordinator.py +++ b/artemis/agents/video_analyzer/native_coordinator.py @@ -34,6 +34,7 @@ from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from artemis.llm.google import normalize_usage from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -171,10 +172,11 @@ def _append_ledger_diff(analyzer, contents: list, last_ledger_index: int) -> int image_parts.append( types.Part.from_text(text=f"PROOF: {Path(screenshot_file).name}") ) + image_bytes = Path(screenshot_file).read_bytes() image_parts.append( types.Part.from_bytes( - data=Path(screenshot_file).read_bytes(), - mime_type="image/jpeg", + data=image_bytes, + mime_type=image_mime_type(image_bytes), ) ) lines.append(line) diff --git a/artemis/agents/video_analyzer/universal_engine.py b/artemis/agents/video_analyzer/universal_engine.py index 96ade3fe..4eaf8fe2 100644 --- a/artemis/agents/video_analyzer/universal_engine.py +++ b/artemis/agents/video_analyzer/universal_engine.py @@ -42,6 +42,7 @@ ) from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -159,7 +160,7 @@ async def _build_universal_user_blocks( user_blocks.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{b64_str}"}, + "image_url": {"url": image_data_uri(b64_str)}, } ) diff --git a/artemis/context.py b/artemis/context.py index 57360c8c..c9738688 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -198,6 +198,17 @@ class ArtemisContext(DeviceClientAccessors, BaseModel): async def __aenter__(self) -> ArtemisContext: return self + async def disconnect_driver(self) -> None: + """Release the cached driver, including partially initialized transports.""" + driver = self._active_driver + self._active_driver = None + self._mobile_controller = None + if driver is not None: + try: + await driver.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug(f"Device driver disconnect failed: {exc}", exc_info=True) + async def __aexit__(self, exc_type, exc_val, exc_tb): # Close the in-process action session before draining background tasks so its # owner task exits cleanly rather than being cancelled below. @@ -283,6 +294,8 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): except Exception as exc: logger.debug(f"DataEngine shutdown failed; skipped: {exc}", exc_info=True) + await self.disconnect_driver() + from artemis.data_engine.engine import DataEngine diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index d0dc1d03..82a74ed4 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -19,12 +19,13 @@ import subprocess import tempfile import time -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 from artemis.config.paths import get_temp_dir from artemis.context import ArtemisContext from artemis.drivers.factory import get_driver +from artemis.drivers.ios.recording import IosRecordingSession from artemis.utils.video import ( ANDROID_RECORDING_SEGMENT_SECONDS, await_scrcpy_first_frame, @@ -48,6 +49,9 @@ set_active_session, ) +if TYPE_CHECKING: + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + logger = get_logger(__name__) @@ -56,6 +60,18 @@ def __init__(self, ctx: ArtemisContext): super().__init__(ctx, get_driver(ctx)) self._segment_cache: dict[tuple[str, int, float, float], VideoRecordingResult] = {} + async def open_url(self, url: str) -> bool: + if self.ctx.device.mobile_platform == "ios": + return await self._driver.open_url(url) + return await super().open_url(url) + + async def erase_text(self, nb_chars: int | None = None) -> bool: + if self.ctx.device.mobile_platform == "ios": + # No native clear/Backspace key exists; the driver raises + # NotImplementedError for unsupported clear operations. + return await self._driver.input_text("", clear_existing=True) + return await super().erase_text(nb_chars) + @staticmethod async def _spawn_scrcpy(command: list[str]) -> asyncio.subprocess.Process: kwargs: dict[str, Any] = { @@ -131,6 +147,21 @@ async def extract_segment_metadata( ) return cached_res + if isinstance(session, IosRecordingSession): + result = await self._extract_ios_segment(session, start_time, end_time, output_path) + if ( + cache_key is not None + and result.success + and result.video_path + and result.video_path.exists() + ): + # Sealing bumps the generation; key the entry under the + # post-seal generation a repeated lookup will observe. + self._segment_cache[ + (cache_key[0], session.generation, cache_key[2], cache_key[3]) + ] = result + return result + try: mkv_path = session.local_video_path if not mkv_path: @@ -397,8 +428,10 @@ async def start_video_recording( output_dir: Path | None = None, max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, ) -> VideoRecordingResult: - """Start screen recording on Android device using scrcpy.""" + """Start screen recording using the platform's native capture.""" self._segment_cache.clear() + if self.ctx.device.mobile_platform == "ios": + return await self._start_ios_recording(output_dir, max_duration_seconds) device_id = self._get_device_id() # Check mock driver first @@ -517,8 +550,10 @@ async def start_video_recording( return abort_recording(device_id, "start", e) async def stop_video_recording(self) -> VideoRecordingResult: - """Stop scrcpy recording and return the converted MP4 video file.""" + """Stop recording and return the finalized MP4 video file.""" self._segment_cache.clear() + if self.ctx.device.mobile_platform == "ios": + return await self._stop_ios_recording() device_id = self._get_device_id() # Check mock driver first @@ -645,5 +680,242 @@ async def stop_video_recording(self) -> VideoRecordingResult: self._record_recording_failure(session, str(e)) return abort_recording(device_id, "stop", e) + def _ios_recording_driver(self) -> "XcodeSimulatorDriver | None": + """The active driver when it supports native iOS capture, else ``None``.""" + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + return self._driver if isinstance(self._driver, XcodeSimulatorDriver) else None + + async def _start_ios_recording( + self, output_dir: Path | None, max_duration_seconds: int + ) -> VideoRecordingResult: + """Start native capture through the iOS driver (simctl or devicectl).""" + device_id = self._get_device_id() + if already_active := recording_already_active(device_id): + return already_active + driver = self._ios_recording_driver() + if driver is None: + return VideoRecordingResult( + success=False, + message="iOS recording requires the iOS driver.", + ) + try: + await driver.start_video_recording( + output_dir, max_duration_seconds=max_duration_seconds + ) + except Exception as exc: + session = driver.recording_session + if session is None: + session = IosRecordingSession( + video_id=uuid4(), + device_id=device_id, + start_time=time.time(), + local_video_path=(Path(output_dir) / "recording.mov" if output_dir else None), + is_active=False, + ) + if session.data_engine_start_time is None and self.ctx and self.ctx.data_engine: + session.data_engine_start_time = ( + self.ctx.data_engine.session_start_time or session.start_time + ) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=session.video_id, + device_id=device_id, + local_video_path=session.local_video_path or "", + start_time=session.start_time, + ) + self._record_recording_failure(session, f"iOS recording failed to start: {exc}") + return VideoRecordingResult( + success=False, message=f"iOS recording failed to start: {exc}" + ) + session = driver.recording_session + if session is None: + return VideoRecordingResult(success=False, message="iOS recorder returned no session") + session.data_engine_start_time = ( + self.ctx.data_engine.session_start_time + if (self.ctx and self.ctx.data_engine) + else session.start_time + ) + set_active_session(device_id, session) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=session.video_id, + device_id=device_id, + local_video_path=session.local_video_path, + start_time=session.start_time, + ) + warning = "; ".join(session.errors) or None + return VideoRecordingResult( + success=True, + message=f"iOS recording started on {device_id}", + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:active", + warning=warning, + ) + + async def _stop_ios_recording(self) -> VideoRecordingResult: + """Finalize the driver's recorder and report the sealed manifest.""" + device_id = self._get_device_id() + driver = self._ios_recording_driver() + session = get_active_session(device_id) + if session is None: + session = driver.recording_session if driver is not None else None + if session is None or session.video_id is None: + return no_active_recording(device_id) + if driver is None: + message = "iOS recording stop requires the Xcode simulator driver." + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + async def _finalize() -> str | None: + session.is_active = False + return await driver.stop_video_recording() + + try: + final_path = await asyncio.shield(_finalize()) + except asyncio.CancelledError: + raise + except Exception as exc: + self._record_recording_failure(session, f"iOS recording stop failed: {exc}") + return VideoRecordingResult(success=False, message=f"iOS recording stop failed: {exc}") + finally: + remove_active_session(device_id) + + missing = [ + record + for record in session.segments + if not ( + record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ) + ] + if final_path is None or not session.segments or missing: + message = "; ".join(session.errors) or "iOS recording finalized with no valid segments" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + final_video_path = Path(final_path) + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_stop( + video_id=session.video_id, + device_id=device_id, + local_video_path=final_video_path, + start_time=session.start_time, + end_time=session.start_time + session.sealed_until, + ) + warning = "; ".join(session.errors) or None + return VideoRecordingResult( + success=True, + message=f"iOS recording stopped, saved {len(session.segments)} segments", + video_path=final_video_path, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=f"{session.video_id}:{session.generation}:ready", + warning=warning, + ) + + async def _extract_ios_segment( + self, + session: IosRecordingSession, + start_time: float, + end_time: float | None, + output_path: Path | None, + ) -> VideoRecordingResult: + """Render a clip from sealed iOS MP4 segments only. + + The open .mov is never handed to FFmpeg: the driver seals the segment + at the requested recording-relative end first, then finalized segments + render through the shared timeline renderer. Recorder restarts leave + their gaps black rather than stretching adjacent frames. + """ + device_id = self._get_device_id() + anchor = session.data_engine_start_time + shift = (session.start_time - anchor) if anchor is not None else 0.0 + recording_start = float(start_time) - shift + recording_end = float(end_time) - shift if end_time is not None else None + + driver = self._ios_recording_driver() + if session.is_active and recording_end is not None and driver is not None: + await driver.seal_recording_segment(through_time=recording_end) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + + warning = None + available_end = session.sealed_until + if recording_end is None or recording_end > available_end: + if recording_end is not None: + warning = ( + "Requested end clipped to latest sealed iOS capture " + f"({available_end + shift:.1f}s of session time)" + ) + recording_end = available_end + + records = [dict(record) for record in session.segments] + first_capture = min((float(record["start"]) for record in records), default=None) + if first_capture is not None and recording_start < first_capture: + leading = "Requested start preceded the first captured frame; clipped to capture start" + warning = f"{warning}; {leading}" if warning else leading + recording_start = first_capture + + if not records or recording_end <= recording_start: + message = "Requested range does not overlap sealed iOS recording" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + timeline = [ + { + "path": str(record["output_path"]), + "start": float(record["start"]), + "end": float(record["end"]), + } + for record in records + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and float(record["end"]) > recording_start + and float(record["start"]) < recording_end + ] + timeline.sort(key=lambda segment: segment["start"]) + if not timeline: + message = "No finalized iOS MP4 segments overlap the requested range" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + trim_output_path = ( + Path(output_path) + if output_path is not None + else Path(tempfile.mkdtemp(prefix="video_trimmed_", dir=get_temp_dir("trimmed_videos"))) + / "segment.mp4" + ) + success = await render_timeline_clip( + timeline, recording_start, recording_end, trim_output_path + ) + if not success or not trim_output_path.exists(): + message = "Failed to render clip from sealed iOS segments" + self._record_recording_failure(session, message) + return VideoRecordingResult(success=False, message=message) + + actual_start = recording_start + shift + actual_end = recording_end + shift + file_size_mb = trim_output_path.stat().st_size / (1024 * 1024) + return VideoRecordingResult( + success=True, + message=( + f"iOS clip rendered for session range {actual_start:.1f}s to {actual_end:.1f}s" + ), + video_path=trim_output_path, + file_size_mb=round(file_size_mb, 2), + duration_seconds=round(recording_end - recording_start, 2), + actual_start_relative_time=actual_start, + warning=warning, + video_id=session.video_id, + generation=session.generation, + sealed_until=session.sealed_until, + source_revision=(f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"), + ) + async def cleanup(self) -> None: await self._driver.disconnect() diff --git a/artemis/core/diagnostics/device_smoke.py b/artemis/core/diagnostics/device_smoke.py index 3c43c9cf..d2469346 100644 --- a/artemis/core/diagnostics/device_smoke.py +++ b/artemis/core/diagnostics/device_smoke.py @@ -56,7 +56,9 @@ import time from typing import Any, TypeVar +from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.runtime.device_lock import DeviceExecutionLock +from artemis.runtime.ios_observation import observe_ios_controller from third_party.mobile_use.utils.logger import get_logger logger = get_logger(__name__) @@ -96,8 +98,55 @@ def _uiautomator_fix(serial: str | None) -> list[str]: ] -def fix_for_error(error: str | None, serial: str | None) -> list[str]: +def _ios_fix_for_error(error: str | None, serial: str | None) -> list[str]: + """iOS repair steps (simctl/Xcode for simulators, devicectl/WDA for hardware).""" + if not error: + return [] + s = serial or "" + text = (error or "").lower() + if "busy" in text or "another task" in text: + return [ + "Wait for the running task to finish, or stop it with mobile_manage_task(action='stop', ...).", + "Then rerun the smoke test.", + ] + if "approv" in text or "authoriz" in text or "permission" in text: + return [ + "Approve Artemis's agent access in Xcode (the first run opens a workspace approval prompt; choose persistent approval).", + "Confirm the permitted agent and workspace under Xcode > Settings > Agentic Tools.", + ] + if "xcode" in text or "macos" in text or "darwin" in text: + return [ + "Install Xcode 27 or later and select it: sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", + ] + if ( + "webdriveragent" in text + or "wda" in text + or "physical" in text + or "pair" in text + or "trust" in text + or "devicectl" in text + or "developer mode" in text + ): + return [ + f"List paired devices: xcrun devicectl list devices; check capture: xcrun devicectl device capture screenshot --device {s} --destination /tmp/ios.png", + "Pair the device (plug in, tap Trust), enable Developer Mode under Settings > Privacy & Security, and ensure WebDriverAgent is installed " + "(build-for-testing via Xcode, or set ARTEMIS_IOS_WDA_URL to a running server).", + ] + if "simulator" in text or "udid" in text or "boot" in text or "not available" in text: + return [ + f"List simulators: xcrun simctl list devices; boot one with: xcrun simctl boot {s}", + "Create a simulator in Xcode > Settings > Platforms if none exist.", + ] + return [ + "Check native capture manually: xcrun simctl io booted screenshot /tmp/ios.png", + 'Verify Xcode MCP access: echo \'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}\' | xcrun mcpbridge', + ] + + +def fix_for_error(error: str | None, serial: str | None, platform: str = "android") -> list[str]: """Map an error string to concrete repair steps (substring match, case-insensitive).""" + if normalize_device_platform(platform, strict=False) == "ios": + return _ios_fix_for_error(error, serial) if not error: return [] text = error.lower() @@ -260,19 +309,31 @@ def _decoded_length(b64: Any) -> int | None: async def smoke_test_device( - device_serial: str | None = None, timeout_seconds: float = 20.0 + device_serial: str | None = None, + timeout_seconds: float = 20.0, + platform: str = "android", ) -> dict[str, Any]: """Observe the device exactly like ``mobile_get_device_state`` and report a verdict. + ``platform="ios"`` exercises the native iOS path (``simctl`` + mcpbridge + for simulators, ``devicectl`` + WebDriverAgent for physical devices) + instead of ADB/UIAutomator. + Never raises. Returns:: {"ok": bool, "serial": str | None, "elapsed_seconds": float, "screenshot_bytes": int | None, "element_count": int | None, - "hierarchy_backend": "helper" | "uiautomator" | None, + "hierarchy_backend": "helper" | "uiautomator" | "xcode" | "wda" | None, "error": str | None, "fix": list[str]} """ started = time.monotonic() - requested_serial = _resolve_requested_serial(device_serial) + is_ios = normalize_device_platform(platform, strict=False) == "ios" + # iOS targets are UDIDs; ADB_DEVICE_SERIAL is Android-only fallback. + requested_serial = ( + (device_serial or os.environ.get("ARTEMIS_DEVICE_ID")) + if is_ios + else _resolve_requested_serial(device_serial) + ) result: dict[str, Any] = { "ok": False, "serial": requested_serial, @@ -289,10 +350,14 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any result["elapsed_seconds"] = round(time.monotonic() - started, 3) result["error"] = error result["ok"] = error is None - result["fix"] = fix_for_error(cause if cause is not None else error, result["serial"]) + result["fix"] = fix_for_error( + cause if cause is not None else error, result["serial"], platform + ) return result - busy = _find_busy_owner(requested_serial) + # iOS skips this unscoped sweep: observe_ios_controller performs atomic, + # scope-correct lease admission on the resolved UDID itself. + busy = None if is_ios else _find_busy_owner(requested_serial) if busy is not None: key, owner = busy description = getattr(owner, "description", "") or "unknown task" @@ -306,20 +371,33 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any try: from artemis.mcp import adb_server - controller = await _run_in_daemon_thread( - lambda: adb_server._get_controller(device_serial=device_serial), - timeout_seconds, - "controller-init", - ) + if is_ios: + # iOS controller construction enumerates simctl/devicectl lazily; + # offload the sync factory instead of the daemon-thread hop used + # for the blocking Android UIAutomator handshake. + controller = await asyncio.wait_for( + asyncio.to_thread( + adb_server._get_controller, + device_serial=device_serial, + target_platform="ios", + ), + timeout_seconds, + ) + else: + controller = await _run_in_daemon_thread( + lambda: adb_server._get_controller(device_serial=device_serial), + timeout_seconds, + "controller-init", + ) except TimeoutError: - return _finish( - f"UIAutomator/controller initialization did not respond within {timeout_seconds:g}s" - ) + label = "iOS driver" if is_ios else "UIAutomator/controller" + return _finish(f"{label} initialization did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught cause = _describe_failure(exc) - return _finish(f"Failed to initialize Android device controller: {cause}", cause) + target = "iOS device" if is_ios else "Android device" + return _finish(f"Failed to initialize {target} controller: {cause}", cause) controller_ctx = getattr(controller, "ctx", None) device = getattr(controller_ctx, "device", None) @@ -329,13 +407,25 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any remaining = max(0.5, timeout_seconds - (time.monotonic() - started)) try: - device_data = await _run_in_daemon_thread( - lambda: _run_coroutine_blocking(controller.get_screen_data), - remaining, - "screen-data", - ) + if is_ios: + # Runs on the caller's loop: the helper owns lease + connect + + # capture + disconnect + release and is cancellation-safe, unlike + # the Android coroutine which must be bounced onto a private + # thread/loop because UIAutomator blocks the loop thread. + device_data = await asyncio.wait_for(observe_ios_controller(controller), remaining) + else: + + async def _observe() -> Any: + return await controller.get_screen_data() + + device_data = await _run_in_daemon_thread( + lambda: _run_coroutine_blocking(_observe), + remaining, + "screen-data", + ) except TimeoutError: - return _finish(f"UIAutomator/screen capture did not respond within {timeout_seconds:g}s") + label = "iOS session capture" if is_ios else "UIAutomator/screen capture" + return _finish(f"{label} did not respond within {timeout_seconds:g}s") except _PASSTHROUGH_EXCEPTIONS: raise except BaseException as exc: # pylint: disable=broad-exception-caught @@ -344,9 +434,24 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any result["screenshot_bytes"] = _decoded_length(getattr(device_data, "base64", None)) result["element_count"] = _count_elements(getattr(device_data, "elements", None)) - from artemis.clients.screen_client_factory import describe_backend + if is_ios: + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + # Report the canonical UDID the observation lease actually resolved. + resolved_serial = getattr(device, "device_id", None) + if isinstance(resolved_serial, str) and resolved_serial: + result["serial"] = resolved_serial + result["hierarchy_backend"] = ( + "wda" + if isinstance(getattr(controller, "_driver", None), PhysicalIosDriver) + else "xcode" + ) + else: + from artemis.clients.screen_client_factory import describe_backend - result["hierarchy_backend"] = describe_backend(getattr(controller_ctx, "ui_adb_client", None)) + result["hierarchy_backend"] = describe_backend( + getattr(controller_ctx, "ui_adb_client", None) + ) if result["screenshot_bytes"] is None: return _finish("Screen capture returned no screenshot data") @@ -355,9 +460,8 @@ def _finish(error: str | None = None, cause: str | None = None) -> dict[str, Any f"Screenshot capture failed (driver returned a {result['screenshot_bytes']}-byte placeholder image)" ) if not result["element_count"]: - return _finish( - "UIAutomator hierarchy dump returned no UI elements (screenshot worked, hierarchy did not)" - ) + backend = "Xcode accessibility tree" if is_ios else "UIAutomator hierarchy dump" + return _finish(f"{backend} returned no UI elements (screenshot worked, hierarchy did not)") return _finish(None) diff --git a/artemis/core/diagnostics/engine.py b/artemis/core/diagnostics/engine.py index 97f9d20c..56b93cf8 100644 --- a/artemis/core/diagnostics/engine.py +++ b/artemis/core/diagnostics/engine.py @@ -27,6 +27,7 @@ LLMCredentialsProbe, VisionOCRProbe, ) +from artemis.core.diagnostics.probes.ios_probe import IosDeviceProbe from artemis.core.diagnostics.probes.runtime_probe import ( PythonRuntimeProbe, SystemConfigProbe, @@ -67,6 +68,7 @@ def __init__(self): self._credentials_probe = LLMCredentialsProbe() self._ocr_probe = VisionOCRProbe() self._adb_probe = AdbDeviceProbe() + self._ios_probe = IosDeviceProbe() self._report_cache: SystemReadinessReport | None = None self._report_cache_time = 0.0 self._report_cache_generation = -1 @@ -80,6 +82,7 @@ def __init__(self): self.register_probe(self._credentials_probe) self.register_probe(self._ocr_probe) self.register_probe(self._adb_probe) + self.register_probe(self._ios_probe) def register_probe(self, probe: BaseProbe) -> None: """Register a new diagnostic probe.""" diff --git a/artemis/core/diagnostics/probes/ios_probe.py b/artemis/core/diagnostics/probes/ios_probe.py new file mode 100644 index 00000000..bb4c7865 --- /dev/null +++ b/artemis/core/diagnostics/probes/ios_probe.py @@ -0,0 +1,170 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Device Readiness Probe. + +Informational only (``is_blocker = False``): iOS support is opt-in per task, +so a missing Xcode toolchain must never block the Android default path. The +probe reports whether this host could run an iOS task today — macOS, Xcode +27+, and at least one available simulator or paired+connected physical +device. +""" + +import asyncio + +from artemis.core.diagnostics.probes.base import BaseProbe +from artemis.core.diagnostics.schema import ( + ProbeAction, + ProbeCategory, + ProbeResult, + ProbeStatus, +) +from artemis.drivers.ios.discovery import ( + is_physical_ios, + list_core_devices, + list_ios_simulators, + parse_xcode_version, + physical_ios_ready, + run_xcrun, + simctl_available, +) + + +class IosDeviceProbe(BaseProbe): + """Reports iOS automation capability: Xcode 27+, simulators, physical.""" + + @property + def probe_id(self) -> str: + # Historical id kept stable: persisted reports reference it. + return "ios_simulators" + + @property + def category(self) -> ProbeCategory: + return ProbeCategory.DEVICE + + @property + def is_blocker(self) -> bool: + return False + + async def probe(self) -> ProbeResult: + if not simctl_available(): + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.SKIPPED, + is_blocker=self.is_blocker, + summary="Unavailable", + description=( + "iOS automation requires macOS with Xcode 27+; " + "this host cannot enumerate iOS devices." + ), + metadata={"platform_supported": False}, + ) + + xcode_ok = False + xcode_version: str | None = None + try: + raw = await asyncio.wait_for(run_xcrun("xcodebuild", "-version"), timeout=15.0) + xcode_version = parse_xcode_version(raw) + if xcode_version: + xcode_ok = int(xcode_version.split(".")[0]) >= 27 + except (OSError, RuntimeError, TimeoutError): + xcode_version = None + + simulators = await list_ios_simulators() + booted = [d for d in (simulators or []) if d.get("state") == "Booted"] + core_devices = await list_core_devices() + physical = [d for d in (core_devices or []) if is_physical_ios(d)] + connected = [d for d in physical if physical_ios_ready(d)] + metadata = { + "platform_supported": True, + "xcode_version": xcode_version, + "xcode_27_or_newer": xcode_ok, + "simulator_count": len(simulators or []), + "booted_udids": [d.get("udid") for d in booted], + "simulators": simulators or [], + "physical_device_count": len(physical), + "connected_physical_devices": [ + { + "udid": d.get("udid"), + "name": d.get("name"), + "os_version": d.get("os_version"), + "product_type": d.get("product_type"), + } + for d in connected + ], + } + + if not xcode_ok: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.WARN, + is_blocker=self.is_blocker, + summary="Xcode 27+ Required", + description=( + "Native iOS automation requires Xcode 27 or later" + + (f" (found {xcode_version})." if xcode_version else "; none detected.") + ), + metadata=metadata, + actions=[ + ProbeAction( + action_type="hint", + label="Select Xcode 27", + payload="sudo xcode-select -s /Applications/Xcode.app/Contents/Developer", + ), + ], + ) + + if not simulators and not connected: + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.WARN, + is_blocker=self.is_blocker, + summary="No Devices", + description=( + "Xcode is installed but no available iOS simulators or connected " + "physical devices were found. Create a simulator in Xcode > " + "Settings > Platforms, or attach a paired iPhone/iPad." + ), + metadata=metadata, + ) + + parts = [] + if booted: + parts.append(f"{len(booted)} simulator(s) booted") + elif simulators: + parts.append(f"{len(simulators)} simulator(s) available") + if connected: + parts.append(f"{len(connected)} physical device(s) connected") + summary = ", ".join(parts) + return ProbeResult( + id=self.probe_id, + category=self.category, + title="iOS Devices", + status=ProbeStatus.PASS, + is_blocker=self.is_blocker, + summary=summary, + description=( + f"Xcode {xcode_version}: {summary}. " + "Submit a task with platform='ios' and a device UDID to use them." + ), + metadata=metadata, + ) + + +# Backward-compatible name from when the probe only covered simulators. +IosSimulatorProbe = IosDeviceProbe diff --git a/artemis/core/diagnostics/readiness.py b/artemis/core/diagnostics/readiness.py index 8f8f0853..03c2e8fb 100644 --- a/artemis/core/diagnostics/readiness.py +++ b/artemis/core/diagnostics/readiness.py @@ -41,8 +41,9 @@ "integration_host": 2, "gemini_api_key": 3, "android_adb": 4, - "toolchain": 5, - "vision_ocr_key": 6, + "ios_simulators": 5, + "toolchain": 6, + "vision_ocr_key": 7, } @@ -70,7 +71,7 @@ def base_verdict(results: Sequence[ProbeResult]) -> Verdict: blockers = [r for r in results if r.is_blocker] if not blockers or any(r.status is not ProbeStatus.PASS for r in blockers): return "blocked" - if any(r.status is not ProbeStatus.PASS for r in results): + if any(r.status not in (ProbeStatus.PASS, ProbeStatus.SKIPPED) for r in results): return "degraded" return "ready" diff --git a/artemis/drivers/factory.py b/artemis/drivers/factory.py index d6283a1c..e997b813 100644 --- a/artemis/drivers/factory.py +++ b/artemis/drivers/factory.py @@ -31,8 +31,57 @@ logger = get_logger(__name__) +def ios_driver_class(device_id: str | None) -> type[BaseDeviceDriver]: + """Pick the driver class for an iOS serial by device reality. + + A serial resolving to a CoreDevice physical entry maps to + ``PhysicalIosDriver``; simulators, ``booted``, and unknown serials keep + ``XcodeSimulatorDriver`` so its existing validation and error paths apply. + """ + from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + if device_id and device_id.strip().lower() != BOOTED_SIMULATOR_ID: + from artemis.drivers.ios.discovery import ( + device_matches_identifier, + find_physical_ios_device_sync, + list_ios_simulators_sync, + ) + + simulators = list_ios_simulators_sync() + if simulators is None or not any( + device_matches_identifier(device, device_id) for device in simulators + ): + if find_physical_ios_device_sync(device_id) is not None: + from artemis.drivers.ios.physical_driver import PhysicalIosDriver + + return PhysicalIosDriver + return XcodeSimulatorDriver + + +def _create_ios_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: + """Instantiate the simulator or physical-device driver for an iOS target.""" + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS support is local only; cloud mode targets Android.") + config = getattr(ctx, "agent_config", None) + workspace_path = getattr(config, "ios_workspace_path", None) if config is not None else None + driver_class = ios_driver_class(ctx.device.device_id) + return driver_class( + device_id=ctx.device.device_id, + width=ctx.device.device_width, + height=ctx.device.device_height, + workspace_path=workspace_path, + ) + + def create_driver(ctx: "ArtemisContext") -> BaseDeviceDriver: """Instantiates the appropriate BaseDeviceDriver based on the runtime context.""" + mobile_platform = getattr(ctx.device, "mobile_platform", "android") + if mobile_platform == "ios": + return _create_ios_driver(ctx) + if mobile_platform not in {"android", "mock"}: + raise ValueError(f"Unsupported mobile platform: {mobile_platform}") + # 1. Cloud mode check. Cloud devices are reached through the gateway's # RemoteUIAutomatorClient; ARTEMIS_HIERARCHY_BACKEND does not apply there # because the Accessibility Helper needs a local adb forward. diff --git a/artemis/drivers/ios/__init__.py b/artemis/drivers/ios/__init__.py new file mode 100644 index 00000000..40da1ece --- /dev/null +++ b/artemis/drivers/ios/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS drivers: Xcode simulators and paired physical devices (loaded only when iOS is selected).""" diff --git a/artemis/drivers/ios/bridge.py b/artemis/drivers/ios/bridge.py new file mode 100644 index 00000000..105db8d5 --- /dev/null +++ b/artemis/drivers/ios/bridge.py @@ -0,0 +1,228 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Task-safe access to Xcode's MCP bridge using Artemis's existing MCP SDK. + +The SDK handles initialization, notifications and response IDs. A single owner +task enters and exits the SDK's AnyIO contexts, and serializes device commands. +A timeout retires the bridge; a possibly executed input is never retried. +""" + +import asyncio +import json +import os +from pathlib import Path +import sys +from typing import Any + +import anyio +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import get_default_environment, stdio_client +from mcp.shared.exceptions import McpError + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + + +_APPROVAL_MESSAGES = ( + "agent isn't approved", + "agent is not approved", + "waiting for the user to approve", +) + + +class XcodeApprovalRequiredError(RuntimeError): + """Xcode refused a tool because this interpreter lacks agent/folder approval.""" + + def __init__( + self, + tool_name: str, + native_message: str, + *, + workspace_path: str | Path | None = None, + ): + super().__init__(f"Xcode tool {tool_name} failed: {native_message}") + self.tool_name = tool_name + self.native_message = native_message + self.workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) + + @property + def guidance(self) -> str: + """Human steps to approve this interpreter; never executes them.""" + return xcode_approval_guidance(self.workspace_path) + + +def xcode_approval_guidance(workspace_path: str | Path | None = None) -> str: + """Human-facing Xcode approval steps; printing is the only action taken.""" + workspace = str(workspace_path) if workspace_path else "not supplied" + return ( + "Xcode approval is required for the Python interpreter running Artemis.\n" + f"Interpreter: {Path(sys.executable).resolve()}\n" + f"Workspace: {workspace}\n\n" + "Open the Xcode MCP menu bar icon and approve this interpreter and the selected " + "project folder. Choose Always Allow or persistent approval there if offered. " + "Alternatively, inspect the relevant pending request IDs and approve only those " + "entries from your own terminal:\n" + " xcrun mcp-server status\n" + " sudo xcrun mcp-server approve --always\n\n" + "Artemis never runs sudo or changes global access settings. After approval, " + "rerun the task. Interpreter upgrades, a different project folder, or expiring " + "grants can require approval again.\n" + "If Xcode has not recorded a request yet, rerun with " + "--platform ios --ios-workspace /absolute/path/YourApp.xcodeproj " + "(or pass workspace_path to for_ios_device())." + ) + + +_ERRORS = ( + OSError, + ValueError, + RuntimeError, + McpError, + ExceptionGroup, + anyio.EndOfStream, + anyio.ClosedResourceError, + anyio.BrokenResourceError, +) + + +class XcodeBridge: + """Own one native bridge subprocess for one simulator driver.""" + + def __init__(self, timeout_seconds: float = 120.0): + self.timeout_seconds = timeout_seconds + self._owner: asyncio.Task[None] | None = None + self._ready: asyncio.Future[set[str]] | None = None + self._requests: asyncio.Queue[Any] = asyncio.Queue() + self._active: asyncio.Future[Any] | None = None + self._start_lock = asyncio.Lock() + self.tools: set[str] = set() + + @property + def connected(self) -> bool: + return self._owner is not None and not self._owner.done() + + async def start(self) -> None: + async with self._start_lock: + if self._owner is not None and not self._owner.done(): + return + self._ready = asyncio.get_running_loop().create_future() + self._owner = asyncio.create_task(self._run(), name="artemis-xcode-bridge") + try: + self.tools = await asyncio.wait_for( + asyncio.shield(self._ready), self.timeout_seconds + ) + except (*_ERRORS, TimeoutError, asyncio.CancelledError): + await self.close() + raise + + async def _run(self) -> None: + try: + # mcpbridge only needs a clean base env plus the Xcode pinning + # vars — do not forward the full environment (API keys) to a + # long-lived child process. + env = get_default_environment() + for key in ("DEVELOPER_DIR", "MCP_XCODE_PID"): + if value := os.environ.get(key): + env[key] = value + params = StdioServerParameters(command="xcrun", args=["mcpbridge"], env=env) + async with stdio_client(params) as (reader, writer): + async with ClientSession(reader, writer) as session: + await session.initialize() + names: set[str] = set() + cursor = None + while True: + page = await session.list_tools(cursor=cursor) + names.update(tool.name for tool in page.tools) + cursor = page.nextCursor + if not cursor: + break + if self._ready is not None and not self._ready.done(): + self._ready.set_result(names) + while True: + request = await self._requests.get() + if request is None: + return + future, name, arguments = request + if future.cancelled(): + continue + self._active = future + result = await session.call_tool(name, arguments) + if not future.done(): + future.set_result(result) + self._active = None + except _ERRORS as exc: + if self._ready is not None and not self._ready.done(): + self._ready.set_exception(RuntimeError(f"Cannot connect to Xcode MCP: {exc}")) + if self._active is not None and not self._active.done(): + self._active.set_exception(RuntimeError(f"Xcode MCP connection failed: {exc}")) + finally: + error = RuntimeError( + "Xcode MCP bridge has closed. Reconnect the driver before retrying." + ) + if self._ready is not None and not self._ready.done(): + self._ready.set_exception(error) + if self._active is not None and not self._active.done(): + self._active.set_exception(error) + self._active = None + while not self._requests.empty(): + request = self._requests.get_nowait() + if request is not None and not request[0].done(): + request[0].set_exception(error) + + async def call(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + if self._owner is None or self._owner.done(): + raise RuntimeError("Xcode MCP bridge is not connected.") + future = asyncio.get_running_loop().create_future() + self._requests.put_nowait((future, name, arguments)) + try: + result = await asyncio.wait_for(future, self.timeout_seconds) + except (TimeoutError, asyncio.CancelledError): + await self.close() + raise + if result.isError: + message = " ".join(block.text for block in result.content if block.type == "text") + if any(marker in message.casefold() for marker in _APPROVAL_MESSAGES): + raise XcodeApprovalRequiredError(name, message) + raise RuntimeError(f"Xcode tool {name} failed: {message}") + if isinstance(result.structuredContent, dict): + return result.structuredContent + # Xcode also supports older clients with JSON text content. + for block in result.content: + if block.type == "text": + try: + data = json.loads(block.text) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + return data + raise RuntimeError(f"Xcode tool {name} returned no structured result.") + + async def close(self) -> None: + owner, self._owner = self._owner, None + if owner is None: + return + if not owner.done(): + owner.cancel() + try: + await owner + except asyncio.CancelledError: + pass + except Exception as exc: + # Owner failures already propagate to requesters via futures; a + # close() must never mask the error its caller is handling. + logger.debug(f"Xcode bridge owner exited with an error: {exc}") + if self._ready is not None and self._ready.done() and not self._ready.cancelled(): + self._ready.exception() # consume an initialization error on cancellation + self.tools.clear() diff --git a/artemis/drivers/ios/discovery.py b/artemis/drivers/ios/discovery.py new file mode 100644 index 00000000..1860a599 --- /dev/null +++ b/artemis/drivers/ios/discovery.py @@ -0,0 +1,439 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared iOS device enumeration: ``simctl`` for simulators, ``devicectl`` for physical. + +Both the Xcode driver (target validation) and the runtime pools (discovery, +explicit-serial validation, auto-selection) read the same +``xcrun simctl list devices --json`` output through this module so parsing +stays in one place; paired physical hardware is enumerated through +``xcrun devicectl list devices``. Every function fails closed to +``None``/``[]`` on missing tooling so Android-only hosts never see iOS errors. +""" + +import asyncio +import json +from pathlib import Path +import re +import shutil +import subprocess +import sys +import time +from typing import Any + +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +SIMCTL_QUERY_TIMEOUT = 15.0 + +# ``simctl`` wildcard that resolves to whichever simulator is currently +# booted; also the public sentinel callers pass to mean "the booted one". +BOOTED_SIMULATOR_ID = "booted" + +# ``simctl list devices`` / ``devicectl list devices`` take seconds on a busy +# host and every iOS consumer (readiness probe, device pool validation, +# /api/devices) enumerates them. Share one result briefly so polling UIs do +# not spawn back-to-back tool invocations. +_ENUMERATION_CACHE_TTL = 10.0 + + +class _TtlCache: + """Monotonic-TTL cache holding one enumeration result (or nothing).""" + + def __init__(self) -> None: + self._devices: list[dict[str, Any]] | None = None + self._time = 0.0 + + def clear(self) -> None: + self._devices = None + self._time = 0.0 + + def store(self, devices: list[dict[str, Any]]) -> list[dict[str, Any]]: + self._devices = devices + self._time = time.monotonic() + return devices + + def get(self) -> list[dict[str, Any]] | None: + if self._devices is None: + return None + if time.monotonic() - self._time > _ENUMERATION_CACHE_TTL: + return None + return self._devices + + +_simulator_cache = _TtlCache() +_core_device_cache = _TtlCache() + + +def clear_ios_simulator_cache() -> None: + """Drop the cached enumeration (e.g. after the driver boots a simulator).""" + _simulator_cache.clear() + + +async def reap_process(process: asyncio.subprocess.Process) -> None: + """Kill a still-running child and drain its pipes; never raises.""" + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.communicate() + + +async def run_xcrun(*arguments: str, timeout: float = 30.0) -> bytes: + """Run argv directly, reporting native errors and reaping cancelled children.""" + process = await asyncio.create_subprocess_exec( + "xcrun", + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except (TimeoutError, asyncio.CancelledError): + await reap_process(process) + raise + if process.returncode: + raise RuntimeError( + f"xcrun {' '.join(arguments[:3])} failed: {stderr.decode(errors='replace').strip()}" + ) + return stdout + + +async def plist_to_json(payload: bytes, timeout: float = 30.0) -> bytes: + """Convert an OpenStep/XML/binary plist payload to JSON via ``plutil``. + + ``simctl listapps`` emits OpenStep (ASCII) plists that ``plistlib`` + cannot read; ``plutil -convert json`` accepts every plist flavor. + """ + process = await asyncio.create_subprocess_exec( + "plutil", + "-convert", + "json", + "-o", + "-", + "--", + "-", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(input=payload), timeout) + except (TimeoutError, asyncio.CancelledError): + await reap_process(process) + raise + if process.returncode != 0: + raise RuntimeError(f"plutil conversion failed: {stderr.decode(errors='replace').strip()}") + return stdout + + +def simctl_available() -> bool: + """Whether this host can enumerate iOS simulators at all.""" + return sys.platform == "darwin" and shutil.which("xcrun") is not None + + +def parse_xcode_version(payload: bytes | str) -> str | None: + """Extract the Xcode version string from ``xcodebuild -version`` output.""" + text = payload.decode(errors="replace") if isinstance(payload, bytes) else payload + match = re.search(r"Xcode\s+(\d+(?:\.\d+)*)", text) + return match.group(1) if match else None + + +def parse_simctl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Flatten ``simctl list devices --json`` into available iOS entries.""" + return [ + { + "udid": device.get("udid"), + "name": device.get("name"), + "state": device.get("state"), + "runtime": runtime, + } + for runtime, entries in payload.get("devices", {}).items() + if ".iOS-" in runtime + for device in entries + if device.get("isAvailable") and device.get("udid") + ] + + +async def list_ios_simulators( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """All available iOS simulators, or ``None`` when enumeration fails. + + Successful enumerations are cached for ``_ENUMERATION_CACHE_TTL`` seconds; + failures are never cached so callers retry against live simctl. + """ + if not simctl_available(): + return None + if not force_refresh: + cached = _simulator_cache.get() + if cached is not None: + return cached + try: + raw = await run_xcrun("simctl", "list", "devices", "--json", timeout=SIMCTL_QUERY_TIMEOUT) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"simctl device enumeration failed: {exc}") + return None + try: + return _simulator_cache.store(parse_simctl_devices(json.loads(raw))) + except (ValueError, TypeError) as exc: + logger.debug(f"simctl device list parse failed: {exc}") + return None + + +def list_ios_simulators_sync( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """Synchronous variant for non-async callers (e.g. replay device lists).""" + if not simctl_available(): + return None + if not force_refresh: + cached = _simulator_cache.get() + if cached is not None: + return cached + try: + completed = subprocess.run( + ["xcrun", "simctl", "list", "devices", "--json"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=SIMCTL_QUERY_TIMEOUT, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.debug(f"simctl device enumeration failed: {exc}") + return None + if completed.returncode != 0: + return None + try: + return _simulator_cache.store(parse_simctl_devices(json.loads(completed.stdout))) + except (ValueError, TypeError) as exc: + logger.debug(f"simctl device list parse failed: {exc}") + return None + + +# --- CoreDevice (physical iPhone/iPad) enumeration ------------------------- + +DEVICECTL_ENUMERATE_TIMEOUT = 20.0 + + +async def devicectl_screenshot(device_id: str, destination: Path, timeout: float) -> None: + """Capture one PNG frame from a paired device via ``devicectl``.""" + await run_xcrun( + "devicectl", + "device", + "capture", + "screenshot", + "--device", + device_id, + "--destination", + str(destination), + timeout=timeout, + ) + + +def _device_property(device: dict[str, Any], section: str, key: str) -> Any: + """Read a property across devicectl's current and deprecated JSON shapes. + + Xcode marks ``hardwareProperties``/``deviceProperties``/``connectionProperties`` + deprecated in favor of a nested ``properties`` dictionary; accept both. + Sections may be present-but-null, so guard every hop. + """ + parent = device.get(section) + value = parent.get(key) if isinstance(parent, dict) else None + if value is not None: + return value + properties = device.get("properties") + if not isinstance(properties, dict): + return None + nested = properties.get(section) + return nested.get(key) if isinstance(nested, dict) else None + + +def _modern_property(device: dict[str, Any], section: str, key: str) -> Any: + """Read ``properties.
.`` from Xcode 27's devicectl shape. + + The modern ``properties`` map uses short section names (``hardware``, + ``software``, ``state``, ``connection``) rather than the deprecated + ``hardwareProperties``/``deviceProperties``/``connectionProperties`` + spellings. Sections may be present-but-null, so guard every hop. + """ + properties = device.get("properties") + if not isinstance(properties, dict): + return None + nested = properties.get(section) + return nested.get(key) if isinstance(nested, dict) else None + + +def _modern_os_version(device: dict[str, Any]) -> Any: + """``properties.software.osVersionNumber`` may carry a stringValue map.""" + version = _modern_property(device, "software", "osVersionNumber") + if isinstance(version, dict): + return version.get("stringValue") or version.get("string") + return version + + +def parse_devicectl_devices(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Flatten ``devicectl list devices --json-output -`` into device entries.""" + devices = [] + result = payload.get("result") + if not isinstance(result, dict): + return devices + entries = result.get("devices") + if not isinstance(entries, list): + return devices + for device in entries: + if not isinstance(device, dict): + continue + udid = ( + _device_property(device, "hardwareProperties", "udid") + or _modern_property(device, "hardware", "udid") + or device.get("identifier") + ) + if not udid: + continue + devices.append( + { + "udid": udid, + "name": _device_property(device, "deviceProperties", "name") + or _modern_property(device, "state", "name"), + "os_version": _device_property(device, "deviceProperties", "osVersionNumber") + or _modern_os_version(device), + "platform": _device_property(device, "hardwareProperties", "platform") + or _modern_property(device, "hardware", "platform"), + "reality": _device_property(device, "hardwareProperties", "reality") + or _modern_property(device, "hardware", "reality"), + "product_type": _device_property(device, "hardwareProperties", "productType") + or _modern_property(device, "hardware", "productType"), + "connection_state": _device_property(device, "connectionProperties", "tunnelState") + or _modern_property(device, "connection", "state"), + "pairing_state": _device_property(device, "connectionProperties", "pairingState") + or _modern_property(device, "connection", "pairingState"), + "visibility": device.get("visibilityClass") + or _modern_property(device, "state", "visibilityClass"), + } + ) + return devices + + +async def list_core_devices( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """All devices known to CoreDevice (physical and simulated), cached briefly.""" + if not simctl_available(): + return None + if not force_refresh: + cached = _core_device_cache.get() + if cached is not None: + return cached + try: + raw = await run_xcrun( + "devicectl", + "list", + "devices", + "--json-output", + "-", + timeout=DEVICECTL_ENUMERATE_TIMEOUT, + ) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"devicectl device enumeration failed: {exc}") + return None + try: + return _core_device_cache.store(parse_devicectl_devices(json.loads(raw))) + except (ValueError, TypeError) as exc: + logger.debug(f"devicectl device list parse failed: {exc}") + return None + + +def list_core_devices_sync( + force_refresh: bool = False, +) -> list[dict[str, Any]] | None: + """Synchronous variant for the driver factory and other sync callers.""" + if not simctl_available(): + return None + if not force_refresh: + cached = _core_device_cache.get() + if cached is not None: + return cached + try: + completed = subprocess.run( + ["xcrun", "devicectl", "list", "devices", "--json-output", "-"], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=DEVICECTL_ENUMERATE_TIMEOUT, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + logger.debug(f"devicectl device enumeration failed: {exc}") + return None + if completed.returncode != 0: + return None + try: + return _core_device_cache.store(parse_devicectl_devices(json.loads(completed.stdout))) + except (ValueError, TypeError) as exc: + logger.debug(f"devicectl device list parse failed: {exc}") + return None + + +def is_physical_ios(device: dict[str, Any]) -> bool: + """Whether a CoreDevice entry is a physical iPhone/iPad (not a simulator).""" + return device.get("platform") in ("iOS", "iPadOS") and device.get("reality") == "physical" + + +def physical_ios_ready(device: dict[str, Any]) -> bool: + """Whether a CoreDevice entry is a paired, reachable physical iOS device. + + ``connection_state`` (CoreDevice ``tunnelState``) may be absent on paired + USB devices running older iOS — an absent value is acceptable; + ``"disconnected"`` is not. Every consumer (pool validators, readiness + probe, replay manager, driver) must agree on this rule. + """ + if not is_physical_ios(device): + return False + if device.get("pairing_state") != "paired": + return False + return device.get("connection_state") in ("connected", None) + + +def device_matches_identifier(device: dict[str, Any], identifier: str) -> bool: + """Match a device entry by case-insensitive UDID or exact name.""" + if not identifier: + return False + return ( + str(device.get("udid") or "").lower() == identifier.lower() + or device.get("name") == identifier + ) + + +def find_physical_ios_device_sync(identifier: str) -> dict[str, Any] | None: + """Match a physical iOS device by UDID or exact name, or ``None``. + + Raises ``ValueError`` when the identifier matches more than one device — + silently picking the first would drive an arbitrary phone. + """ + devices = list_core_devices_sync() + if devices is None or not identifier: + return None + matches = [ + device + for device in devices + if is_physical_ios(device) and device_matches_identifier(device, identifier) + ] + if len(matches) > 1: + udids = sorted(str(device.get("udid") or "?") for device in matches) + raise ValueError( + f"{len(matches)} physical iOS devices match {identifier!r} " + f"({', '.join(udids)}); use the device UDID instead." + ) + return matches[0] if matches else None diff --git a/artemis/drivers/ios/hierarchy.py b/artemis/drivers/ios/hierarchy.py new file mode 100644 index 00000000..b2d12fdf --- /dev/null +++ b/artemis/drivers/ios/hierarchy.py @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Convert Xcode's textual accessibility frames and hit points to screen pixels.""" + +import re +from typing import Any + +_NUMBER = r"(-?\d+(?:\.\d+)?)" +_FRAME = re.compile( + r"\{\{\s*" + + _NUMBER + + r",\s*" + + _NUMBER + + r"\},\s*\{\s*" + + _NUMBER + + r",\s*" + + _NUMBER + + r"\}\}" +) +_HIT = re.compile(r"hitPoint:\s*\{\s*" + _NUMBER + r",\s*" + _NUMBER + r"\}") +_BUNDLE = re.compile(r"Application bundle identifier:\s*(\S+)") + + +def application_bundle(hierarchy: str) -> str | None: + bundles = set(_BUNDLE.findall(hierarchy)) + return next(iter(bundles)) if len(bundles) == 1 else None + + +def pixel_element( + *, text: str, resource_id: str, class_name: str, left: int, top: int, right: int, bottom: int +) -> dict[str, Any]: + """Element dict in the shared Android-shape format used by both iOS parsers.""" + return { + "text": text, + "resource_id": resource_id, + "class": class_name, + "bounds": f"[{left},{top}][{right},{bottom}]", + "parsed_bounds": {"left": left, "top": top, "right": right, "bottom": bottom}, + } + + +def parse_hierarchy( + hierarchy: str, width: int, height: int +) -> tuple[list[dict[str, Any]], tuple[float, float]]: + """Determine scale from the screen window; never assume a Retina factor. + + UIKit hierarchy geometry is in the native interaction coordinate space. + Artemis observations/actions use full screenshot pixels. Missing or rotated + geometry is an error rather than silently tapping with an unverified scale. + """ + lines = hierarchy.splitlines() + windows = [] + for line in lines: + match = _FRAME.search(line) + if match and re.match(r"\s*(?:UI)?Window\b", line): + x, y, w, h = map(float, match.groups()) + if x == 0 and y == 0 and w > 0 and h > 0: + windows.append((w, h)) + matching = [(w, h) for w, h in windows if abs(width / w - height / h) < 0.05] + if windows and not matching: + raise RuntimeError( + "Xcode hierarchy has no screen window matching the screenshot. Recapture before interacting." + ) + # Xcode 27 exports the full screenshot at logical screen dimensions. An + # inaccessible custom canvas can have no AX Window; visual targeting still + # works in this documented native coordinate space. + w, h = max(matching, key=lambda size: size[0] * size[1]) if matching else (width, height) + scale = width / w, height / h + elements = [] + for line in lines: + match = _FRAME.search(line) + if not match: + continue + x, y, w, h = map(float, match.groups()) + if w <= 0 or h <= 0: + continue + left, top = round(x * scale[0]), round(y * scale[1]) + right, bottom = round((x + w) * scale[0]), round((y + h) * scale[1]) + if right <= 0 or bottom <= 0 or left >= width or top >= height: + continue + label = re.search(r"label:\s*'((?:\\.|[^'])*)'", line) + quoted = re.search(r'"([^"\n]*)"', line[: match.start()]) + identifier = re.search(r"identifier:\s*'((?:\\.|[^'])*)'", line) + placeholder = re.search(r"placeholderValue:\s*'((?:\\.|[^'])*)'", line) + value = re.search(r"value:\s*'((?:\\.|[^'])*)'", line) + if value: + value_text = value.group(1) + else: + # Xcode elides long values and drops quotes, e.g. `value: Text...`, + # so read until the next native metadata delimiter verbatim. + unquoted = re.search( + r"\bvalue:\s*(.*?)(?=,\s*(?:Keyboard Focused\b|Selected\b|Disabled\b" + r"|hitPoint:|activationBundleId:|identifier:|label:|placeholderValue:)|$)", + line, + ) + value_text = unquoted.group(1).strip() if unquoted else None + hit = _HIT.search(line) + activation = re.search(r"activationBundleId:\s*(\S+)", line) + text = "" + if label: + text = label.group(1) + elif quoted: + text = quoted.group(1) + elif value_text: + text = value_text + elif placeholder and placeholder.group(1): + text = placeholder.group(1) + element = pixel_element( + text=text, + resource_id=identifier.group(1) if identifier else "", + class_name=line.strip().split(",", 1)[0].split(" ", 1)[0], + left=left, + top=top, + right=right, + bottom=bottom, + ) + if hit: + hx, hy = map(float, hit.groups()) + element["hit_point"] = [round(hx * scale[0]), round(hy * scale[1])] + if activation: + element["activation_bundle_id"] = activation.group(1) + if placeholder: + element["placeholder"] = placeholder.group(1) + if value_text is not None: + element["value"] = value_text + elements.append(element) + return elements, scale diff --git a/artemis/drivers/ios/physical_driver.py b/artemis/drivers/ios/physical_driver.py new file mode 100644 index 00000000..8ab5d428 --- /dev/null +++ b/artemis/drivers/ios/physical_driver.py @@ -0,0 +1,790 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iPhone/iPad support via ``devicectl`` plus WebDriverAgent. + +Lifecycle operations (install, launch, terminate, app list, URL open, +screenshots) run through ``xcrun devicectl`` against a paired device. Xcode's +``DeviceInteraction*`` MCP tools accept simulators only, so UI observation and +input on hardware go through WebDriverAgent (WDA), the XCUITest bridge Appium +uses: the driver finds a signed WDA runner on the device, launches it through +``devicectl``, and talks to its HTTP endpoint over the CoreDevice tunnel, a +LAN address, or a forwarded port. + +Prerequisites surface as actionable errors: the device must appear in +``devicectl list devices`` as ``paired`` and ``connected`` (USB or network), +Developer Mode must be on, and a WebDriverAgent runner must be installed — +set ``ARTEMIS_IOS_WDA_URL`` to reach an existing server directly. +""" + +import asyncio +import base64 +from io import BytesIO +import json +import os +from pathlib import Path +import plistlib +import re +import tempfile +from typing import Any, Literal + +from PIL import Image + +from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection +from artemis.drivers.ios.discovery import ( + BOOTED_SIMULATOR_ID, + device_matches_identifier, + devicectl_screenshot, + is_physical_ios, + list_core_devices, + reap_process, + run_xcrun, +) +from artemis.drivers.ios.physical_recording import PhysicalIosRecorder +from artemis.drivers.ios.wda import ( + WdaClient, + WdaUnavailableError, + parse_wda_elements, + probe_wda, + wda_url_candidates, +) +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from third_party.mobile_use.controllers.types import ElementQuery +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session, remove_active_session + +logger = get_logger(__name__) + +DEVICECTL_OP_TIMEOUT = 30.0 +DEVICECTL_LAUNCH_TIMEOUT = 60.0 +DEVICECTL_INSTALL_TIMEOUT = 300.0 +WDA_START_TIMEOUT = 45.0 +WDA_RUNNER_PATTERN = re.compile(r"webdriveragent", re.IGNORECASE) +WDA_BUNDLE_ENV = "ARTEMIS_IOS_WDA_BUNDLE_ID" +WDA_XCTESTRUN_ENV = "ARTEMIS_IOS_WDA_XCTESTRUN" + +_WDA_SETUP_HINT = ( + "Physical iOS UI automation needs WebDriverAgent on the device. Build it once with " + "'xcodebuild build-for-testing -project /WebDriverAgent.xcodeproj " + "-scheme WebDriverAgentRunner -destination id= -allowProvisioningUpdates " + "DEVELOPMENT_TEAM=', install the produced WebDriverAgentRunner-Runner.app via " + "'xcrun devicectl device install app', point ARTEMIS_IOS_WDA_XCTESTRUN at the " + "generated .xctestrun for Artemis to host it, or expose a running server through " + "ARTEMIS_IOS_WDA_URL (iproxy, pymobiledevice3, or the device LAN address)." +) + + +class PhysicalIosDriver(XcodeSimulatorDriver): + """One paired physical iOS device driven by devicectl and WebDriverAgent. + + Shares lifecycle conventions with ``XcodeSimulatorDriver`` but replaces the + interaction core entirely: CoreDevice handles discovery and app lifecycle, + WDA supplies hierarchy, screenshots, taps, swipes, text, and keys. The + simulator-only Xcode MCP bridge is never started for a physical target. + """ + + def __init__( + self, + device_id: str, + width: int = 0, + height: int = 0, + *, + workspace_path: str | Path | None = None, + ): + super().__init__( + device_id=device_id, width=width, height=height, workspace_path=workspace_path + ) + self._launched_pids: dict[str, int] = {} + self._wda: WdaClient | None = None + self._wda_runner_pid: int | None = None + self._wda_test_process: asyncio.subprocess.Process | None = None + + # --- Resolution and connection --- + + async def _resolve_device(self) -> dict[str, Any]: + self._validate_workspace() + if not self._device_id.strip() or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: + raise ValueError( + "Physical iOS devices require --device-serial ; " + "'booted' only selects simulators. Find UDIDs via 'xcrun devicectl list devices'." + ) + await self._require_ios_host() + devices = await list_core_devices(force_refresh=True) + if devices is None: + raise RuntimeError( + "Could not enumerate physical devices; 'xcrun devicectl list devices' failed." + ) + matches = [ + device + for device in devices + if is_physical_ios(device) and device_matches_identifier(device, self._device_id) + ] + if not matches: + needle = self._device_id.lower() + simulator = [ + device + for device in devices + if device.get("udid", "").lower() == needle and device.get("reality") == "simulated" + ] + if simulator: + raise ValueError( + f"UDID {self._device_id} is an iOS Simulator, not a physical device." + ) + raise ValueError( + f"No paired physical iOS device matches {self._device_id!r}. " + "Attach it, trust this Mac, and verify 'xcrun devicectl list devices'." + ) + if len(matches) > 1: + udids = sorted(device.get("udid", "?") for device in matches) + raise ValueError( + f"{len(matches)} physical iOS devices match {self._device_id!r} " + f"({', '.join(udids)}); target the device UDID instead." + ) + candidate = matches[0] + self._device_id = candidate["udid"] + return candidate + + async def _prepare_device(self, candidate: dict[str, Any]) -> None: + """Verify the paired device is reachable instead of booting it.""" + if candidate.get("pairing_state") != "paired": + raise RuntimeError( + f"iOS device {self._device_id} is not paired. Connect it and tap Trust." + ) + connection_state = candidate.get("connection_state") + # Older iOS versions over USB expose no CoreDevice tunnelState — an + # absent value on a paired device is acceptable; "disconnected" is not. + if connection_state is not None and connection_state != "connected": + raise RuntimeError( + f"iOS device {self._device_id} ({candidate.get('name') or 'unknown'}) is not " + "connected. Attach it over USB or ensure network pairing is reachable; " + "on iOS 16+ also enable Developer Mode in Settings > Privacy & Security." + ) + + async def connect(self) -> None: + async with self._connect_lock: + if self._session_key: + return + candidate = await self._resolve_device() + await self._prepare_device(candidate) + connected = False + try: + self._wda = await self._ensure_wda() + # The reachable WDA endpoint must belong to THIS device before + # we open a session or send input. WDA reports the product + # family name ("iPhone"), not the personalized devicectl name + # ("Dana's iPhone"), and uuid is identifierForVendor — so a + # *specific* conflicting name is the wrong-device signal; + # generic family names are accepted. + info = await self._wda.device_info() + expected_name = (candidate.get("name") or "").strip() + wda_name = str(info.get("name") or "").strip() + generic_names = { + "iphone", + "ipad", + "ipod touch", + "apple watch", + "apple tv", + } + name_conflict = ( + expected_name + and wda_name + and wda_name.lower() not in generic_names + and wda_name != expected_name + ) + if info.get("isSimulator") is not False or name_conflict: + raise RuntimeError( + f"The WebDriverAgent at {self._wda.base_url} does not report " + f"the selected physical device {self._device_id} " + f"(expected name {expected_name!r}, got " + f"{info.get('name')!r}, isSimulator={info.get('isSimulator')!r}). " + "Point ARTEMIS_IOS_WDA_URL at a WDA server running on the " + "selected device." + ) + # A session auto-created by a runner WE launched is ours to + # adopt; an endpoint discovered via env/probe keeps the + # foreign-session refusal. + runner_owned = ( + self._wda_runner_pid is not None or self._wda_test_process is not None + ) + self._session_key = await self._wda.open_session(adopt_existing=runner_owned) + await self.get_screen_data(skip_settling=True) + connected = True + finally: + # Any incomplete setup must release a half-started WDA + # runner, xcodebuild session, or server-side session. + if not connected: + try: + await self.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as cleanup_error: + logger.warning( + "Could not release the WDA session after a connection failure: " + f"{cleanup_error}", + ) + + async def disconnect(self) -> None: + try: + if self._recorder is not None: + session = self._recorder.session + if session is not None: + # A failed (non-active) session still owns frames and + # conversions — finalize it too so nothing is lost. + try: + await self._recorder.stop() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.error( + f"Physical iOS recording finalization failed during disconnect: {exc}" + ) + if get_active_session(self._device_id) is session: + remove_active_session(self._device_id) + finally: + async with self._operation_lock: + self._session_key = None + self._scale = None + self._launched_pids.clear() + client, self._wda = self._wda, None + runner_pid, self._wda_runner_pid = self._wda_runner_pid, None + test_process, self._wda_test_process = self._wda_test_process, None + # Each cleanup step is isolated so a failure in one never skips + # the owned WDA session, the xcodebuild child, or the runner pid. + if client is not None: + try: + await client.close_session() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug(f"WDA session close failed during disconnect: {exc}") + if test_process is not None and test_process.returncode is None: + try: + test_process.terminate() + await asyncio.wait_for(test_process.wait(), timeout=10.0) + except (TimeoutError, OSError): + # Graceful terminate failed — reap (kill + drain) the + # xcodebuild child so it cannot outlive the driver. + try: + await reap_process(test_process) + except (OSError, RuntimeError, TimeoutError) as exc: + logger.debug(f"WDA xcodebuild reap failed: {exc}") + if runner_pid is not None: + try: + await self._terminate_pid(runner_pid) + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + logger.debug(f"WDA runner termination failed: {exc}") + + def _require_connected(self) -> None: + if not self._session_key or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: + raise RuntimeError("Connect the physical iOS driver before interacting.") + + def _require_wda(self) -> WdaClient: + """Bound WDA client for input paths; fails clearly when detached.""" + self._require_connected() + client = self._wda + if client is None: + raise RuntimeError("Connect the physical iOS driver before interacting.") + return client + + async def _ensure_wda(self) -> WdaClient: + """Attach to a reachable WDA server, starting one when possible. + + Order: probe known endpoints, then try launching an installed runner + app directly (newer WDA builds self-host), then — when + ``ARTEMIS_IOS_WDA_XCTESTRUN`` names a ``.xctestrun`` file from a + ``build-for-testing`` — spawn ``xcodebuild test-without-building``, + which is the canonical way to boot the XCTest session that hosts + WDA's HTTP server. + """ + candidates = wda_url_candidates(tunnel_ip=await self._tunnel_ip()) + client = await probe_wda(candidates) + if client is not None: + return client + xctestrun = os.environ.get(WDA_XCTESTRUN_ENV) + runner = await self._wda_runner_bundle() + if xctestrun: + await self._start_xctest_session(xctestrun) + elif runner is not None: + try: + self._wda_runner_pid = await self._launch_bundle(runner, terminate_existing=False) + logger.info( + f"Launched WebDriverAgent runner {runner} on {self._device_id}; " + "waiting for its HTTP server" + ) + except (OSError, RuntimeError, TimeoutError) as exc: + # A stale runner may already be hosting the server — the + # probe loop below still gets a chance to attach. + logger.debug(f"WDA runner launch failed ({runner}): {exc}") + if runner is None and not xctestrun: + raise RuntimeError( + f"No WebDriverAgent server answers on {self._device_id} and no WDA " + f"runner is installed. {_WDA_SETUP_HINT}" + ) + deadline = asyncio.get_running_loop().time() + WDA_START_TIMEOUT + while True: + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + raise RuntimeError( + f"WebDriverAgent did not answer within {WDA_START_TIMEOUT:.0f}s " + f"on {self._device_id} at {candidates}. If the device is showing " + "a passcode prompt to enable UI Automation, enter it on the " + f"device first. {_WDA_SETUP_HINT}" + ) + client = await probe_wda(candidates, timeout=min(5.0, remaining)) + if client is not None: + logger.info(f"WebDriverAgent attached at {client.base_url} for {self._device_id}") + return client + await asyncio.sleep(min(1.0, remaining)) + + async def _start_xctest_session(self, xctestrun: str) -> None: + """Hold WDA alive through ``xcodebuild test-without-building``.""" + path = Path(xctestrun).expanduser() + if path.suffix != ".xctestrun" or not path.is_file(): + raise RuntimeError( + f"{WDA_XCTESTRUN_ENV} must point at an existing .xctestrun file " + "produced by 'xcodebuild build-for-testing'." + ) + if self._wda_test_process is not None and self._wda_test_process.returncode is None: + return + self._wda_test_process = await asyncio.create_subprocess_exec( + "xcodebuild", + "test-without-building", + "-xctestrun", + str(path), + "-destination", + f"id={self._device_id}", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + + async def _tunnel_ip(self) -> str | None: + """CoreDevice's managed tunnel address for this device, when present.""" + try: + details = await self._devicectl_json("info", "details") + except (OSError, RuntimeError, TimeoutError, ValueError): + return None + # The tunnel address has moved across Xcode releases: probe every + # observed shape, canonical first. + paths = ( + ((details.get("connectionProperties") or {}), "tunnelIPAddress"), + ( + ((details.get("properties") or {}).get("connection") or {}), + "tunnelIPAddressString", + ), + ((details.get("tunnel") or {}), "ipAddress"), + ) + for section, key in paths: + address = section.get(key) + if isinstance(address, str) and ":" in address: + return address + return None + + async def _wda_runner_bundle(self) -> str | None: + override = os.environ.get(WDA_BUNDLE_ENV) + try: + apps = await self._devicectl_json("info", "apps") + except (OSError, RuntimeError, TimeoutError, ValueError) as exc: + logger.debug(f"Could not list apps while looking for a WDA runner: {exc}") + return override or None + bundles = { + app.get("bundleIdentifier") or "" + for app in apps.get("apps", []) + if isinstance(app, dict) + } + if override: + return override if override in bundles else None + matches = sorted(b for b in bundles if WDA_RUNNER_PATTERN.search(b)) + return matches[0] if matches else None + + async def _launch_bundle(self, bundle: str, terminate_existing: bool = True) -> int | None: + arguments = [ + "devicectl", + "device", + "process", + "launch", + "--device", + self._device_id, + ] + if terminate_existing: + arguments.append("--terminate-existing") + arguments += ["--json-output", "-", bundle] + raw = await run_xcrun(*arguments, timeout=DEVICECTL_LAUNCH_TIMEOUT) + return self._launched_pid(raw) + + async def _devicectl_json(self, *arguments: str) -> dict[str, Any]: + """Run a ``devicectl`` info subcommand and return its ``result`` JSON. + + ``--json-output -`` still writes a human table to stdout for ``info`` + subcommands, so the JSON must go to a scratch file. + """ + with tempfile.TemporaryDirectory(prefix="artemis-devicectl-") as tmp: + target = Path(tmp) / "out.json" + await run_xcrun( + "devicectl", + "device", + *arguments, + "--device", + self._device_id, + "--json-output", + str(target), + timeout=DEVICECTL_OP_TIMEOUT, + ) + payload = json.loads(target.read_text(encoding="utf-8")) + result = payload.get("result") + return result if isinstance(result, dict) else payload + + # --- Observation --- + + async def _screenshot_png(self) -> bytes: + """WDA screenshot first (same framebuffer as the hierarchy); devicectl fallback.""" + if self._wda is not None: + try: + return await self._wda.screenshot_png() + except (RuntimeError, WdaUnavailableError, OSError) as exc: + logger.debug(f"WDA screenshot failed, falling back to devicectl: {exc}") + with tempfile.TemporaryDirectory(prefix="artemis-shot-") as tmp: + target = Path(tmp) / "shot.png" + await devicectl_screenshot(self._device_id, target, timeout=DEVICECTL_OP_TIMEOUT) + data = target.read_bytes() + if not data: + raise RuntimeError("devicectl produced an empty screenshot.") + return data + + async def _capture(self) -> ScreenData: + if self._wda is None: + raise RuntimeError("Connect the physical iOS driver before observing.") + screenshot = await self._screenshot_png() + with Image.open(BytesIO(screenshot)) as image: + self._width, self._height = image.size + win_w, win_h = await self._wda.window_size() + if win_w <= 0 or win_h <= 0: + raise RuntimeError("WebDriverAgent reported an unusable window size.") + if abs(self._width / win_w - self._height / win_h) > 0.05: + self._scale = None + raise RuntimeError( + "The device screenshot and hierarchy disagree on orientation. " + "Capture again after the rotation settles." + ) + scale = (self._width / win_w, self._height / win_h) + if not (0.9 <= scale[0] <= 4.5 and 0.9 <= scale[1] <= 4.5): + # iOS displays render at 1x-3x; a wildly off scale means the WDA + # window is not full-screen (e.g. iPad multitasking) and every + # element bound would be wrong. + self._scale = None + raise RuntimeError( + f"WebDriverAgent window {win_w}x{win_h} does not match the " + f"{self._width}x{self._height} framebuffer (scale {scale}); " + "bring the session app full-screen before interacting." + ) + self._scale = scale + tree = await self._wda.source_json() + elements = parse_wda_elements(tree, self._scale, self._width, self._height) + return ScreenData( + screenshot_bytes=screenshot, + screenshot_base64=base64.b64encode(screenshot).decode("ascii"), + ui_elements=elements, + width=self._width, + height=self._height, + platform="ios", + ) + + # --- Input --- + + async def tap( + self, x: int, y: int, duration_ms: int = 100, times: int = 1, delay_ms: int = 100 + ) -> bool: + if times < 1 or duration_ms < 0 or delay_ms < 0: + raise ValueError("Tap count must be positive and durations nonnegative.") + async with self._operation_lock: + wda = self._require_wda() + await self._capture_unchanged("tapping") + point = self._scaled_point(x, y) + for index in range(times): + await wda.tap(*point, hold_ms=duration_ms) + if index < times - 1: + await asyncio.sleep(delay_ms / 1000) + return True + + async def swipe( + self, start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 800 + ) -> bool: + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + wda = self._require_wda() + await self._capture_unchanged("swiping") + start = self._scaled_point(start_x, start_y) + end = self._scaled_point(end_x, end_y) + await wda.swipe(*start, *end, duration_ms) + return True + + async def swipe_direction( + self, + direction: SwipeDirection | Literal["up", "down", "left", "right"], + duration_ms: int = 800, + ) -> bool: + direction = SwipeDirection(direction).value + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + # Rotation-tolerant like the simulator path: recapture and compute + # from the *current* size rather than refusing like tap. + wda = self._require_wda() + await self._capture() + sx, sy, ex, ey = self._direction_points(direction, *self.screen_size) + start = self._scaled_point(sx, sy) + end = self._scaled_point(ex, ey) + await wda.swipe(*start, *end, duration_ms) + return True + + async def input_text(self, text: str, clear_existing: bool = True) -> bool: + if clear_existing: + raise NotImplementedError( + "Physical iOS typing appends to the focused field; clear it through the " + "UI or pass clear_existing=False. This matches the simulator behavior." + ) + async with self._operation_lock: + wda = self._require_wda() + await wda.type_text(text) + return True + + async def press_key(self, key: KeyCode | str | int) -> bool: + key = key.value if isinstance(key, KeyCode) else str(key).lower() + async with self._operation_lock: + wda = self._require_wda() + if key in ("home", "app_switch"): + presses = 2 if key == "app_switch" else 1 + for index in range(presses): + if not await wda.press_button("home"): + if key == "app_switch": + # A backend that cannot press Home cannot + # double-press for app switching — never report + # success from a homescreen fallback. + raise NotImplementedError( + "App switching is unavailable from this WebDriverAgent backend." + ) + await wda.homescreen() + if index < presses - 1: + await asyncio.sleep(0.4) + elif key == "enter": + await wda.type_text("\n") + else: + buttons = { + "volume_up": "volumeUp", + "volume_down": "volumeDown", + "power": "power", + } + if key not in buttons or not await wda.press_button(buttons[key]): + raise NotImplementedError( + f"Key {key!r} is not supported by the physical iOS driver." + ) + return True + + async def tap_element( + self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 + ) -> bool: + async with self._operation_lock: + wda = self._require_wda() + data = await self._capture() + element, center, error = await self.find_element(query, data) + if error or element is None or center is None: + return False + point = self._scaled_point(*center) + await wda.tap(*point, hold_ms=duration_ms if long_press else 0) + return True + + # --- App lifecycle via devicectl --- + + async def launch_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + pid = await self._launch_bundle(package_name) + if pid is not None: + self._launched_pids[package_name] = pid + return True + + @staticmethod + def _launched_pid(raw: bytes) -> int | None: + """Best-effort process id from devicectl's launch JSON or text output.""" + try: + payload = json.loads(raw) + except (ValueError, TypeError): + payload = None + if isinstance(payload, dict): + result = payload.get("result") + result = result if isinstance(result, dict) else {} + process = result.get("process") + candidates = [c for c in (process, result, payload) if isinstance(c, dict)] + for candidate in candidates: + pid = candidate.get("processIdentifier") or candidate.get("pid") + if isinstance(pid, int): + return pid + if isinstance(pid, str) and pid.isdigit(): + return int(pid) + match = re.search(r"pid[:= ]+(\d+)", raw.decode(errors="replace")) + return int(match.group(1)) if match else None + + @staticmethod + def _normalize_executable(raw: Any) -> str: + """devicectl reports executables as file:// URLs or paths; normalize.""" + value = raw if isinstance(raw, str) else str(raw or "") + if isinstance(raw, dict): + value = str(raw.get("url") or raw.get("path") or "") + value = value.removeprefix("file://") + return value.removeprefix("/private") or value + + async def _resolve_pid(self, package_name: str) -> int: + """Verified live pid: the executable must sit under the app's own URL. + + A cached launch pid only prioritizes among processes already proven to + belong to the app — a recycled pid must never select a foreign process. + """ + apps = await self._devicectl_json("info", "apps") + url_prefix = "" + for app in apps.get("apps", []): + if isinstance(app, dict) and app.get("bundleIdentifier") == package_name: + url_prefix = self._normalize_executable(app.get("url")).rstrip("/") + break + if not url_prefix: + raise ValueError( + f"{package_name!r} is not installed on {self._device_id} or its app URL " + "could not be determined; cannot verify a process to terminate." + ) + processes = await self._devicectl_json("info", "processes") + candidates: list[int] = [] + for process in processes.get("runningProcesses", []): + if not isinstance(process, dict): + continue + executable = self._normalize_executable(process.get("executable")) + if executable != url_prefix and not executable.startswith(url_prefix + "/"): + continue + pid = process.get("processIdentifier") or process.get("pid") or process.get("processID") + if isinstance(pid, int): + candidates.append(pid) + if not candidates: + raise ValueError( + f"No running process found for {package_name!r} on {self._device_id}; " + "launch it with launch_app before stopping." + ) + tracked = self._launched_pids.get(package_name) + if tracked is not None and tracked in candidates: + return tracked + return candidates[0] + + async def _terminate_pid(self, pid: int, kill: bool = False) -> None: + arguments = [ + "devicectl", + "device", + "process", + "terminate", + "--device", + self._device_id, + "--pid", + str(pid), + ] + if kill: + arguments.append("--kill") + await run_xcrun(*arguments, timeout=DEVICECTL_OP_TIMEOUT) + + async def stop_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + # Verify the live pid first — a recycled cached pid could belong to + # a different app now, and terminating it would kill the wrong app. + pid = await self._resolve_pid(package_name) + await self._terminate_pid(pid, kill=True) + self._launched_pids.pop(package_name, None) + return True + + async def install_app(self, app_path: Path) -> str: + self._require_connected() + path = app_path.expanduser().resolve() + if path.suffix.lower() == ".ipa": + if not path.is_file(): + raise ValueError("iOS installation requires an existing .ipa file.") + bundle = self._ipa_bundle_id(path) or path.stem + elif path.suffix == ".app" and path.is_dir(): + with (path / "Info.plist").open("rb") as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + if not isinstance(bundle, str) or not bundle: + raise ValueError("The .app has no CFBundleIdentifier in Info.plist.") + else: + raise ValueError( + "Physical iOS installation requires a signed .app directory or .ipa built " + "for a device (arm64) with a valid provisioning profile." + ) + async with self._operation_lock: + self._require_connected() + await run_xcrun( + "devicectl", + "device", + "install", + "app", + "--device", + self._device_id, + str(path), + timeout=DEVICECTL_INSTALL_TIMEOUT, + ) + return bundle + + @staticmethod + def _ipa_bundle_id(path: Path) -> str | None: + """Read CFBundleIdentifier from an IPA's embedded app Info.plist.""" + import zipfile + + try: + with zipfile.ZipFile(path) as archive: + for name in archive.namelist(): + if name.startswith("Payload/") and name.endswith(".app/Info.plist"): + with archive.open(name) as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + except (OSError, zipfile.BadZipFile, KeyError, plistlib.InvalidFileException): + return None + return None + + async def list_apps(self) -> dict[str, str]: + async with self._operation_lock: + self._require_connected() + apps = await self._devicectl_json("info", "apps") + result = {} + for app in apps.get("apps", []): + bundle = app.get("bundleIdentifier") or app.get("bundleID") + if not bundle: + continue + result[bundle] = app.get("name") or bundle + return result + + async def open_url(self, url: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun( + "devicectl", + "device", + "process", + "openURL", + "--device", + self._device_id, + url, + timeout=DEVICECTL_OP_TIMEOUT, + ) + return True + + async def get_current_package(self) -> str | None: + async with self._operation_lock: + self._require_connected() + if self._wda is not None: + return await self._wda.active_app() + return None + + async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: + raise NotImplementedError("Android shell commands are unavailable on iOS devices.") + + # --- Recording (devicectl screenshot polling) --- + + def _new_recorder(self) -> PhysicalIosRecorder: + """Physical recorder: devicectl screenshot polling + ffconcat encode.""" + return PhysicalIosRecorder(self._device_id) diff --git a/artemis/drivers/ios/physical_recording.py b/artemis/drivers/ios/physical_recording.py new file mode 100644 index 00000000..06cc9f51 --- /dev/null +++ b/artemis/drivers/ios/physical_recording.py @@ -0,0 +1,493 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iOS device recording via polled ``devicectl`` screenshots. + +``simctl io recordVideo`` is simulator-only, so physical capture polls +``xcrun devicectl device capture screenshot`` into timestamped PNG frames and +assembles each contiguous run of equal-sized frames into a CFR MP4 with the +ffconcat demuxer, preserving real per-frame timing. Segments roll on rotation +(dimension change), max duration, or capture failures. Physical capture has no +audio and a lower frame rate than simulator recording (~1-3 fps bounded by +devicectl round-trip time). +""" + +import asyncio +import contextlib +from pathlib import Path +import tempfile +import time +from typing import Any +from uuid import uuid4 + +from PIL import Image + +from artemis.config.paths import get_temp_dir +from artemis.drivers.ios.discovery import devicectl_screenshot, reap_process +from artemis.drivers.ios.recording import ( + DEFAULT_MAX_DURATION_SECONDS, + SEGMENT_PROBE_TIMEOUT_SECONDS, + IosRecordingSession, +) +from artemis.utils.video import get_ffmpeg_path, probe_video_segment, write_recording_manifest +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +CAPTURE_TIMEOUT_SECONDS = 15.0 +WATCHDOG_INTERVAL_SECONDS = 0.5 +# devicectl frame polls flake more often than the managed simctl process, so +# the failure cap and the ffmpeg budget are looser than in recording.py. +MAX_CONSECUTIVE_FAILURES = 5 + + +FFMPEG_TIMEOUT_SECONDS = 300.0 + + +async def _run_ffmpeg(arguments: list[str]) -> tuple[int, bytes]: + """Seam for segment assembly so tests can intercept the ffmpeg process.""" + process = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + *arguments, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + _stdout, stderr = await asyncio.wait_for( + process.communicate(), timeout=FFMPEG_TIMEOUT_SECONDS + ) + except TimeoutError: + await reap_process(process) + return -1, b"ffmpeg timed out" + except asyncio.CancelledError: + await reap_process(process) + # Drop only our own in-flight .part output before propagating. + part = Path(str(arguments[-1])) + if part.suffix == ".mp4" and ".part" in part.name: + part.unlink(missing_ok=True) + raise + rc = process.returncode + if rc is None: + raise RuntimeError("ffmpeg completed without an exit code") + return rc, stderr + + +class IosPhysicalRecordingSession(IosRecordingSession): + """Recording session for polled physical-device frame capture.""" + + poll_task: asyncio.Task | None = None + frames_dir: Path | None = None + frames: list[dict[str, Any]] = [] + frame_index: int = 0 + + +class PhysicalIosRecorder: + """Owns one screenshot-poll recording lifecycle for one physical UDID.""" + + def __init__(self, device_id: str): + self._device_id = device_id + self._session: IosPhysicalRecordingSession | None = None + self._output_dir: Path | None = None + self._lock = asyncio.Lock() + self._max_duration_seconds = DEFAULT_MAX_DURATION_SECONDS + self._consecutive_failures = 0 + + @property + def session(self) -> IosPhysicalRecordingSession | None: + """The latest session, kept after stop/failure for error reporting.""" + return self._session + + def _segment_output_path(self, session: IosPhysicalRecordingSession, index: int) -> Path: + assert self._output_dir is not None + return self._output_dir / f"segment_{index:04d}.mp4" + + async def _capture_frame(self, session: IosPhysicalRecordingSession) -> None: + """Append one timestamped screenshot frame to the open segment.""" + assert session.frames_dir is not None + session.frame_index += 1 + path = session.frames_dir / f"frame_{session.frame_index:06d}.png" + await devicectl_screenshot(self._device_id, path, timeout=CAPTURE_TIMEOUT_SECONDS) + captured_at = time.monotonic() + if not path.exists() or path.stat().st_size == 0: + raise RuntimeError("devicectl produced no screenshot frame") + with Image.open(path) as image: + width, height = image.size + if session.capture_width is None: + session.capture_width, session.capture_height = width, height + session.segment_started_monotonic = captured_at + session.segment_started_at = time.time() + if session.anchor_monotonic is None: + session.anchor_monotonic = captured_at + session.start_time = session.segment_started_at + session.frames.append({"path": path, "at": captured_at}) + if (width, height) != (session.capture_width, session.capture_height): + await self._roll(session, captured_at, "rotation") + + async def _poll_loop(self, session: IosPhysicalRecordingSession) -> None: + """Capture frames back-to-back; devicectl's own latency paces the loop.""" + try: + while session.is_active: + failed = False + async with self._lock: + if not session.is_active: + return + try: + await self._capture_frame(session) + self._consecutive_failures = 0 + except asyncio.CancelledError: + raise + except Exception as exc: + # Count any unexpected failure (PIL errors, devicectl + # surprises) toward the limit so the loop can never + # die silently with the session still active. + failed = True + self._consecutive_failures += 1 + session.errors.append(f"Frame capture failed: {exc}") + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + session.errors.append( + "Frame capture failure limit reached; session stopped" + ) + session.is_active = False + return + # Yield every iteration: callers and the watchdog share this loop. + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS if failed else 0) + except asyncio.CancelledError: + return + + async def _start_segment(self, session: IosPhysicalRecordingSession) -> None: + assert self._output_dir is not None + session.frames = [] + # frame_index stays monotonic for the whole session: rolled leftover + # frames keep their original names in the new segment directory, so a + # fresh index range can never collide with them. + session.capture_width = session.capture_height = None + session.segment_started_monotonic = None + session.frames_dir = self._output_dir / f"frames_{session.segment_index:04d}" + session.frames_dir.mkdir(parents=True, exist_ok=True) + + async def _poll_first_frame(self, session: IosPhysicalRecordingSession) -> None: + """Block until the first frame anchors the recording timeline.""" + deadline = time.monotonic() + CAPTURE_TIMEOUT_SECONDS * 2 + while session.anchor_monotonic is None: + if time.monotonic() > deadline: + raise RuntimeError("Timed out waiting for the first devicectl screenshot frame.") + if not session.is_active: + raise RuntimeError( + session.errors[-1] + if session.errors + else "Frame capture stopped before the first frame." + ) + await asyncio.sleep(0.05) + + def _seal_current_segment( + self, session: IosPhysicalRecordingSession, end_monotonic: float + ) -> dict[str, Any] | None: + frames = [f for f in session.frames if f["at"] < end_monotonic] + session.frames = [f for f in session.frames if f["at"] >= end_monotonic] + if not frames: + return None + record = { + "frames": frames, + "output_path": self._segment_output_path(session, session.segment_index), + "start": max(0.0, frames[0]["at"] - (session.anchor_monotonic or 0.0)), + "end": max(0.0, end_monotonic - (session.anchor_monotonic or 0.0)), + "width": session.capture_width, + "height": session.capture_height, + "generation": session.generation, + "conversion_done": False, + "conversion_error": None, + } + session.segments.append(record) + session.sealed_until = max(session.sealed_until, record["end"]) + session.generation += 1 + session.segment_index += 1 + return record + + @staticmethod + def _ffconcat_file_line(path: Path) -> str: + """Quote a frame path for ffconcat: ' inside a quoted name -> '\\''.""" + escaped = path.as_posix().replace("'", "'\\''") + return f"file '{escaped}'" + + def _write_concat(self, session: IosPhysicalRecordingSession, record: dict[str, Any]) -> Path: + """Write an ffconcat list preserving real capture timestamps.""" + frames = record["frames"] + end_monotonic = record["end"] + (session.anchor_monotonic or 0.0) + lines = ["ffconcat version 1.0"] + for index, frame in enumerate(frames): + lines.append(self._ffconcat_file_line(Path(frame["path"]))) + following = frames[index + 1]["at"] if index + 1 < len(frames) else end_monotonic + lines.append(f"duration {max(0.001, following - frame['at']):.3f}") + # The concat demuxer ignores the final frame's duration unless the file + # line repeats, so list the last frame twice to keep its tail visible. + lines.append(self._ffconcat_file_line(Path(frames[-1]["path"]))) + concat_path = Path(frames[0]["path"]).parent / "frames.txt" + concat_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return concat_path + + async def _convert_record( + self, session: IosPhysicalRecordingSession, record: dict[str, Any] + ) -> None: + output = Path(record["output_path"]) + width = max(2, int(record["width"] or 1080)) // 2 * 2 + height = max(2, int(record["height"] or 1920)) // 2 * 2 + temporary = output.with_name(f"{output.stem}.part.mp4") + try: + if temporary.exists(): + temporary.unlink() + concat = self._write_concat(session, record) + returncode, stderr = await _run_ffmpeg( + [ + "-y", + "-f", + "concat", + "-safe", + "0", + "-i", + str(concat), + "-vf", + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:" + "color=black,setsar=1", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary), + ] + ) + metadata = await probe_video_segment( + temporary, timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS + ) + valid = ( + returncode == 0 + and temporary.exists() + and metadata.get("duration", 0) > 0 + and metadata.get("width", 0) > 0 + and metadata.get("height", 0) > 0 + ) + if valid: + temporary.replace(output) + record["conversion_done"] = True + return + record["conversion_error"] = "finalization produced no valid MP4" + logger.error( + f"Physical iOS recording finalization failed (code {returncode}): " + f"{stderr.decode(errors='replace')[-2000:]}" + ) + except TimeoutError as exc: + # A bounded probe that expires records a failure — never report a + # segment whose metadata could not be verified as converted. + record["conversion_error"] = f"metadata probe timed out: {exc}" + logger.error(f"Physical iOS recording probe timed out: {exc}") + except OSError as exc: + record["conversion_error"] = str(exc) + logger.error(f"Physical iOS recording finalization failed: {exc}") + try: + if temporary.exists(): + temporary.unlink() + except OSError: + pass + session.errors.append(f"Segment {record['output_path']} failed finalization; frames kept") + + async def _roll( + self, session: IosPhysicalRecordingSession, end_monotonic: float, reason: str + ) -> None: + """Seal the current frames and open the next segment directory.""" + record = self._seal_current_segment(session, end_monotonic) + leftover = session.frames + if record is not None: + logger.info(f"Rolling physical iOS recording segment after {reason}") + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.is_active: + await self._start_segment(session) + frames_dir = session.frames_dir + if frames_dir is None: + raise RuntimeError("Physical iOS recording roll found no frame directory.") + for frame in leftover: + target = frames_dir / Path(frame["path"]).name + Path(frame["path"]).rename(target) + frame["path"] = target + session.frames = leftover + if leftover: + with Image.open(leftover[0]["path"]) as image: + session.capture_width, session.capture_height = image.size + session.segment_started_monotonic = leftover[0]["at"] + + async def _watchdog(self, session: IosPhysicalRecordingSession) -> None: + try: + while True: + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS) + async with self._lock: + if not session.is_active: + return + poll = session.poll_task + if poll is not None and poll.done() and not poll.cancelled(): + exc = poll.exception() + if exc is not None: + session.errors.append(f"Frame capture task died: {exc}") + session.is_active = False + return + if ( + session.anchor_monotonic is not None + and time.monotonic() - session.anchor_monotonic + >= self._max_duration_seconds + ): + session.errors.append("Recording reached its duration limit") + session.is_active = False + return + except asyncio.CancelledError: + return + except Exception as exc: + logger.error(f"Physical iOS recording supervisor failed: {exc}") + session.errors.append(f"recording supervisor failed: {exc}") + + async def start( + self, + output_dir: Path | None = None, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, + ) -> IosPhysicalRecordingSession: + async with self._lock: + if self._session is not None and self._session.is_active: + raise RuntimeError(f"iOS recording is already active on {self._device_id}") + output = ( + Path(output_dir) + if output_dir is not None + else Path(tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings"))) + ) + output.mkdir(parents=True, exist_ok=True) + self._output_dir = output + self._max_duration_seconds = max_duration_seconds + self._consecutive_failures = 0 + session = IosPhysicalRecordingSession( + video_id=uuid4(), + device_id=self._device_id, + start_time=time.time(), + is_active=True, + ) + self._session = session + await self._start_segment(session) + session.poll_task = asyncio.create_task(self._poll_loop(session)) + # The first-frame wait must run outside the lock: the poll loop takes + # the same lock to append captured frames. + completed = False + try: + await self._poll_first_frame(session) + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + completed = True + return session + finally: + if not completed: + session.is_active = False + if session.poll_task and not session.poll_task.done(): + session.poll_task.cancel() + if session.poll_task is not None: + with contextlib.suppress( + asyncio.CancelledError, OSError, ValueError, RuntimeError, TimeoutError + ): + await session.poll_task + + async def seal(self, through_time: float | None = None) -> None: + """Seal the current segment so its final MP4 can be read safely.""" + async with self._lock: + session = self._session + if session is None or not session.is_active: + return + if session.anchor_monotonic is None: + return + if through_time is not None and session.sealed_until >= through_time: + return + end_monotonic = ( + session.anchor_monotonic + through_time + if through_time is not None + else time.monotonic() + ) + end_monotonic = min(end_monotonic, time.monotonic()) + await self._roll(session, end_monotonic, "seal") + + async def _finalize(self, session: IosPhysicalRecordingSession) -> Path | None: + session.is_active = False + for task in (session.poll_task, session.watchdog_task): + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.debug(f"Physical iOS recording task ended with an error: {exc}") + async with self._lock: + record = self._seal_current_segment(session, time.monotonic()) + if record is not None: + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + mp4_paths = [ + Path(record["output_path"]) + for record in session.segments + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ] + if not mp4_paths: + session.errors.append("No finalized physical iOS recording segments") + return None + shift = 0.0 + if session.data_engine_start_time is not None: + shift = session.start_time - session.data_engine_start_time + offsets = { + Path(record["output_path"]): max(0.0, float(record["start"]) + shift) + for record in session.segments + if record.get("conversion_done") and Path(record["output_path"]).exists() + } + output_dir = mp4_paths[0].parent + try: + manifest = await write_recording_manifest( + output_dir, + mp4_paths, + offsets, + probe_timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + session.errors.append(f"Recording manifest probe failed: {exc}") + return None + if manifest is None: + session.errors.append("Recording manifest has no valid segments") + return None + return mp4_paths[0] + + async def stop(self) -> Path | None: + """Finalize all owned segments and write the recording manifest.""" + session = self._session + if session is None: + return None + # _finalize takes _lock itself for the seal — do not hold it here. + finalize = asyncio.ensure_future(self._finalize(session)) + try: + # Shield so a cancelled caller cannot abort mid-finalize and + # lose already-captured frames (mirrors IosScreenRecorder). + return await asyncio.shield(finalize) + except asyncio.CancelledError: + await finalize + raise diff --git a/artemis/drivers/ios/recording.py b/artemis/drivers/ios/recording.py new file mode 100644 index 00000000..b056e044 --- /dev/null +++ b/artemis/drivers/ios/recording.py @@ -0,0 +1,613 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS Simulator recording via ``xcrun simctl io recordVideo``. + +simctl writes a variable frame-rate H.264 ``.mov`` and announces the first +processed frame on stderr as ``Recording started``; that marker anchors the +recording timeline, not process spawn. Each owned ``.mov`` is finalized to a +browser-safe CFR MP4 whose last frame is cloned across the segment's static +tail, so recorded event timestamps are never stretched. iOS capture has no +audio. Segments roll on display-dimension change, max duration, or recorder +exit; restart gaps stay gaps in rendered clips. +""" + +import asyncio +from pathlib import Path +import re +import signal +import tempfile +import time +from typing import Any +from uuid import uuid4 + +from artemis.config.paths import get_temp_dir +from artemis.drivers.ios.discovery import reap_process +from artemis.utils.video import ( + get_ffmpeg_path, + probe_video_segment, + write_recording_manifest, +) +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import RecordingSession + +logger = get_logger(__name__) + +RECORDING_STARTED_MARKER = "Recording started" +# Longest a single iOS capture may run before auto-stopping (15 minutes). +DEFAULT_MAX_DURATION_SECONDS = 900 +STARTUP_TIMEOUT_SECONDS = 30.0 +SIGINT_FLUSH_TIMEOUT_SECONDS = 10.0 +TERMINATE_TIMEOUT_SECONDS = 3.0 +PROBES_TIMEOUT_SECONDS = 3.0 +SEGMENT_PROBE_TIMEOUT_SECONDS = 30.0 +FFMPEG_TIMEOUT_SECONDS = 120.0 +WATCHDOG_INTERVAL_SECONDS = 0.5 +MAX_CONSECUTIVE_FAILURES = 3 +MIN_HEALTHY_SEGMENT_SECONDS = 10.0 +MAX_CONCURRENT_CONVERSIONS = 2 +STDERR_BUFFER_LINES = 200 + + +class IosRecordingSession(RecordingSession): + """Recording session for native iOS ``simctl`` capture.""" + + segments: list[dict[str, Any]] = [] + segment_index: int = 0 + segment_started_at: float | None = None + segment_started_monotonic: float | None = None + anchor_monotonic: float | None = None + stderr_task: asyncio.Task | None = None + stderr_lines: list[str] = [] + conversion_tasks: list[asyncio.Task] = [] + + +def _parse_display_dimensions(text: str) -> tuple[int, int] | None: + """Largest ``IOSurface port`` (width, height) from ``simctl io enumerate``. + + ``recordVideo`` captures the device LCD without ``--display``; only that + framebuffer reports an ``IOSurface port`` (external scene displays carry + only ``Default width``/``height``), and its dimensions swap on rotation, + so the largest reported surface is the segment's coded size. + """ + best: tuple[int, int] | None = None + best_area = 0 + for match in re.finditer( + r"IOSurface port:\s*\n\s*width\s*=\s*(\d+)\s*\n\s*height\s*=\s*(\d+)", text + ): + width, height = int(match.group(1)), int(match.group(2)) + if width * height > best_area: + best, best_area = (width, height), width * height + return best + + +async def probe_display_dimensions(device_id: str) -> tuple[int, int] | None: + """Read the recordVideo target display's pixel size via simctl.""" + try: + process = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + device_id, + "enumerate", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + except OSError: + return None + try: + stdout, _stderr = await asyncio.wait_for(process.communicate(), PROBES_TIMEOUT_SECONDS) + except TimeoutError: + await _reap_probe(process) + return None + except asyncio.CancelledError: + await _reap_probe(process) + raise + if process.returncode != 0: + return None + return _parse_display_dimensions(stdout.decode(errors="replace")) + + +_reap_probe = reap_process + + +async def finalize_mov_to_mp4( + source_path: Path, + output_path: Path, + width: int, + height: int, + wall_span_seconds: float, +) -> bool: + """Atomically finalize an owned .mov to a fixed-canvas CFR MP4. + + ``tpad`` clones the final frame across the segment's remaining wall span + so sparse VFR tails pad without stretching any recorded event time. + """ + if not source_path.exists() or source_path.stat().st_size == 0: + return False + width = max(2, int(width)) // 2 * 2 + height = max(2, int(height)) // 2 * 2 + span = max(0.001, float(wall_span_seconds)) + # fps resamples the raw VFR timeline before scale: scale buffers a frame + # and feeding it to fps first drops the final frame on sparse sources. + video_filter = ( + "setpts=PTS-STARTPTS,fps=30," + f"scale={width}:{height}:force_original_aspect_ratio=decrease:" + f"force_divisible_by=2,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:" + f"color=black,setsar=1," + f"tpad=stop_mode=clone:stop_duration={span:.3f}" + ) + temporary_path = output_path.with_name(f"{output_path.stem}.part.mp4") + if temporary_path.exists(): + temporary_path.unlink() + try: + process = await asyncio.create_subprocess_exec( + get_ffmpeg_path(), + "-y", + "-i", + str(source_path), + "-map", + "0:v:0", + "-vf", + video_filter, + "-t", + f"{span:.3f}", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(temporary_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + _stdout, stderr = await asyncio.wait_for(process.communicate(), FFMPEG_TIMEOUT_SECONDS) + metadata = await probe_video_segment( + temporary_path, timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS + ) + except TimeoutError: + await reap_process(process) + raise RuntimeError("iOS recording finalization timed out") + except asyncio.CancelledError: + # Reap the owned ffmpeg child and drop the partial .mp4 before + # cancellation propagates — never leave an orphan encoder behind. + await reap_process(process) + if temporary_path.exists(): + temporary_path.unlink() + raise + valid = ( + process.returncode == 0 + and temporary_path.exists() + and metadata.get("duration", 0) > 0 + and metadata.get("width", 0) > 0 + and metadata.get("height", 0) > 0 + ) + if valid: + temporary_path.replace(output_path) + return True + logger.error( + f"iOS recording finalization failed (code {process.returncode}): " + f"{stderr.decode(errors='replace')[-2000:]}" + ) + except OSError as exc: + logger.error(f"iOS recording finalization failed: {exc}") + except RuntimeError as exc: + logger.error(f"iOS recording finalization failed: {exc}") + if temporary_path.exists(): + temporary_path.unlink() + return False + + +class IosScreenRecorder: + """Owns one simctl recording lifecycle for one pinned simulator UDID.""" + + def __init__(self, device_id: str): + self._device_id = device_id + self._session: IosRecordingSession | None = None + self._output_dir: Path | None = None + self._lock = asyncio.Lock() + self._max_duration_seconds = DEFAULT_MAX_DURATION_SECONDS + self._consecutive_failures = 0 + self._conversion_semaphore = asyncio.Semaphore(MAX_CONCURRENT_CONVERSIONS) + + @property + def session(self) -> IosRecordingSession | None: + """The latest session, kept after stop/failure for error reporting.""" + return self._session + + def _output_root(self) -> Path: + if self._output_dir is None: + raise RuntimeError("iOS recording has no output directory") + return self._output_dir + + def _segment_source_path(self, session: IosRecordingSession, index: int) -> Path: + output_dir = self._output_root() + while True: + name = "recording.mov" if index == 0 else f"recording_{index:03d}.mov" + candidate = output_dir / name + if not candidate.exists(): + return candidate + index += 1 + session.segment_index = index + + def _segment_output_path(self, session: IosRecordingSession, index: int) -> Path: + output_dir = self._output_root() + if index == 0: + return output_dir / "recording.mp4" + return output_dir / f"recording_{index:03d}.mp4" + + async def _drain_stderr( + self, + process: asyncio.subprocess.Process, + session: IosRecordingSession, + first_frame: asyncio.Future, + ) -> None: + try: + assert process.stderr is not None + while True: + line = await process.stderr.readline() + if not line: + if not first_frame.done(): + first_frame.set_exception( + RuntimeError( + "simctl recordVideo closed stderr before 'Recording started'" + ) + ) + return + text = line.decode(errors="replace").rstrip() + session.stderr_lines.append(text) + if len(session.stderr_lines) > STDERR_BUFFER_LINES: + del session.stderr_lines[: len(session.stderr_lines) - STDERR_BUFFER_LINES] + if RECORDING_STARTED_MARKER in text and not first_frame.done(): + first_frame.set_result((time.time(), time.monotonic())) + except asyncio.CancelledError: + raise + except Exception as exc: + if not first_frame.done(): + first_frame.set_exception(exc) + + async def _reap(self, process: asyncio.subprocess.Process) -> float: + """SIGINT to flush, escalate to terminate/kill; returns end monotonic.""" + end_monotonic = time.monotonic() + if process.returncode is not None: + return end_monotonic + try: + process.send_signal(signal.SIGINT) + await asyncio.wait_for(process.wait(), SIGINT_FLUSH_TIMEOUT_SECONDS) + except (ProcessLookupError, TimeoutError): + if process.returncode is None: + try: + process.terminate() + except ProcessLookupError: + return end_monotonic + try: + await asyncio.wait_for(process.wait(), TERMINATE_TIMEOUT_SECONDS) + except (ProcessLookupError, TimeoutError): + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + await process.wait() + return end_monotonic + + async def _spawn_recorder(self, session: IosRecordingSession) -> None: + """Spawn simctl recordVideo and anchor at its first-frame marker.""" + index = session.segment_index + source_path = self._segment_source_path(session, index) + process = await asyncio.create_subprocess_exec( + "xcrun", + "simctl", + "io", + self._device_id, + "recordVideo", + "--codec=h264", + str(source_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + first_frame: asyncio.Future = asyncio.get_running_loop().create_future() + session.stderr_task = asyncio.create_task(self._drain_stderr(process, session, first_frame)) + started = False + try: + marker_wall, marker_monotonic = await asyncio.wait_for( + asyncio.shield(first_frame), STARTUP_TIMEOUT_SECONDS + ) + session.process = process + session.local_video_path = source_path + session.segment_started_at = marker_wall + session.segment_started_monotonic = marker_monotonic + if session.anchor_monotonic is None: + session.start_time = marker_wall + session.anchor_monotonic = marker_monotonic + dimensions = await probe_display_dimensions(self._device_id) + if dimensions: + session.capture_width, session.capture_height = dimensions + logger.info( + f"iOS recording segment {index} first frame " + f"{marker_wall - session.start_time:.2f}s after anchor on {self._device_id}" + ) + started = True + finally: + if not started: + # Startup never completed: the spawned recorder is still + # owned by this coroutine — drain stderr and reap it before + # the original exception/cancellation propagates. + if not first_frame.done(): + first_frame.cancel() + session.stderr_task.cancel() + try: + await session.stderr_task + except asyncio.CancelledError: + pass + except (OSError, RuntimeError, ValueError, TimeoutError) as stderr_error: + logger.debug(f"iOS recorder stderr drain ended with an error: {stderr_error}") + reap = asyncio.ensure_future(self._reap(process)) + try: + await asyncio.shield(reap) + except asyncio.CancelledError: + await reap # finish reaping even when the caller was cancelled + + def _seal_current_segment( + self, session: IosRecordingSession, end_monotonic: float + ) -> dict[str, Any]: + if session.segment_started_monotonic is None or session.anchor_monotonic is None: + raise RuntimeError("No started iOS recording segment to seal") + record = { + "path": session.local_video_path, + "output_path": self._segment_output_path(session, session.segment_index), + "start": max(0.0, session.segment_started_monotonic - session.anchor_monotonic), + "end": max(0.0, end_monotonic - session.anchor_monotonic), + "width": session.capture_width, + "height": session.capture_height, + "generation": session.generation, + "conversion_done": False, + "conversion_error": None, + } + session.segments.append(record) + session.sealed_until = max(session.sealed_until, record["end"]) + session.generation += 1 + session.segment_index += 1 + session.segment_started_monotonic = None + session.segment_started_at = None + return record + + async def _convert_record(self, session: IosRecordingSession, record: dict[str, Any]) -> None: + source = Path(record["path"]) + output = Path(record["output_path"]) + span = max(0.001, float(record["end"]) - float(record["start"])) + async with self._conversion_semaphore: + ok = await finalize_mov_to_mp4( + source, output, int(record["width"] or 1080), int(record["height"] or 1920), span + ) + if ok: + record["conversion_done"] = True + else: + record["conversion_error"] = "finalization produced no valid MP4" + session.errors.append( + f"Segment {record['output_path']} failed finalization; raw capture kept" + ) + + async def _roll(self, session: IosRecordingSession, end_monotonic: float, reason: str) -> None: + """Seal the current segment and start the next recorder first.""" + record = None + if session.segment_started_monotonic is not None: + process = session.process + if process is not None and process.returncode is None: + end_monotonic = await self._reap(process) + if session.stderr_task and not session.stderr_task.done(): + session.stderr_task.cancel() + record = self._seal_current_segment(session, end_monotonic) + logger.info(f"Rolling iOS recording segment after {reason}") + restarted = False + if session.is_active: + if self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES: + # Spawns that keep dying young are a crash loop: stop + # respawning (the sealed segment still converts below). + session.is_active = False + session.errors.append("Recording recovery limit reached; session stopped") + else: + try: + await self._spawn_recorder(session) + restarted = True + except asyncio.CancelledError: + raise + except (OSError, RuntimeError, TimeoutError) as exc: + self._consecutive_failures += 1 + session.errors.append(f"Recorder restart failed: {exc}") + if record is not None: + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if ( + not restarted + and session.is_active + and self._consecutive_failures >= MAX_CONSECUTIVE_FAILURES + ): + session.errors.append("Recording recovery limit reached; session stopped") + session.is_active = False + + async def _watchdog(self, session: IosRecordingSession) -> None: + try: + while True: + await asyncio.sleep(WATCHDOG_INTERVAL_SECONDS) + async with self._lock: + if not session.is_active: + return + # Re-verify the trigger under the lock: a concurrent roll + # may have already replaced the dead process. + process = session.process + crashed = process is None or process.returncode is not None + dimensions = await probe_display_dimensions(self._device_id) + rotated = bool( + dimensions + and session.capture_width + and session.capture_height + and dimensions != (session.capture_width, session.capture_height) + ) + age = time.monotonic() - (session.segment_started_monotonic or time.monotonic()) + overdue = age >= self._max_duration_seconds + if crashed and age < MIN_HEALTHY_SEGMENT_SECONDS: + # A respawn that dies before surviving a healthy interval + # is a crash loop, not recovery — count it even though a + # successful respawn would otherwise reset the counter. + self._consecutive_failures += 1 + elif not crashed and age >= MIN_HEALTHY_SEGMENT_SECONDS: + self._consecutive_failures = 0 + if not crashed and not rotated and not overdue: + continue + reason = ( + "recorder exit" if crashed else "rotation" if rotated else "duration limit" + ) + await self._roll(session, time.monotonic(), reason) + except asyncio.CancelledError: + return + except Exception as exc: + logger.error(f"iOS recording supervisor failed: {exc}") + session.errors.append(f"recording supervisor failed: {exc}") + + async def start( + self, + output_dir: Path | None = None, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, + ) -> IosRecordingSession: + async with self._lock: + if self._session is not None and self._session.is_active: + raise RuntimeError(f"iOS recording is already active on {self._device_id}") + output = ( + Path(output_dir) + if output_dir is not None + else Path(tempfile.mkdtemp(prefix="ios_recording_", dir=get_temp_dir("recordings"))) + ) + output.mkdir(parents=True, exist_ok=True) + self._output_dir = output + self._max_duration_seconds = max_duration_seconds + self._consecutive_failures = 0 + session = IosRecordingSession( + video_id=uuid4(), + device_id=self._device_id, + start_time=time.time(), + local_video_path=output / "recording.mov", + is_active=True, + ) + self._session = session + started = False + try: + await self._spawn_recorder(session) + session.watchdog_task = asyncio.create_task(self._watchdog(session)) + started = True + return session + finally: + if not started: + session.is_active = False + + async def seal(self, through_time: float | None = None) -> None: + """Seal the current segment through ``through_time`` (recording-relative).""" + async with self._lock: + session = self._session + if session is None or not session.is_active: + return + if session.anchor_monotonic is None: + return + if through_time is not None and session.sealed_until >= through_time: + return + end_monotonic = ( + session.anchor_monotonic + through_time + if through_time is not None + else time.monotonic() + ) + end_monotonic = min(end_monotonic, time.monotonic()) + if ( + session.segment_started_monotonic is not None + and end_monotonic <= session.segment_started_monotonic + ): + return + await self._roll(session, end_monotonic, "seal") + + async def _finalize(self, session: IosRecordingSession) -> Path | None: + if session.is_active: + session.is_active = False + if session.watchdog_task and not session.watchdog_task.done(): + session.watchdog_task.cancel() + try: + await session.watchdog_task + except asyncio.CancelledError: + pass + except Exception as exc: + logger.debug(f"iOS recording watchdog ended with an error: {exc}") + end_monotonic = time.monotonic() + if session.process is not None and session.process.returncode is None: + end_monotonic = await self._reap(session.process) + if session.stderr_task and not session.stderr_task.done(): + session.stderr_task.cancel() + if session.segment_started_monotonic is not None: + record = self._seal_current_segment(session, end_monotonic) + session.conversion_tasks.append( + asyncio.create_task(self._convert_record(session, record)) + ) + if session.conversion_tasks: + await asyncio.gather(*session.conversion_tasks, return_exceptions=True) + mp4_paths = [ + Path(record["output_path"]) + for record in session.segments + if record.get("conversion_done") + and Path(record["output_path"]).exists() + and Path(record["output_path"]).stat().st_size > 0 + ] + if not mp4_paths: + session.errors.append("No finalized iOS recording segments") + return None + shift = 0.0 + if session.data_engine_start_time is not None: + shift = session.start_time - session.data_engine_start_time + offsets = { + Path(record["output_path"]): max(0.0, float(record["start"]) + shift) + for record in session.segments + if record.get("conversion_done") and Path(record["output_path"]).exists() + } + output_dir = mp4_paths[0].parent + try: + manifest = await write_recording_manifest( + output_dir, + mp4_paths, + offsets, + probe_timeout_seconds=SEGMENT_PROBE_TIMEOUT_SECONDS, + ) + except (OSError, TimeoutError) as exc: + session.errors.append(f"Recording manifest probe failed: {exc}") + return None + if manifest is None: + session.errors.append("Recording manifest has no valid segments") + return None + return mp4_paths[0] + + async def stop(self) -> Path | None: + """Finalize all owned segments and write the version-2 manifest.""" + async with self._lock: + session = self._session + if session is None: + return None + finalize = asyncio.ensure_future(self._finalize(session)) + try: + return await asyncio.shield(finalize) + except asyncio.CancelledError: + await finalize # let cleanup finish before propagating + raise diff --git a/artemis/drivers/ios/wda.py b/artemis/drivers/ios/wda.py new file mode 100644 index 00000000..27515b00 --- /dev/null +++ b/artemis/drivers/ios/wda.py @@ -0,0 +1,492 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""WebDriverAgent HTTP client for physical iOS UI automation. + +Xcode's ``DeviceInteraction*`` MCP tools accept simulators only, so physical +devices are driven through WebDriverAgent — the same XCUITest bridge Appium +uses. The client speaks plain HTTP to the WDA server running on the device +and has no third-party dependencies; every call is offloaded to a thread so +the driver stays fully async. + +Reaching the device-side server needs one of: + +- ``ARTEMIS_IOS_WDA_URL``: an explicit endpoint such as + ``http://127.0.0.1:8100`` for ``iproxy``/``pymobiledevice3`` forwards or + ``http://:8100`` when the phone shares the LAN. +- The CoreDevice tunnel address from ``devicectl device info details`` — + paired devices already hold a managed IPv6 tunnel that routes TCP. +- ``ARTEMIS_IOS_WDA_HOST``: just a host/IP; port 8100 is assumed. +""" + +import asyncio +import base64 +import http.client +import ipaddress +import json +import os +from typing import Any +import urllib.error +import urllib.request + +from artemis.drivers.ios.hierarchy import pixel_element +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +WDA_URL_ENV = "ARTEMIS_IOS_WDA_URL" +WDA_HOST_ENV = "ARTEMIS_IOS_WDA_HOST" +WDA_DEFAULT_PORT = 8100 +WDA_REQUEST_TIMEOUT = 30.0 + + +class WdaUnavailableError(RuntimeError): + """The WebDriverAgent server could not be reached or did not respond.""" + + +class WdaClient: + """Minimal WebDriverAgent client covering Artemis's interaction surface.""" + + def __init__(self, base_url: str, timeout: float = WDA_REQUEST_TIMEOUT): + self._base = base_url.rstrip("/") + self._timeout = timeout + self._session_id: str | None = None + + @property + def base_url(self) -> str: + return self._base + + @property + def session_id(self) -> str | None: + return self._session_id + + @property + def connected(self) -> bool: + return self._session_id is not None + + def _sync_request( + self, method: str, path: str, payload: dict[str, Any] | None, timeout: float + ) -> Any: + body = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request( + f"{self._base}{path}", + data=body, + method=method, + headers={"Content-Type": "application/json"} if body else {}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + raw = response.read() + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace")[:500] + raise RuntimeError(f"WebDriverAgent {method} {path} failed: HTTP {error.code} {detail}") + except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException) as error: + # HTTPException covers garbage services answering probed ports — + # BadStatusLine is not an OSError, so it needs mapping here. + raise WdaUnavailableError( + f"WebDriverAgent at {self._base} is unreachable: {error}. " + "Check the device connection, the WDA runner process, and any " + "port forwarding (iproxy or 'pymobiledevice3 remote')." + ) + try: + payload_out = json.loads(raw) + except ValueError: + return raw + if isinstance(payload_out, dict) and "value" in payload_out: + value = payload_out["value"] + if isinstance(value, dict) and value.get("error"): + message = value.get("message") or value["error"] + raise RuntimeError(f"WebDriverAgent {method} {path} failed: {message}") + # WDA reports the live session in the outer /status envelope, not + # inside value — surface it so a foreign session is detectable. + if ( + path == "/status" + and isinstance(value, dict) + and isinstance(payload_out.get("sessionId"), str) + and payload_out["sessionId"] + ): + value["sessionId"] = payload_out["sessionId"] + return value + return payload_out + + async def _request( + self, + method: str, + path: str, + payload: dict[str, Any] | None = None, + timeout: float | None = None, + ) -> Any: + request_task = asyncio.create_task( + asyncio.to_thread(self._sync_request, method, path, payload, timeout or self._timeout) + ) + try: + return await asyncio.shield(request_task) + except asyncio.CancelledError: + # The blocking urllib call keeps running on its thread; drain it + # before propagating so a cancelled input cannot still land on the + # device after the caller (and its device lease) moved on. + try: + await request_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"WDA request drain failed while cancelling: {drain_error}") + raise + + # --- Session lifecycle --- + + async def status(self, timeout: float = 5.0) -> dict[str, Any] | None: + try: + value = await self._request("GET", "/status", timeout=timeout) + except (RuntimeError, WdaUnavailableError, OSError) as error: + logger.debug(f"WDA status probe failed for {self._base}: {error}") + return None + # Reject non-JSON/foreign bodies instead of wrapping them: a proxy or + # unrelated service answering on this port is not WebDriverAgent. + return value if isinstance(value, dict) else None + + async def device_info(self, timeout: float = 10.0) -> dict[str, Any]: + """GET /wda/device/info — available without a session. + + ``uuid`` here is ``identifierForVendor``, not the device UDID, so + identity checks must rely on ``name``/``isSimulator`` instead. + """ + value = await self._request("GET", "/wda/device/info", timeout=timeout) + if not isinstance(value, dict): + raise RuntimeError( + f"WebDriverAgent /wda/device/info returned a non-JSON response: {value!r:.300}" + ) + return value + + async def _create_owned_session(self, adopt_existing: bool) -> str: + """Status preflight, POST /session, and ID assignment as one unit.""" + if self._session_id: + return self._session_id + status = await self.status() + if status is None: + raise WdaUnavailableError(f"WebDriverAgent at {self._base} did not answer /status.") + active = status.get("sessionId") + if isinstance(active, str) and active: + if adopt_existing: + # The driver launched this runner itself, so its auto-created + # session is ours to reuse — not a foreign client to protect. + self._session_id = active + return active + raise RuntimeError( + "Refusing to replace the active WebDriverAgent session " + f"({active}), which this client does not own — POST /session " + "would kill it. Point ARTEMIS_IOS_WDA_URL at a dedicated WDA " + "server or close the existing session first." + ) + payload = { + "capabilities": { + "alwaysMatch": {"platformName": "iOS"}, + "firstMatch": [{}], + } + } + value = await self._request("POST", "/session", payload, timeout=60.0) + session_id = None + if isinstance(value, dict): + session_id = value.get("sessionId") + if session_id is None and isinstance(value.get("capabilities"), dict): + session_id = value.get("capabilities", {}).get("sessionId") + if not isinstance(session_id, str) or not session_id: + raise RuntimeError( + f"WebDriverAgent did not return a session id (response: {value!r:.300})." + ) + self._session_id = session_id + return session_id + + async def open_session(self, adopt_existing: bool = False) -> str: + """Create a WDA session, refusing to take over a foreign one. + + ``adopt_existing=True`` reuses the active session reported by + ``/status``; only safe when the caller owns the WDA runner process + (auto-launched runners create a session on startup). Foreign or + user-provisioned endpoints must keep the default refusal. + + Cancellation-safe: when the caller is cancelled while creation is in + flight, the request is drained and any session it produced is closed + before the CancelledError propagates. + """ + create_task = asyncio.ensure_future(self._create_owned_session(adopt_existing)) + try: + return await asyncio.shield(create_task) + except asyncio.CancelledError: + session_id = None + try: + session_id = await create_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"WDA session-create drain failed while cancelling: {drain_error}") + if session_id: + try: + await self.close_session() + except (OSError, ValueError, RuntimeError, TimeoutError) as close_error: + # Best-effort teardown of the orphaned session. + logger.debug( + f"WDA session close after cancelled open_session failed: {close_error}" + ) + raise + + async def close_session(self) -> None: + session_id, self._session_id = self._session_id, None + if session_id is None: + return + try: + await self._request("DELETE", f"/session/{session_id}", timeout=10.0) + except (RuntimeError, WdaUnavailableError, OSError) as error: + logger.debug(f"WDA session delete failed: {error}") + + # --- Observation --- + + async def screenshot_png(self) -> bytes: + value = await self._request("GET", "/screenshot") + if not isinstance(value, str) or not value: + raise RuntimeError("WebDriverAgent returned no screenshot data.") + return base64.b64decode(value) + + async def source_json(self) -> dict[str, Any]: + value = await self._request("GET", "/source?format=json") + if not isinstance(value, dict): + raise RuntimeError("WebDriverAgent returned a non-JSON hierarchy.") + return value + + async def window_size(self) -> tuple[float, float]: + path = f"/session/{self._session_id}/window/size" if self._session_id else "/window/size" + value = await self._request("GET", path) + if isinstance(value, dict) and "width" in value and "height" in value: + return float(value["width"]), float(value["height"]) + raise RuntimeError(f"WebDriverAgent returned an unexpected window size: {value!r}") + + async def active_app(self) -> str | None: + path = "/wda/activeAppInfo" + if self._session_id: + path = f"/session/{self._session_id}/wda/activeAppInfo" + try: + value = await self._request("GET", path) + except (RuntimeError, WdaUnavailableError): + return None + if isinstance(value, dict): + bundle = value.get("bundleId") or value.get("bundleIdentifier") + return bundle if isinstance(bundle, str) and bundle else None + return None + + # --- Input --- + + async def _actions(self, pointer_actions: list[dict[str, Any]]) -> None: + session = self._require_session() + payload = { + "actions": [ + { + "type": "pointer", + "id": "artemis-finger", + "parameters": {"pointerType": "touch"}, + "actions": pointer_actions, + } + ] + } + await self._request("POST", f"/session/{session}/actions", payload) + + def _require_session(self) -> str: + if not self._session_id: + raise RuntimeError("WebDriverAgent session is not open.") + return self._session_id + + async def tap(self, x: float, y: float, hold_ms: int = 0) -> None: + actions: list[dict[str, Any]] = [ + {"type": "pointerMove", "duration": 0, "x": x, "y": y}, + {"type": "pointerDown", "button": 0}, + ] + if hold_ms > 0: + actions.append({"type": "pause", "duration": hold_ms}) + actions.append({"type": "pointerUp", "button": 0}) + await self._actions(actions) + + async def swipe(self, sx: float, sy: float, ex: float, ey: float, duration_ms: int) -> None: + await self._actions( + [ + {"type": "pointerMove", "duration": 0, "x": sx, "y": sy}, + {"type": "pointerDown", "button": 0}, + {"type": "pause", "duration": 50}, + {"type": "pointerMove", "duration": duration_ms, "x": ex, "y": ey}, + {"type": "pointerUp", "button": 0}, + ] + ) + + async def type_text(self, text: str) -> None: + session = self._require_session() + # ``value`` is a list of Unicode code points — newlines and non-ASCII + # text carry their literal values to the focused field. + payload = {"value": list(text)} + await self._request("POST", f"/session/{session}/wda/keys", payload) + + async def press_button(self, name: str) -> bool: + """WDA hardware buttons: home, volumeUp, volumeDown, power.""" + session = self._require_session() + try: + await self._request("POST", f"/session/{session}/wda/pressButton", {"name": name}) + return True + except WdaUnavailableError: + # The transport died — propagate so callers don't misreport an + # outage as an unsupported button. + raise + except RuntimeError: + return False + + async def homescreen(self) -> None: + await self._request("POST", "/wda/homescreen") + + async def lock(self) -> None: + await self._request("POST", "/wda/lock") + + async def unlock(self) -> None: + await self._request("POST", "/wda/unlock") + + +def normalize_wda_url(raw: str) -> str: + """Accept bare hosts, IPv6 literals, or full URLs and return a base URL.""" + value = raw.strip() + if not value: + return value + if "://" not in value: + # Bare IPv6 literals need brackets once a port is attached. A bare + # "v6:port" string is ambiguous — treat the last group as a port only + # when the address part parses as a real IPv6 literal. + if value.count(":") > 1 and not value.startswith("["): + try: + ipaddress.IPv6Address(value) + value = f"[{value}]" + except ValueError: + address, _, port = value.rpartition(":") + try: + ipaddress.IPv6Address(address) + value = f"[{address}]:{port}" + except ValueError: + value = f"[{value}]" + value = f"http://{value}" + value = value.rstrip("/") + scheme, _, remainder = value.partition("://") + host_port, _, path = remainder.partition("/") + # A port is present when ':' follows the host (or the IPv6 ']' bracket). + has_port = ( + host_port.rsplit("]", 1)[-1].startswith(":") if "]" in host_port else ":" in host_port + ) + if has_port: + return value + # The default port belongs to the authority, ahead of any path suffix. + suffix = f"/{path}" if path else "" + return f"{scheme}://{host_port}:{WDA_DEFAULT_PORT}{suffix}" + + +def wda_url_candidates( + env_url: str | None = None, + env_host: str | None = None, + tunnel_ip: str | None = None, +) -> list[str]: + """Ordered endpoints to probe for a running WebDriverAgent server.""" + candidates: list[str] = [] + for raw in ( + env_url if env_url is not None else os.environ.get(WDA_URL_ENV), + env_host if env_host is not None else os.environ.get(WDA_HOST_ENV), + ( + f"[{tunnel_ip}]:{WDA_DEFAULT_PORT}" + if ":" in tunnel_ip + else f"{tunnel_ip}:{WDA_DEFAULT_PORT}" + ) + if tunnel_ip + else None, + f"127.0.0.1:{WDA_DEFAULT_PORT}", + ): + if not raw: + continue + url = normalize_wda_url(raw) + if url and url not in candidates: + candidates.append(url) + return candidates + + +async def probe_wda(candidates: list[str], timeout: float = 5.0) -> WdaClient | None: + """Return a client bound to the first endpoint that answers ``/status``.""" + for url in candidates: + client = WdaClient(url) + status = await client.status(timeout=timeout) + if status is not None: + return client + return None + + +_ELEMENT_TYPE_PREFIX = "XCUIElementType" + + +def parse_wda_elements( + node: dict[str, Any], + scale: tuple[float, float], + width: int, + height: int, +) -> list[dict[str, Any]]: + """Flatten a WDA ``/source?format=json`` tree into Artemis ui_elements. + + Mirrors ``parse_hierarchy``'s output: ``text``, ``resource_id``, ``class``, + ``bounds``, ``parsed_bounds``, and ``hit_point`` in screenshot pixels. + WDA's ``isVisible`` is advisory; elements with usable geometry are kept so + downstream consumers see everything XCTest reports. + """ + elements: list[dict[str, Any]] = [] + + def visit(entry: dict[str, Any]) -> None: + rect = entry.get("rect") + if not isinstance(rect, dict): + rect = {} + try: + x = float(rect.get("x") or 0.0) + y = float(rect.get("y") or 0.0) + w = float(rect.get("width") or 0.0) + h = float(rect.get("height") or 0.0) + except (TypeError, ValueError, AttributeError): + x = y = w = h = 0.0 + if w > 0 and h > 0: + left, top = round(x * scale[0]), round(y * scale[1]) + right, bottom = round((x + w) * scale[0]), round((y + h) * scale[1]) + if right > 0 and bottom > 0 and left < width and top < height: + label = entry.get("label") + value = entry.get("value") + name = entry.get("name") + text = "" + if isinstance(label, str) and label: + text = label + elif value is not None and str(value): + text = str(value) + element_type = str(entry.get("type") or "") + element = pixel_element( + text=text, + resource_id=name if isinstance(name, str) else "", + class_name=element_type.removeprefix(_ELEMENT_TYPE_PREFIX), + left=left, + top=top, + right=right, + bottom=bottom, + ) + element["hit_point"] = [(left + right) // 2, (top + bottom) // 2] + element["visible"] = bool(entry.get("isVisible", True)) + if value is not None and str(value) != text: + element["value"] = str(value) + elements.append(element) + children = entry.get("children") + if isinstance(children, list): + for child in children: + if isinstance(child, dict): + visit(child) + + visit(node) + return elements diff --git a/artemis/drivers/ios/xcode_driver.py b/artemis/drivers/ios/xcode_driver.py new file mode 100644 index 00000000..14360229 --- /dev/null +++ b/artemis/drivers/ios/xcode_driver.py @@ -0,0 +1,558 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator support using Xcode 27's native MCP and simctl tools.""" + +from __future__ import annotations + +import asyncio +import base64 +from io import BytesIO +import json +from pathlib import Path +import plistlib +import sys +from typing import TYPE_CHECKING, Any, Literal +from uuid import uuid4 + +from PIL import Image + +from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, XcodeBridge +from artemis.drivers.ios.discovery import ( + clear_ios_simulator_cache, + BOOTED_SIMULATOR_ID, + device_matches_identifier, + parse_simctl_devices, + parse_xcode_version, + plist_to_json, + run_xcrun, +) +from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy +from artemis.drivers.ios.recording import ( + DEFAULT_MAX_DURATION_SECONDS, + IosRecordingSession, + IosScreenRecorder, +) +from third_party.mobile_use.controllers.types import ElementQuery +from third_party.mobile_use.utils.logger import get_logger +from third_party.mobile_use.utils.video import get_active_session, remove_active_session + +if TYPE_CHECKING: + from artemis.drivers.ios.physical_recording import PhysicalIosRecorder + +logger = get_logger(__name__) + + +async def _parse_listapps_output(raw: bytes) -> dict[str, Any]: + """Parse ``simctl listapps`` output into a bundle-id keyed dict. + + simctl emits OpenStep (ASCII) plists on older Xcode, XML/binary on newer + ones; ``plistlib`` only reads the latter, so OpenStep payloads are routed + through ``plutil -convert json``. + """ + try: + return plistlib.loads(raw) + except (plistlib.InvalidFileException, ValueError): + pass + return json.loads(await plist_to_json(raw)) + + +class XcodeSimulatorDriver(BaseDeviceDriver): + """One explicitly selected iOS simulator and one native interaction session. + + Android remains the default platform. This driver neither enables Xcode + permissions nor controls physical devices. Unsupported Android operations + report an error instead of returning a synthetic success. + """ + + def __init__( + self, + device_id: str = BOOTED_SIMULATOR_ID, + width: int = 0, + height: int = 0, + *, + workspace_path: str | Path | None = None, + ): + self._device_id = device_id + self._width, self._height = width, height + self._workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) + self._scale: tuple[float, float] | None = None + self._session_key: str | None = None + self._bridge = XcodeBridge() + self._connect_lock = asyncio.Lock() + self._operation_lock = asyncio.Lock() + self._recorder: IosScreenRecorder | PhysicalIosRecorder | None = None + + @property + def device_id(self) -> str: + return self._device_id + + @property + def screen_size(self) -> tuple[int, int]: + return self._width, self._height + + async def resolve_device(self) -> str: + """Read-only validation and UDID selection, safe before the execution lease.""" + async with self._connect_lock: + await self._resolve_device() + return self._device_id + + async def _require_ios_host(self) -> None: + """Validate the macOS/Xcode host requirements shared by all iOS drivers.""" + if sys.platform != "darwin": + raise RuntimeError("iOS support requires macOS and Xcode 27 or later.") + version = parse_xcode_version(await run_xcrun("xcodebuild", "-version")) + if version is None or int(version.split(".")[0]) < 27: + raise RuntimeError( + "Native iOS interaction requires Xcode 27 or later. Set DEVELOPER_DIR to select it." + ) + + def _validate_workspace(self) -> None: + if self._workspace_path is not None and ( + self._workspace_path.suffix.lower() not in {".xcodeproj", ".xcworkspace"} + or not self._workspace_path.is_dir() + ): + raise ValueError( + "iOS workspace must be an existing .xcodeproj or .xcworkspace directory." + ) + + async def _resolve_device(self) -> dict[str, Any]: + self._validate_workspace() + await self._require_ios_host() + devices = json.loads(await run_xcrun("simctl", "list", "devices", "--json")) + available = parse_simctl_devices(devices) + if self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: + candidates = [device for device in available if device.get("state") == "Booted"] + if len(candidates) != 1: + raise ValueError( + "Select --device-serial ; 'booted' requires exactly one booted iOS simulator." + ) + else: + candidates = [ + device for device in available if device_matches_identifier(device, self._device_id) + ] + if len(candidates) > 1: + udids = sorted(str(d.get("udid") or "?") for d in candidates) + raise ValueError( + f"{len(candidates)} iOS simulators match {self._device_id!r} " + f"({', '.join(udids)}); use the simulator UDID instead." + ) + if not candidates: + raise ValueError(f"Unavailable iOS simulator UDID: {self._device_id}") + self._device_id = candidates[0]["udid"] + return candidates[0] + + async def _prepare_device(self, candidate: dict[str, Any]) -> None: + """Bring the resolved device to an interactable state before the session.""" + if candidate.get("state") == "Shutdown": + await run_xcrun("simctl", "boot", self._device_id) + clear_ios_simulator_cache() + await run_xcrun("simctl", "bootstatus", self._device_id, "-b", timeout=180.0) + + def _session_label(self) -> str: + return f"Artemis Simulator {uuid4().hex[:8]}" + + def _validate_session_device(self, session: dict[str, Any]) -> None: + if ( + not session.get("deviceIsSimulator") + or str(session.get("deviceUUID") or "").lower() != self._device_id.lower() + ): + raise RuntimeError("Xcode selected a different device; refusing to interact.") + + async def _start_interaction_session(self) -> dict[str, Any]: + start_arguments = { + "deviceIdentifier": self._device_id, + "sessionIdentifier": self._session_label(), + } + try: + return await self._bridge.call("DeviceInteractionStartSession", start_arguments) + except XcodeApprovalRequiredError: + if self._workspace_path is None: + raise + if "XcodeOpenWorkspace" not in self._bridge.tools: + raise XcodeApprovalRequiredError( + "DeviceInteractionStartSession", + "Xcode's XcodeOpenWorkspace tool is unavailable; " + "approve Artemis's access in Xcode.", + ) + opened = await self._bridge.call( + "XcodeOpenWorkspace", {"path": str(self._workspace_path)} + ) + identifier = opened.get("workspaceIdentifier") + if not isinstance(identifier, str) or not identifier: + raise RuntimeError("XcodeOpenWorkspace returned no usable workspace identifier.") + return await self._bridge.call("DeviceInteractionStartSession", start_arguments) + + async def connect(self) -> None: + async with self._connect_lock: + if self._session_key: + if self._bridge.connected: + return + # The bridge retires itself on tool-call failures; release the + # orphaned native session before starting over. + await self.disconnect() + candidate = await self._resolve_device() + await self._prepare_device(candidate) + await self._bridge.start() + required = { + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", + } + if not required.issubset(self._bridge.tools): + await self._bridge.close() + raise RuntimeError( + "Xcode's native device interaction tools are unavailable. Select Xcode 27 and approve Artemis's access in Xcode." + ) + try: + session = await self._start_interaction_session() + self._session_key = session.get("interactionSessionKey") + if not self._session_key: + raise RuntimeError("Xcode did not return a device interaction session key.") + self._validate_session_device(session) + await self.get_screen_data(skip_settling=True) + except ( + OSError, + ValueError, + RuntimeError, + TimeoutError, + asyncio.CancelledError, + ) as error: + if isinstance(error, XcodeApprovalRequiredError): + error.workspace_path = self._workspace_path + try: + await self.disconnect() + except (Exception, asyncio.CancelledError) as cleanup_error: + logger.warning( + f"Could not release the Xcode session after a connection failure: {cleanup_error}", + ) + raise + + async def disconnect(self) -> None: + try: + if self._recorder is not None: + session = self._recorder.session + if session is not None: + if session.is_active: + try: + await self._recorder.stop() + except Exception as exc: + logger.error( + f"iOS recording finalization failed during disconnect: {exc}" + ) + if get_active_session(self._device_id) is session: + remove_active_session(self._device_id) + finally: + async with self._operation_lock: + key, self._session_key = self._session_key, None + try: + if key: + # A timeout retires the old connection; cleanup may + # create a fresh bridge solely to close the known + # native session. + if not self._bridge.connected: + await self._bridge.start() + await self._bridge.call( + "DeviceInteractionEndSession", + {"interactionSessionKey": key}, + ) + finally: + self._scale = None + await self._bridge.close() + + async def _synthesize(self, command: str = "", activation: str | None = None) -> dict[str, Any]: + self._require_connected() + arguments = {"interactSessionKey": self._session_key, "interactionCommand": command} + if activation: + arguments["activationBundleId"] = activation + return await self._bridge.call("DeviceInteractionSynthesize", arguments) + + def _require_connected(self) -> None: + if not self._session_key or self._device_id.strip().lower() == BOOTED_SIMULATOR_ID: + raise RuntimeError("Connect the iOS simulator driver before interacting.") + + async def _capture(self) -> ScreenData: + result = await self._synthesize() + image_path = result.get("screenshotPath") + if not image_path: + raise RuntimeError("Xcode returned no screenshot; no observation is available.") + screenshot = await asyncio.to_thread(Path(image_path).read_bytes) + with Image.open(BytesIO(screenshot)) as image: + self._width, self._height = image.size + hierarchy_path = result.get("hierarchyPath") + if not hierarchy_path: + self._scale = None + raise RuntimeError( + "Xcode returned no accessibility hierarchy. Recapture after the UI settles." + ) + hierarchy = await asyncio.to_thread(Path(hierarchy_path).read_text, encoding="utf-8") + self._scale = None + elements, self._scale = parse_hierarchy(hierarchy, self._width, self._height) + return ScreenData( + screenshot_bytes=screenshot, + screenshot_base64=base64.b64encode(screenshot).decode("ascii"), + ui_elements=elements, + width=self._width, + height=self._height, + platform="ios", + ) + + async def get_screen_data(self, skip_settling: bool = False) -> ScreenData: + # Xcode captures after animations settle; an extra Android delay is unnecessary. + async with self._operation_lock: + return await self._capture() + + async def _capture_unchanged(self, action: str) -> ScreenData: + """Capture, refusing to continue when the screen geometry changed.""" + previous_size = self.screen_size + data = await self._capture() + if self.screen_size != previous_size: + raise ValueError( + f"The iOS screen changed orientation or size. Observe it again before {action}." + ) + return data + + def _scaled_point(self, x: int, y: int) -> tuple[float, float]: + """Convert screenshot pixels to the driver's logical point space.""" + if self._scale is None: + raise RuntimeError("Capture an iOS screen before coordinate interaction.") + if not 0 <= x < self._width or not 0 <= y < self._height: + raise ValueError("iOS input coordinates are outside the current screenshot.") + return x / self._scale[0], y / self._scale[1] + + def _point(self, x: int, y: int) -> str: + px, py = self._scaled_point(x, y) + return f"{px:.4f} {py:.4f}" + + @staticmethod + def _direction_points(direction: str, w: int, h: int) -> tuple[int, int, int, int]: + points = { + "up": (w // 2, h * 3 // 4, w // 2, h // 4), + "down": (w // 2, h // 4, w // 2, h * 3 // 4), + "left": (w * 3 // 4, h // 2, w // 4, h // 2), + "right": (w // 4, h // 2, w * 3 // 4, h // 2), + } + return points[direction] + + @staticmethod + def _activation_for(data: ScreenData, x: int, y: int) -> str | None: + """Native overlapping-app elements require activation before input.""" + candidates = [] + for element in data.ui_elements: + bundle = element.get("activation_bundle_id") + bounds = element.get("parsed_bounds") + if not bundle or not isinstance(bounds, dict): + continue + if bounds["left"] <= x < bounds["right"] and bounds["top"] <= y < bounds["bottom"]: + area = (bounds["right"] - bounds["left"]) * (bounds["bottom"] - bounds["top"]) + candidates.append((area, bundle)) + if not candidates: + return None + minimum = min(area for area, _ in candidates) + bundles = {bundle for area, bundle in candidates if area == minimum} + if len(bundles) != 1: + raise ValueError( + "The target overlaps multiple iOS applications; activate the intended app first." + ) + return bundles.pop() + + async def tap( + self, x: int, y: int, duration_ms: int = 100, times: int = 1, delay_ms: int = 100 + ) -> bool: + if times < 1 or duration_ms < 0 or delay_ms < 0: + raise ValueError("Tap count must be positive and durations nonnegative.") + async with self._operation_lock: + data = await self._capture_unchanged("tapping") + command = f"t {self._point(x, y)} {duration_ms / 1000:.3f}" + for index in range(times): + await self._synthesize(command, activation=self._activation_for(data, x, y)) + if index < times - 1: + await asyncio.sleep(delay_ms / 1000) + return True + + async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool: + return await self.tap(x, y, duration_ms=duration_ms) + + async def swipe( + self, start_x: int, start_y: int, end_x: int, end_y: int, duration_ms: int = 800 + ) -> bool: + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + data = await self._capture_unchanged("swiping") + await self._synthesize( + f"t {self._point(start_x, start_y)} f {self._point(end_x, end_y)} {duration_ms / 1000:.3f}", + activation=self._activation_for(data, start_x, start_y), + ) + return True + + async def swipe_direction( + self, + direction: SwipeDirection | Literal["up", "down", "left", "right"], + duration_ms: int = 800, + ) -> bool: + direction = SwipeDirection(direction).value + if duration_ms <= 0: + raise ValueError("Swipe duration must be positive.") + async with self._operation_lock: + # swipe_direction is rotation-tolerant by design: recapture and + # compute from the *current* size rather than refusing like tap. + data = await self._capture() + sx, sy, ex, ey = self._direction_points(direction, *self.screen_size) + await self._synthesize( + f"t {self._point(sx, sy)} f {self._point(ex, ey)} {duration_ms / 1000:.3f}", + activation=self._activation_for(data, sx, sy), + ) + return True + + async def input_text(self, text: str, clear_existing: bool = True) -> bool: + if clear_existing: + raise NotImplementedError( + "Xcode has no verified replace-text operation. Use clear_existing=False to append, or clear the field through its UI." + ) + # Encode every code point: literal native escape sequences cannot become + # unintended control characters, and newlines/Unicode retain their value. + escaped = "".join(f"\\u{{{ord(char):04X}}}" for char in text) + async with self._operation_lock: + await self._synthesize(f"sender keyboard kbd {escaped}") + return True + + async def press_key(self, key: KeyCode | str | int) -> bool: + key = key.value if isinstance(key, KeyCode) else str(key).lower() + commands = { + "home": "b h", + "power": "b p", + "volume_up": "b u", + "volume_down": "b d", + "app_switch": "b h b h", + "enter": r"sender keyboard kbd \u{000A}", + } + if key not in commands: + raise NotImplementedError(f"Key {key!r} is not supported by the native iOS driver.") + async with self._operation_lock: + await self._synthesize(commands[key]) + return True + + async def launch_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "launch", self._device_id, package_name) + return True + + async def stop_app(self, package_name: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "terminate", self._device_id, package_name) + return True + + async def install_app(self, app_path: Path) -> str: + self._require_connected() + path = app_path.expanduser().resolve() + if path.suffix != ".app" or not path.is_dir(): + raise ValueError("iOS installation requires a simulator-built .app directory.") + with (path / "Info.plist").open("rb") as stream: + bundle = plistlib.load(stream).get("CFBundleIdentifier") + if not isinstance(bundle, str) or not bundle: + raise ValueError("The .app has no CFBundleIdentifier in Info.plist.") + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "install", self._device_id, str(path), timeout=120) + return bundle + + async def list_apps(self) -> dict[str, str]: + async with self._operation_lock: + self._require_connected() + data = await _parse_listapps_output( + await run_xcrun("simctl", "listapps", self._device_id) + ) + return { + bundle: info.get("CFBundleDisplayName") or info.get("CFBundleName") or bundle + for bundle, info in data.items() + } + + async def open_url(self, url: str) -> bool: + async with self._operation_lock: + self._require_connected() + await run_xcrun("simctl", "openurl", self._device_id, url) + return True + + async def get_current_package(self) -> str | None: + async with self._operation_lock: + result = await self._synthesize() + path = result.get("hierarchyPath") + if not path: + return None + hierarchy = await asyncio.to_thread(Path(path).read_text, encoding="utf-8") + return application_bundle(hierarchy) + + async def find_element( + self, query: ElementQuery, screen_data: ScreenData | None = None + ) -> tuple[dict[str, Any] | None, list[int] | None, str | None]: + element, center, error = await super().find_element(query, screen_data) + if element is not None: + center = element.get("hit_point", center) + return element, center, error + + async def tap_element( + self, query: ElementQuery, long_press: bool = False, duration_ms: int = 1000 + ) -> bool: + async with self._operation_lock: + data = await self._capture() + element, center, error = await self.find_element(query, data) + if error or element is None or center is None: + return False + await self._synthesize( + f"t {self._point(*center)} {duration_ms / 1000 if long_press else 0.1:.3f}", + activation=element.get("activation_bundle_id"), + ) + return True + + async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str: + raise NotImplementedError("Android shell commands are unavailable on iOS Simulator.") + + # --- Recording (native segmented capture) --- + + def _new_recorder(self) -> IosScreenRecorder | PhysicalIosRecorder: + """Recorder implementation for this driver (sim: simctl recordVideo).""" + return IosScreenRecorder(self._device_id) + + @property + def recording_session(self) -> IosRecordingSession | None: + """Latest recording session, retained for error reporting after stop.""" + if self._recorder is None: + return None + return self._recorder.session + + async def start_video_recording( + self, + output_dir: Path | None = None, + max_duration_seconds: int = DEFAULT_MAX_DURATION_SECONDS, + ) -> None: + self._require_connected() + if not self._device_id: + raise RuntimeError("iOS recording requires a pinned device UDID") + if self._recorder is None: + self._recorder = self._new_recorder() + await self._recorder.start(output_dir, max_duration_seconds) + + async def seal_recording_segment(self, through_time: float | None = None) -> None: + """Seal the current segment so its final MP4 can be read safely.""" + if self._recorder is not None: + await self._recorder.seal(through_time) + + async def stop_video_recording(self) -> str | None: + if self._recorder is None: + return None + path = await self._recorder.stop() + return str(path) if path is not None else None diff --git a/artemis/interfaces/cli/commands/batch.py b/artemis/interfaces/cli/commands/batch.py index 225ba98f..7215ecee 100644 --- a/artemis/interfaces/cli/commands/batch.py +++ b/artemis/interfaces/cli/commands/batch.py @@ -21,6 +21,7 @@ from typing import Annotated from artemis.config import initialize_llm_config +from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.sdk import Agent from artemis.sdk.builders import Builders from third_party.mobile_use.sdk.types.task import AgentProfile @@ -38,6 +39,9 @@ async def run_batch_tasks( delay_seconds: float = 5.0, verification_level: str | None = None, explorer_pro_mode: str | None = None, + platform: str = "android", + device_serial: str | None = None, + ios_workspace: str | None = None, ) -> None: """Executes a list of automation tasks sequentially. @@ -49,6 +53,9 @@ async def run_batch_tasks( 'strict') for the Pro profile; ignored by Flash. explorer_pro_mode: Explorer tier ('flash', 'pro', 'ultra') behind ``ask_explorer`` under the Pro profile; ignored by Flash. + platform: 'android' (default) or 'ios' (local simulator or paired device). + device_serial: Android serial or iOS device UDID for all tasks. + ios_workspace: Xcode project/workspace for first-run iOS approval. """ if not os.environ.get("ARTEMIS_TASK_INGRESS"): os.environ["ARTEMIS_TASK_INGRESS"] = "cli" @@ -59,6 +66,15 @@ async def run_batch_tasks( config_builder.with_verification_level(verification_level) if explorer_pro_mode is not None: config_builder.with_explorer(pro_mode=explorer_pro_mode) + if normalize_device_platform(platform, strict=False) == "ios": + config_builder.for_ios_device( + device_id=device_serial or "booted", + workspace_path=ios_workspace, + ) + elif device_serial: + from artemis.context import DevicePlatform + + config_builder.for_device(DevicePlatform.ANDROID, device_serial) config = config_builder.build() agent = Agent(config=config) @@ -162,8 +178,44 @@ def batch_command( help="Explorer tier behind ask_explorer under the Pro profile ('flash', 'pro', 'ultra').", ), ] = None, + platform: Annotated[ + str, + typer.Option( + "--platform", + help="Target mobile platform: 'android' (default) or 'ios' (local simulator or paired device).", + ), + ] = "android", + device_serial: Annotated[ + str | None, + typer.Option( + "--device-serial", + help="Android serial or iOS device UDID (simulator or paired physical device) for every task in the batch.", + ), + ] = None, + ios_workspace: Annotated[ + Path | None, + typer.Option( + "--ios-workspace", + exists=True, + dir_okay=True, + file_okay=False, + resolve_path=True, + help="Existing Xcode project/workspace to request iOS first-run agent approval.", + ), + ] = None, ) -> None: """Execute multiple automation tasks in sequence.""" + try: + platform = normalize_device_platform(platform) + except ValueError as exc: + typer.secho( + f"Error: --platform must be 'android' or 'ios' (got '{platform}').", + fg=typer.colors.RED, + ) + raise typer.Exit(1) from exc + if ios_workspace is not None and platform != "ios": + typer.secho("Error: --ios-workspace requires --platform ios.", fg=typer.colors.RED) + raise typer.Exit(1) task_list: list[str] = [] if tasks_file: @@ -222,8 +274,11 @@ def batch_command( resp = submit_batch_to_daemon( task_list, profile=profile, + device_serial=device_serial, verification_level=verification_level, explorer_mode=explorer_pro_mode, + platform=platform, + ios_workspace=str(ios_workspace) if ios_workspace else None, base_url=base_url, ) if resp and resp.get("tasks"): @@ -274,5 +329,8 @@ def batch_command( delay_seconds=delay, verification_level=verification_level, explorer_pro_mode=explorer_pro_mode, + platform=platform, + device_serial=device_serial, + ios_workspace=str(ios_workspace) if ios_workspace else None, ) ) diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 89dca25a..5d0ab5c1 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -16,10 +16,14 @@ import asyncio import os +from pathlib import Path from typing import Annotated from langchain_core.callbacks.base import Callbacks from artemis.config import checker_overrides_for_level, settings +from artemis.context import DevicePlatform +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError, xcode_approval_guidance +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.utils.startup_progress import publish_startup_progress from third_party.mobile_use.main import ( GoalArgument, @@ -36,6 +40,7 @@ import signal from rich.console import Console from rich.panel import Panel +from rich.text import Text import typer logger = get_logger(__name__) @@ -63,6 +68,8 @@ async def execute_task( explorer_flash_mode: str | None = None, explorer_pro_mode: str | None = None, verification_level: str | None = None, + platform: DevicePlatform = DevicePlatform.ANDROID, + ios_workspace_path: Path | None = None, ) -> None: """Executes a single mobile automation task end-to-end. @@ -84,6 +91,11 @@ async def execute_task( verification_level: Coarse Checker preset ('off', 'final', 'checkpoints', 'strict'); applied before the explicit ``enable_checker`` switch. """ + platform = DevicePlatform(platform) + if ios_workspace_path is not None and platform != DevicePlatform.IOS: + raise ValueError("--ios-workspace requires --platform ios.") + if platform == DevicePlatform.IOS and os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise ValueError("iOS support is local only; cloud mode targets Android.") effective_sid = ( session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") ) @@ -134,10 +146,14 @@ async def execute_task( pro_mode=explorer_pro_mode, ) - target_serial = ( - device_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") - ) - if not target_serial: + target_serial = device_serial + if platform == DevicePlatform.IOS: + target_serial = target_serial or BOOTED_SIMULATOR_ID + else: + target_serial = ( + target_serial or settings.ADB_DEVICE_SERIAL or os.environ.get("ADB_DEVICE_SERIAL") + ) + if not target_serial and platform == DevicePlatform.ANDROID: try: from artemis.runtime import device_pool @@ -146,9 +162,10 @@ async def execute_task( target_serial = None if target_serial: - from artemis.context import DevicePlatform + config.for_device(platform, target_serial) - config.for_device(DevicePlatform.ANDROID, target_serial) + if ios_workspace_path is not None: + config.with_ios_workspace(ios_workspace_path) if graph_config_callbacks: config.with_graph_config_callbacks(graph_config_callbacks) @@ -176,6 +193,28 @@ def run_command( help="Execution profile ('flash' for fast reactive, 'pro' for full graph).", ), ] = "pro", + platform: Annotated[ + DevicePlatform, + typer.Option( + "--platform", + metavar="PLATFORM", + rich_help_panel="Platform", + help="Target mobile platform: Android (default) or a local iOS device.", + ), + ] = DevicePlatform.ANDROID, + ios_workspace: Annotated[ + Path | None, + typer.Option( + "--ios-workspace", + metavar="PATH", + rich_help_panel="Platform", + exists=True, + dir_okay=True, + file_okay=False, + resolve_path=True, + help="Existing Xcode project/workspace to request iOS first-run agent approval.", + ), + ] = None, locked_app_package: Annotated[ str | None, typer.Option( @@ -192,7 +231,7 @@ def run_command( str | None, typer.Option( "--app-path", - help="Local APK path to install before starting the task.", + help="Local Android APK or signed iOS .app/.ipa to install before the task.", ), ] = None, enable_planner_validation: Annotated[ @@ -278,7 +317,7 @@ def run_command( typer.Option( "--device-serial", "-s", - help="Target specific Android device by serial number (e.g. emulator-5554).", + help="Android serial or iOS device UDID (simulator or paired physical device); iOS defaults to the single 'booted' simulator.", ), ] = None, session_id: Annotated[ @@ -296,8 +335,7 @@ def run_command( ), ] = False, ) -> None: - """Run an autonomous UI automation task on the connected Android device.""" - ensure_video_recording_available(with_video_recording_tools) + """Run an autonomous UI automation task on an Android or iOS device.""" console = Console() @@ -307,6 +345,16 @@ def run_command( ) is_standalone = standalone or os.environ.get("ARTEMIS_STANDALONE") == "1" + if ios_workspace is not None and platform != DevicePlatform.IOS: + raise typer.BadParameter("--ios-workspace requires --platform ios.") + + if platform == DevicePlatform.IOS: + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise typer.BadParameter("iOS support is local only; cloud mode targets Android.") + if locked_app_package: + raise typer.BadParameter("--locked-app is unavailable for iOS tasks.") + ensure_video_recording_available(with_video_recording_tools) + # All platforms route through unified Artemis Daemon unless specifically configured as standalone if not is_worker and not is_standalone: try: @@ -335,6 +383,10 @@ def run_command( app_path=app_path, session_id=target_sid, ingress="cli", + platform=platform.value, + ios_workspace=str(ios_workspace) if ios_workspace else None, + verification_level=verification_level, + explorer_mode=explorer_pro_mode, base_url=base_url, ) if resp and resp.get("tasks"): @@ -390,7 +442,8 @@ def on_status(sess_info): f"[yellow]Daemon routing notice: {exc}. Falling back to local execution...[/yellow]" ) - display_local_device_status(console, host=settings.ADB_HOST, port=settings.ADB_PORT) + if platform == DevicePlatform.ANDROID: + display_local_device_status(console, host=settings.ADB_HOST, port=settings.ADB_PORT) cancelled = False original_sigterm = None @@ -427,6 +480,8 @@ def on_status(sess_info): explorer_flash_mode=explorer_flash_mode, explorer_pro_mode=explorer_pro_mode, verification_level=verification_level, + platform=platform, + ios_workspace_path=ios_workspace, ) ) except (KeyboardInterrupt, asyncio.CancelledError): @@ -434,9 +489,20 @@ def on_status(sess_info): # this worker to stop through a cancel marker and the Agent cancelled # its own task so the recording and trace could be finalized. cancelled = True + except XcodeApprovalRequiredError as e: + console.print() + console.print( + Panel( + Text(f"{e}\n\n{xcode_approval_guidance(workspace_path=ios_workspace)}"), + title="Xcode Approval Required", + expand=False, + ) + ) + console.print() + raise SystemExit(2) except Exception as e: err_msg = str(e) - if "API_KEY" in err_msg or "requires" in err_msg: + if "API_KEY" in err_msg or "api key" in err_msg.lower(): console.print() console.print( Panel( diff --git a/artemis/mcp/action_names.py b/artemis/mcp/action_names.py index 78cef0a5..88ca45ad 100644 --- a/artemis/mcp/action_names.py +++ b/artemis/mcp/action_names.py @@ -151,7 +151,19 @@ def to_canonical_call( # bare key word for the common keys. Anything else keeps its original spelling # so the driver can forward it verbatim (arbitrary KEYCODE_* / numeric codes). bare = keycode.removeprefix("KEYCODE_").lower() if keycode else "" - known = {"home", "back", "enter", "delete", "tab", "search", "menu", "app_switch"} + known = { + "home", + "back", + "enter", + "delete", + "tab", + "search", + "menu", + "app_switch", + "power", + "volume_up", + "volume_down", + } return "press_key", {"key": bare if bare in known else keycode} if verb == "back": diff --git a/artemis/mcp/action_server.py b/artemis/mcp/action_server.py index be32b95c..f810d132 100644 --- a/artemis/mcp/action_server.py +++ b/artemis/mcp/action_server.py @@ -46,6 +46,7 @@ from artemis.mcp.actuators.base import Actuator from artemis.mcp.observation import observe as observe_impl from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_mime_type logger = get_logger(__name__) @@ -112,7 +113,7 @@ async def observe_screen( ImageContent( type="image", data=_b64.b64encode(img_bytes).decode("utf-8"), - mimeType="image/jpeg", + mimeType=image_mime_type(img_bytes), ) ) return CallToolResult( diff --git a/artemis/mcp/action_session.py b/artemis/mcp/action_session.py index 06e512ca..f5f58ca0 100644 --- a/artemis/mcp/action_session.py +++ b/artemis/mcp/action_session.py @@ -233,7 +233,14 @@ async def get_action_session(ctx: Any, actuator: Any = None) -> ActionSession: from artemis.mcp.action_server import build_action_server from artemis.mcp.actuators.adb import AdbActuator - actuator = actuator or getattr(ctx, "actuator", None) or AdbActuator(ctx) + actuator = actuator or getattr(ctx, "actuator", None) + if actuator is None: + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + from artemis.mcp.actuators.ios import IosActuator + + actuator = IosActuator(ctx) + else: + actuator = AdbActuator(ctx) server = build_action_server(actuator) session = ActionSession(server) await session.start() diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 1ea69a11..d52c5349 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -565,13 +565,15 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action operator=OperatorDialect( description=( "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK," - " HOME, APP_SWITCH)." + " HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN)." ), params=( ParamSpec( "key", - Literal["ENTER", "BACK", "HOME", "APP_SWITCH"], - "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", + Literal[ + "ENTER", "BACK", "HOME", "APP_SWITCH", "POWER", "VOLUME_UP", "VOLUME_DOWN" + ], + "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", ), ), ), diff --git a/artemis/mcp/actuators/ios.py b/artemis/mcp/actuators/ios.py new file mode 100644 index 00000000..a650b525 --- /dev/null +++ b/artemis/mcp/actuators/ios.py @@ -0,0 +1,79 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS actuator sharing Artemis's normalized action contract.""" + +from artemis.mcp.action_types import ActionCode, ActionResult +from artemis.mcp.action_manifest import DEVICE_ACTIONS +from artemis.mcp.actuators.adb import AdbActuator, ensure_focus_at_coords + + +class IosActuator(AdbActuator): + """Reuse controller actions while keeping Android shell conventions out of iOS.""" + + def capabilities(self) -> frozenset[str]: + return DEVICE_ACTIONS - {"erase_one_char", "focus_and_clear_text"} + + def _dims(self) -> tuple[int, int]: + # The simulator may rotate after SDK initialization. Its most recent + # screenshot dimensions own the coordinate space for the next action. + return self.controller.driver.screen_size + + async def input_text( + self, text: str, target: tuple[int, int] | None = None, clear_exist: bool = True + ) -> ActionResult: + if clear_exist: + return ActionResult.failure( + "input_text", + "Whole-field clearing is unavailable on iOS. Focus an empty field or place " + "the cursor manually, then set clear_exist=false.", + code=ActionCode.UNSUPPORTED, + ) + if target: + error = await ensure_focus_at_coords(self.controller, *self._to_px(*target)) + if error: + return ActionResult.failure("input_text", error) + success = await self.controller.type_text(text, clear_existing=clear_exist) + if not success: + return ActionResult.failure("input_text", "Failed to type text on the iOS device.") + return ActionResult.success("input_text", f"Typed '{text}'.") + + async def press_key(self, key: str) -> ActionResult: + if await self.controller.press_key(key): + return ActionResult.success("press_key", f"Pressed key '{key}'.") + return ActionResult.failure( + "press_key", f"iOS key '{key}' is unsupported or failed.", code=ActionCode.UNSUPPORTED + ) + + async def manage_app(self, action: str, app_name: str) -> ActionResult: + if action.lower() not in {"launch", "stop"}: + return ActionResult.failure( + "manage_app", f"Invalid manage_app action: {action}", code=ActionCode.INVALID_ARGS + ) + from artemis.tools.mobile.launch_app import find_package + + bundle_id = await find_package(self.ctx, app_name, use_fallback=False) + if not bundle_id: + return ActionResult.failure( + "manage_app", + f"Installed iOS app not found: {app_name}", + code=ActionCode.PACKAGE_NOT_FOUND, + ) + if action.lower() == "launch": + success = await self.controller.launch_app(bundle_id) + else: + success = await self.controller.terminate_app(bundle_id) + if not success: + return ActionResult.failure("manage_app", f"Failed to {action} iOS app '{bundle_id}'.") + return ActionResult.success("manage_app", f"Dispatched {action} for iOS app '{bundle_id}'.") diff --git a/artemis/mcp/adb_server.py b/artemis/mcp/adb_server.py index 14da4d5c..c6e82354 100644 --- a/artemis/mcp/adb_server.py +++ b/artemis/mcp/adb_server.py @@ -44,8 +44,15 @@ from artemis.context import ArtemisContext, DeviceContext, DevicePlatform from artemis.controllers.unified_controller import UnifiedMobileController from artemis.platform import platform +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID +from artemis.runtime.adb_endpoint import normalize_device_platform from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries +# Placeholder iOS screen metrics for the lazy context — the driver replaces +# them with the real display size when it connects. +_LAZY_IOS_PLACEHOLDER_WIDTH = 1206 +_LAZY_IOS_PLACEHOLDER_HEIGHT = 2622 + def configure_stdio_mode() -> None: """Applies process-wide settings required when serving MCP over stdio. @@ -115,12 +122,42 @@ def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: _CONTROLLERS: dict[str, Any] = {} -def _get_controller(device_serial: str | None = None): - """Lazy-load device controller on-demand, caching per device serial.""" +def _get_controller(device_serial: str | None = None, target_platform: str | None = None): + """Lazy-load device controller on-demand, caching per device serial. + + ``target_platform="ios"`` builds a native iOS controller: the serial is a + simulator UDID (or "booted") or a paired physical device UDID, the context carries + ``DevicePlatform.IOS`` so the factory selects the Xcode driver, and the + cache key is namespaced so a UDID can never collide with an Android + serial. Callers must ``await driver.connect()`` before interacting -- + the native session opens lazily on first use. + """ global _GLOBAL_CONTROLLER, _CONTROLLERS - target_serial = ( - device_serial or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL") - ) + is_ios_target = normalize_device_platform(target_platform, strict=False) == "ios" + # ADB_DEVICE_SERIAL is Android-only and must never leak into an iOS target. + target_serial = device_serial or os.environ.get("ARTEMIS_DEVICE_ID") + if not is_ios_target: + target_serial = target_serial or os.environ.get("ADB_DEVICE_SERIAL") + if is_ios_target: + cache_key = f"ios:{target_serial or BOOTED_SIMULATOR_ID}" + if cache_key in _CONTROLLERS: + return _CONTROLLERS[cache_key] + logger.info("Initializing lazy iOS device controller...") + ctx = ArtemisContext( + trace_id="mcp-session", + device=DeviceContext( + host_platform=platform.os_type.name, + mobile_platform=DevicePlatform.IOS, + device_id=target_serial or BOOTED_SIMULATOR_ID, + # Placeholder until the driver reports real screen metrics. + device_width=_LAZY_IOS_PLACEHOLDER_WIDTH, + device_height=_LAZY_IOS_PLACEHOLDER_HEIGHT, + ), + ) + controller = UnifiedMobileController(ctx=ctx) + _CONTROLLERS[cache_key] = controller + return controller + if target_serial and target_serial in _CONTROLLERS: return _CONTROLLERS[target_serial] if not target_serial and _GLOBAL_CONTROLLER is not None: diff --git a/artemis/runtime/__init__.py b/artemis/runtime/__init__.py index 9821beec..08106288 100644 --- a/artemis/runtime/__init__.py +++ b/artemis/runtime/__init__.py @@ -21,13 +21,16 @@ ) from artemis.runtime.adb_endpoint import ( ADB_ENDPOINT_ID_ENV, + IOS_LOCK_SCOPE, AdbEndpoint, AdbSession, AdbTarget, InvalidAdbEndpoint, + IosTarget, current_adb_endpoint, ) from artemis.runtime.device_pool import DevicePool, DeviceStatus, device_pool +from artemis.runtime.ios_device_pool import IosDevicePool, ios_device_pool from artemis.runtime.process_probe import pid_is_alive from artemis.runtime.daemon_client import ( ensure_daemon_running, @@ -66,6 +69,7 @@ __all__ = [ "ADB_ENDPOINT_ID_ENV", + "IOS_LOCK_SCOPE", "AdbEndpoint", "AdbSession", "AdbTarget", @@ -75,6 +79,8 @@ "DevicePool", "DeviceStatus", "InvalidAdbEndpoint", + "IosDevicePool", + "IosTarget", "clear_cancel_request", "clear_server_info", "current_adb_endpoint", @@ -87,6 +93,7 @@ "get_server_status", "is_artemis_daemon", "is_daemon_running", + "ios_device_pool", "is_cancel_requested", "is_port_in_use", "pid_is_alive", diff --git a/artemis/runtime/adb_endpoint.py b/artemis/runtime/adb_endpoint.py index 9729f96a..029018b2 100644 --- a/artemis/runtime/adb_endpoint.py +++ b/artemis/runtime/adb_endpoint.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Immutable ADB endpoint and task target primitives. +"""Immutable device endpoint and task target primitives. -The selected endpoint is a user preference. An :class:`AdbTarget` is an -execution snapshot. Keeping those concepts separate prevents a queued or -running task from silently moving to another ADB server when the preference -changes in the Admin Console. +The selected endpoint is a user preference. An :class:`AdbTarget` or +:class:`IosTarget` is an execution snapshot. Keeping those concepts separate +prevents a queued or running task from silently moving to another ADB server +or simulator when the preference changes in the Admin Console. """ from __future__ import annotations @@ -37,11 +37,65 @@ _LOCAL_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"}) _SAFE_HOST_PATTERN = re.compile(r"^[A-Za-z0-9._:\-\[\]]+$") +# User-facing platform vocabulary shared by CLI, console, daemon, and MCP. +SUPPORTED_PLATFORMS: tuple[str, ...] = ("android", "ios") +DEFAULT_PLATFORM = "android" + + +def normalize_device_platform( + value: object, + *, + default: str = DEFAULT_PLATFORM, + strict: bool = True, +) -> str: + """Canonicalize a platform token to ``"android"`` or ``"ios"``. + + Strips whitespace and case-folds. ``strict=True`` (entry points) raises + ``ValueError`` on anything else; ``strict=False`` (fail-open internals) + falls back to *default*. Surfaces adapt the ``ValueError`` to their own + error type (HTTP 400, Typer error, blocked verdict, ...). + """ + text = str(value).strip().lower() if value is not None else "" + if not text: + return default + if text in SUPPORTED_PLATFORMS: + return text + if strict: + raise ValueError(f"Unsupported platform '{value}'. Expected 'android' or 'ios'.") + return default + class InvalidAdbEndpoint(ValueError): """Raised when an ADB server endpoint is malformed.""" +def device_pool_for(platform: object): + """The device pool for a platform token (lazy imports avoid a cycle). + + Both pools expose the same admission surface (``validate_explicit_serial``, + ``validate_explicit_serial_async``, ``select_device_async``, + ``list_devices_async``). + """ + if normalize_device_platform(platform, strict=False) == "ios": + from artemis.runtime.ios_device_pool import ios_device_pool + + return ios_device_pool + from artemis.runtime.device_pool import device_pool + + return device_pool + + +def target_for_platform( + platform: object, + serial: str | None = None, + endpoint: AdbEndpoint | None = None, +) -> AdbTarget | IosTarget: + """Build the execution target for a platform; ``lock_scope`` comes with it.""" + if normalize_device_platform(platform, strict=False) == "ios": + return IosTarget(serial=serial) + return AdbTarget(endpoint=endpoint or current_adb_endpoint(), serial=serial) + + @dataclass(frozen=True, slots=True) class AdbEndpoint: """Network address of one ADB server.""" @@ -120,6 +174,10 @@ class AdbTarget: endpoint: AdbEndpoint serial: str | None = None + @property + def platform(self) -> str: + return "android" + @property def lock_scope(self) -> str: return self.endpoint.identity @@ -128,8 +186,60 @@ def lock_scope(self) -> str: def lock_key(self) -> str: return f"{self.lock_scope}/{self.serial or 'pending'}" + def apply_to_environment( + self, + environment: MutableMapping[str, str] | None = None, + ) -> MutableMapping[str, str]: + return self.endpoint.apply_to_environment(environment) + + def to_dict(self) -> dict[str, Any]: + return { + "platform": self.platform, + "endpoint": self.endpoint.to_dict(), + "serial": self.serial, + } + + +# Execution lock scope shared by every iOS simulator task. The device UDID +# distinguishes simulators inside the scope; "ios" keeps an iOS lock file +# namespaced away from any Android serial of the same text. +IOS_LOCK_SCOPE = "ios" + + +@dataclass(frozen=True, slots=True) +class IosTarget: + """An iOS device UDID (simulator or paired physical device).""" + + serial: str | None = None + + @property + def platform(self) -> str: + return "ios" + + @property + def lock_scope(self) -> str: + return IOS_LOCK_SCOPE + + @property + def lock_key(self) -> str: + return f"{self.lock_scope}/{self.serial or 'pending'}" + + def apply_to_environment( + self, + environment: MutableMapping[str, str] | None = None, + ) -> MutableMapping[str, str]: + target = environment if environment is not None else os.environ + # LOCK_SCOPE_ENV names the execution scope generically: the ADB + # endpoint identity for Android, the platform tag for iOS devices. + target[ADB_ENDPOINT_ID_ENV] = self.lock_scope + # An iOS worker never touches ADB: a stale serial must not leak in. + target.pop("ADB_DEVICE_SERIAL", None) + # The worker's device id comes from the queue target, not a parent env. + target.pop("ARTEMIS_DEVICE_ID", None) + return target + def to_dict(self) -> dict[str, Any]: - return {"endpoint": self.endpoint.to_dict(), "serial": self.serial} + return {"platform": self.platform, "serial": self.serial} class AdbSession: diff --git a/artemis/runtime/daemon_client.py b/artemis/runtime/daemon_client.py index 85471489..fae28ad6 100644 --- a/artemis/runtime/daemon_client.py +++ b/artemis/runtime/daemon_client.py @@ -221,6 +221,8 @@ def submit_task_to_daemon( conversation_id: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, base_url: str | None = None, timeout: float = 15.0, ) -> dict[str, Any] | None: @@ -230,6 +232,9 @@ def submit_task_to_daemon( ``explorer_mode`` ('flash' | 'pro' | 'ultra') are the Pro-profile tuning knobs of ``/api/run``; they are forwarded verbatim and ignored by Flash. + ``platform="ios"`` submits an iOS task: ``device_serial`` is a simulator + or physical UDID and ``ios_workspace`` optionally scopes Xcode approval. + Returns the response JSON dict if successfully enqueued, or None on error. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" @@ -246,6 +251,8 @@ def submit_task_to_daemon( "session_id": session_id, "ingress": ingress, "conversation_id": conversation_id, + "platform": platform, + "ios_workspace": ios_workspace, } try: @@ -345,13 +352,16 @@ def submit_batch_to_daemon( ingress: str = "cli", verification_level: str | None = None, explorer_mode: str | None = None, + platform: str | None = None, + ios_workspace: str | None = None, base_url: str | None = None, timeout: float = 15.0, ) -> dict[str, Any] | None: """Submit a batch of goals to the running Daemon. ``verification_level`` / ``explorer_mode`` apply to every goal of the batch - (see :func:`submit_task_to_daemon`). + (see :func:`submit_task_to_daemon`). ``platform="ios"`` targets an iOS + device; ``device_serial`` is then the device UDID. """ url = f"{base_url or f'http://{DEFAULT_DAEMON_HOST}:{DEFAULT_DAEMON_PORT}'}/api/run" payload = { @@ -361,6 +371,8 @@ def submit_batch_to_daemon( "ingress": ingress, "verification_level": verification_level, "explorer_mode": explorer_mode, + "platform": platform, + "ios_workspace": ios_workspace, } try: data = json.dumps(payload).encode("utf-8") diff --git a/artemis/runtime/device_lock.py b/artemis/runtime/device_lock.py index 9579d234..1b8e92f6 100644 --- a/artemis/runtime/device_lock.py +++ b/artemis/runtime/device_lock.py @@ -939,9 +939,15 @@ def annotate_active_owner( lock_dir = get_temp_dir("device-locks") if device_id is not None: clean_id = cls._normalize_device_id(device_id) - path = lock_dir / f"artemis-device-{clean_id}.lock" - if not path.exists(): - path = lock_dir / "artemis-global-device.lock" + candidates = [ + lock_dir / f"artemis-device-{clean_id}.lock", + *sorted(lock_dir.glob(f"artemis-device-*__{clean_id}.lock")), + lock_dir / "artemis-global-device.lock", + ] + path = next( + (candidate for candidate in candidates if candidate.exists()), + candidates[-1], + ) else: path = cls._find_lock_by_pid(os.getpid()) if path is None: @@ -959,6 +965,7 @@ def annotate_active_owner( "acquired_at": owner.acquired_at, "session_id": str(session_id), "ingress": ingress or owner.ingress or "sdk", + "lock_scope": owner.lock_scope, } return cls._atomic_write_json(path, payload) @@ -971,7 +978,12 @@ def cleanup_stale_locks(cls, device_id: str | None = None) -> int: return removed patterns = ( - [f"artemis-device-{cls._normalize_device_id(device_id)}.lock"] + [ + # Both the bare key and every scoped variant + # (e.g. artemis-device-ios__.lock) target this device. + f"artemis-device-{cls._normalize_device_id(device_id)}.lock", + f"artemis-device-*__{cls._normalize_device_id(device_id)}.lock", + ] if device_id else ["artemis-device-*.lock", "artemis-global-device.lock", "*.lock"] ) diff --git a/artemis/runtime/device_pool.py b/artemis/runtime/device_pool.py index c67aa538..388fe826 100644 --- a/artemis/runtime/device_pool.py +++ b/artemis/runtime/device_pool.py @@ -37,7 +37,7 @@ class DeviceStatus: """State and lock allocation metadata for a connected device.""" serial: str - state: str # "device", "offline", "unauthorized", etc. + state: str # "device", "offline", "unauthorized", "Shutdown", etc. model: str | None = None product: str | None = None is_emulator: bool = False @@ -46,6 +46,7 @@ class DeviceStatus: active_task_desc: str | None = None active_session_id: str | None = None acquired_at: str | None = None + platform: str = "android" def to_dict(self) -> dict[str, Any]: return { @@ -59,6 +60,7 @@ def to_dict(self) -> dict[str, Any]: "active_task_desc": self.active_task_desc, "active_session_id": self.active_session_id, "acquired_at": self.acquired_at, + "platform": self.platform, } diff --git a/artemis/runtime/ios_device_pool.py b/artemis/runtime/ios_device_pool.py new file mode 100644 index 00000000..3a6576e0 --- /dev/null +++ b/artemis/runtime/ios_device_pool.py @@ -0,0 +1,291 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS device pool: simulator and physical discovery, locks, selection. + +The iOS counterpart of :class:`~artemis.runtime.device_pool.DevicePool`. +Enumeration rides ``xcrun simctl list devices`` for simulators and +``xcrun devicectl list devices`` for paired physical hardware; lock ownership +shares the same :class:`DeviceExecutionLock` registry under the ``ios`` scope +so a simulator, a physical device, and an Android serial can never share a +lock identity. Auto-selection only ever picks simulators — physical hardware +requires an explicit serial. All methods fail open on missing Xcode tooling +or an enumeration error, mirroring the Android pool's admission contract. +""" + +from __future__ import annotations + +from artemis.drivers.ios.discovery import ( + BOOTED_SIMULATOR_ID, + is_physical_ios, + list_core_devices, + list_core_devices_sync, + list_ios_simulators, + list_ios_simulators_sync, + physical_ios_ready, +) +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.device_lock import DeviceExecutionLock +from artemis.runtime.device_pool import DeviceStatus +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +# simctl states the driver can use as-is ("device"/Booted) or boot itself +# ("Shutdown"). Anything else (e.g. "Creating") is rejected on explicit picks. +ACCEPTABLE_STATES = frozenset({"device", "Shutdown"}) + +# devicectl states that allow driving a physical device right now. +PHYSICAL_ACCEPTABLE_STATES = frozenset({"device"}) + + +def _ios_lock_owners() -> dict: + """Active lock owners scoped to iOS, keyed by normalized device id. + + Owner map keys are only scope-prefixed for multi-owner collisions, so + match by owner payload: an iOS lock always carries lock_scope="ios", + and an Android lock on the same text must never mark a device busy. + """ + return { + DeviceExecutionLock._normalize_device_id(o.device_id): o + for o in DeviceExecutionLock.get_active_owners().values() + if o and getattr(o, "lock_scope", None) == IOS_LOCK_SCOPE + } + + +def _owner_fields(owner) -> dict: + return { + "is_busy": owner is not None, + "active_pid": owner.pid if owner else None, + "active_task_desc": owner.description if owner else None, + "active_session_id": owner.session_id if owner else None, + "acquired_at": owner.acquired_at if owner else None, + } + + +def _match_statuses(devices: list[DeviceStatus], requested_serial: str) -> list[DeviceStatus]: + """All statuses matching a requested UDID or device name.""" + needle = str(requested_serial).lower() + return [d for d in devices if d.serial.lower() == needle or (d.model or "") == requested_serial] + + +def _match_or_reject( + devices: list[DeviceStatus], requested_serial: str +) -> tuple[DeviceStatus | None, str | None]: + """Return (match, None), (None, ambiguity rejection), or (None, None).""" + matches = _match_statuses(devices, requested_serial) + if len(matches) > 1: + serials = sorted(d.serial for d in matches) + return None, ( + f"{len(matches)} iOS devices match '{requested_serial}' " + f"({', '.join(serials)}); pass the device UDID instead." + ) + return (matches[0] if matches else None), None + + +def _booted_rejection(raw: list[dict] | None, requested_serial: str) -> str | None: + """``booted`` needs exactly one booted simulator to resolve safely.""" + if str(requested_serial).strip().lower() != BOOTED_SIMULATOR_ID: + return None + if raw is None: + # Enumeration cannot answer: fail open to the driver's own check. + return None + booted = [d for d in raw if d.get("state") == "Booted"] + if len(booted) == 1: + return None + return ( + f"'{BOOTED_SIMULATOR_ID}' requires exactly one booted iOS simulator " + f"({len(booted)} found); pass an explicit simulator UDID instead." + ) + + +def _state_rejection(match: DeviceStatus, requested_serial: str) -> str | None: + """The shared explicit-target state check for both validator variants.""" + acceptable = ACCEPTABLE_STATES if match.is_emulator else PHYSICAL_ACCEPTABLE_STATES + if match.state not in acceptable: + kind = "simulator" if match.is_emulator else "physical device" + return f"iOS {kind} '{requested_serial}' is in state '{match.state}' and cannot be used." + return None + + +class IosDevicePool: + """Discovers iOS simulators and physical devices and reports lock state.""" + + @staticmethod + def _build_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: + ios_owners = _ios_lock_owners() + devices: list[DeviceStatus] = [] + for device in raw_devices: + udid = str(device.get("udid") or "") + runtime = str(device.get("runtime") or "") + state = ( + "device" + if device.get("state") == "Booted" + else str(device.get("state") or "unknown") + ) + owner = ios_owners.get(DeviceExecutionLock._normalize_device_id(udid)) + devices.append( + DeviceStatus( + serial=udid, + state=state, + model=device.get("name"), + product=runtime.removeprefix("com.apple.CoreSimulator.SimRuntime.").replace( + "-", " " + ), + is_emulator=True, + platform="ios", + **_owner_fields(owner), + ) + ) + return devices + + @staticmethod + def _build_physical_statuses(raw_devices: list[dict]) -> list[DeviceStatus]: + """CoreDevice entries -> statuses; usable devices read as "device".""" + ios_owners = _ios_lock_owners() + devices: list[DeviceStatus] = [] + for device in raw_devices: + if not is_physical_ios(device): + continue + udid = str(device.get("udid") or "") + ready = physical_ios_ready(device) + state = ( + "device" + if ready + else "unpaired" + if device.get("pairing_state") != "paired" + else "offline" + ) + owner = ios_owners.get(DeviceExecutionLock._normalize_device_id(udid)) + devices.append( + DeviceStatus( + serial=udid, + state=state, + model=device.get("name"), + product=f"iOS {device.get('os_version') or '?'} physical", + is_emulator=False, + platform="ios", + **_owner_fields(owner), + ) + ) + return devices + + async def list_devices_async(self) -> list[DeviceStatus]: + """All simulators plus physical devices; enumeration failures degrade to [].""" + simulators = self._build_statuses(await list_ios_simulators() or []) + physical = self._build_physical_statuses(await list_core_devices() or []) + return simulators + physical + + async def try_list_devices_async(self) -> list[DeviceStatus] | None: + """Like list_devices_async, but ``None`` when enumeration cannot answer.""" + raw = await list_ios_simulators() + core = await list_core_devices() + if raw is None and core is None: + return None + return self._build_statuses(raw or []) + self._build_physical_statuses(core or []) + + async def validate_explicit_serial_async(self, requested_serial: str) -> str | None: + """Reject an explicitly requested UDID or device name, ``None`` when usable. + + Mirrors the Android validator: only a successful, non-empty + enumeration may reject, and an enumeration that cannot answer (None) + must not reject. ``Shutdown`` simulators are valid targets — the + driver boots them on connect. Physical devices must be paired and + connected (``"device"``); offline devices fail here. + """ + raw = await list_ios_simulators() + core = await list_core_devices() + if raw is None and core is None: + return None + if str(requested_serial).strip().lower() == BOOTED_SIMULATOR_ID: + # The documented selector: accepted by exactly one booted + # simulator, rejected otherwise, fail-open without enumeration. + return _booted_rejection(raw, requested_serial) + devices = self._build_statuses(raw or []) + self._build_physical_statuses(core or []) + match, ambiguity = _match_or_reject(devices, requested_serial) + if ambiguity is not None: + return ambiguity + if match is None: + # A failed enumeration cannot prove the serial is absent — + # defer to the driver's own resolution rather than reject. + if raw is None or core is None: + return None + return ( + f"iOS device '{requested_serial}' is not available. " + f"Known devices: {sorted(d.serial for d in devices)}." + ) + return _state_rejection(match, requested_serial) + + def validate_explicit_serial(self, requested_serial: str) -> str | None: + """Synchronous validator for non-async admission paths (MCP tools). + + Shares the async validator's matching and state rules so both + admission paths accept and reject the same targets. + """ + raw = list_ios_simulators_sync() + core = list_core_devices_sync() + if raw is None and core is None: + return None + if str(requested_serial).strip().lower() == BOOTED_SIMULATOR_ID: + return _booted_rejection(raw, requested_serial) + devices = self._build_statuses(raw or []) + self._build_physical_statuses(core or []) + match, ambiguity = _match_or_reject(devices, requested_serial) + if ambiguity is not None: + return ambiguity + if match is None: + # An enumeration that could not answer must not reject the serial. + if raw is None or core is None or not devices: + return None + return ( + f"iOS device '{requested_serial}' is not available. " + f"Known devices: {sorted(d.serial for d in devices)}." + ) + return _state_rejection(match, requested_serial) + + async def select_device_async(self, preferred_serial: str | None = None) -> str | None: + """Pick a simulator UDID for task execution. + + An explicit serial wins as-is. Otherwise prefer an idle booted + simulator, then any booted one, then the only available simulator + when none are booted (the driver boots it on connect). Physical + devices are never auto-selected — hardware always needs an explicit + serial. ``None`` when no simulator exists; strict ``booted`` + ambiguity rejection is the driver's job for users who literally + target "booted". + """ + if preferred_serial: + if str(preferred_serial).strip().lower() != BOOTED_SIMULATOR_ID: + return preferred_serial + # "booted" is a documented selector: pin it to the unique booted + # simulator when enumeration can answer. Zero/multiple booted sims + # keep the literal so the driver's resolution reports the + # explicit-UDID guidance; a failed enumeration fails open the same + # way. Physical hardware is still never auto-selected. + booted_devices = await self.try_list_devices_async() + if booted_devices is None: + return preferred_serial + booted_only = [d for d in booted_devices if d.is_emulator and d.state == "device"] + if len(booted_only) == 1: + return booted_only[0].serial + return preferred_serial + devices = await self.try_list_devices_async() or [] + simulators = [d for d in devices if d.is_emulator] + booted = [d for d in simulators if d.state == "device"] + if len(booted) == 1: + return booted[0].serial + if not booted and len(simulators) == 1: + return simulators[0].serial + idle_booted = [d for d in booted if not d.is_busy] + return idle_booted[0].serial if idle_booted else (booted[0].serial if booted else None) + + +ios_device_pool = IosDevicePool() diff --git a/artemis/runtime/ios_observation.py b/artemis/runtime/ios_observation.py new file mode 100644 index 00000000..60ce665a --- /dev/null +++ b/artemis/runtime/ios_observation.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared iOS observation lifecycle for one-shot native captures. + +``mobile_get_device_state`` and the device smoke check both need a screen +capture from an iOS driver that was constructed lazily — no lease, no native +session. This helper performs the complete, cancellation-safe lifecycle: +resolve the device identity, take the real iOS-scoped lease, connect, capture, +then disconnect and release in that order, even when the caller is cancelled. +""" + +import asyncio +from typing import Any + +from artemis.controllers.device_controller import ScreenDataResponse +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.device_lock import DeviceExecutionLock +from third_party.mobile_use.utils.logger import get_logger + +logger = get_logger(__name__) + +_DISCONNECT_ERRORS = (OSError, ValueError, RuntimeError, TimeoutError) + + +async def observe_ios_controller(controller: Any) -> ScreenDataResponse: + """Capture screen data from an unconnected iOS controller. + + Acquires the ``ios``-scoped device lease before any native setup, connects + the driver, captures, then always disconnects (when connect was attempted) + and releases the lease — including on cancellation, where cleanup is + shielded from the caller's CancelledError before it propagates. + """ + driver = controller._driver + resolved = await driver.resolve_device() + controller.ctx.device.device_id = resolved + lease = DeviceExecutionLock( + resolved, + description="Artemis iOS observation", + ingress="mcp", + concurrency_mode="per_device", + lock_scope=IOS_LOCK_SCOPE, + ) + connect_attempted = False + + async def _cleanup() -> None: + try: + if connect_attempted: + try: + await driver.disconnect() + except _DISCONNECT_ERRORS as exc: + logger.warning(f"iOS observation native disconnect failed: {exc}") + finally: + lease.release() + + acquire_task = asyncio.create_task(asyncio.to_thread(lease.acquire, blocking=False)) + try: + try: + # A cancelled waiter must still drain the acquisition thread: if the + # lease landed concurrently, skipping the drain would leak it. + await asyncio.shield(acquire_task) + except asyncio.CancelledError: + try: + await acquire_task + except asyncio.CancelledError: + raise + except (OSError, ValueError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"iOS observation lease drain failed while cancelling: {drain_error}") + raise + connect_attempted = True + await driver.connect() + data = await controller.get_screen_data() + device = controller.ctx.device + device.device_width = data.width + device.device_height = data.height + return data + finally: + cleanup = asyncio.ensure_future(_cleanup()) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + await cleanup + raise diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index c0c0da08..85cb25c4 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -13,16 +13,20 @@ # limitations under the License. import asyncio +import contextlib import inspect import os import re +import threading import uuid +from io import BytesIO from pathlib import Path from dotenv import load_dotenv from google import genai from langchain_google_genai import ChatGoogleGenerativeAI +from PIL import Image from artemis.clients.screen_client_factory import ( describe_backend, @@ -36,17 +40,19 @@ ) from artemis.context import ( ArtemisContext, + DeviceContext, DevicePlatform, ExecutionSetup, ) from artemis.data_engine.engine import DataEngine from artemis.graph.state import State -from artemis.runtime import trace_store +from artemis.runtime import DeviceExecutionLock, trace_store +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.runtime.cancel_requests import watch_for_cancel_request from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix from artemis.sdk.types.agent import AgentConfig from artemis.utils.startup_progress import publish_startup_progress -from third_party.mobile_use.sdk.types.exceptions import AgentError +from third_party.mobile_use.sdk.types.exceptions import AgentError, AgentNotInitializedError from third_party.mobile_use.sdk.types.task import Task from third_party.mobile_use.sdk.agent import AgentBase, TOutput from third_party.mobile_use.utils.logger import get_logger @@ -113,7 +119,7 @@ def __init__( updates = {} if target_dev: updates["device_id"] = target_dev - updates["device_platform"] = DevicePlatform.ANDROID + updates["device_platform"] = config.device_platform or DevicePlatform.ANDROID if concurrency_mode: updates["concurrency_mode"] = str(concurrency_mode).strip().lower() if max_concurrency is not None: @@ -125,6 +131,148 @@ def __init__( self._tmp_traces_dir = Path(settings.TRACES_PATH) self._initialized = False self._task_lock = asyncio.Lock() + self._ios_driver = None + self._adb_client = None + self._ui_adb_client = None + + async def _init_internal( + self, + api_key: str | None = None, + retry_count: int = 5, + retry_wait_seconds: int = 5, + ): + if self._config.device_platform != DevicePlatform.IOS: + return await super()._init_internal(api_key, retry_count, retry_wait_seconds) + if os.environ.get("ARTEMIS_CLOUD_MODE") == "1": + raise AgentError("iOS support is local only; cloud mode targets Android.") + if self._initialized: + return True + from artemis.drivers.factory import ios_driver_class + from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID + + publish_startup_progress( + "device_check", "Checking the iOS device", session_id=self._session_id + ) + # The picker runs simctl/devicectl subprocesses — keep them off the + # event loop so init timeouts and progress stays responsive. + driver_class = await asyncio.to_thread( + ios_driver_class, self._config.device_id or BOOTED_SIMULATOR_ID + ) + driver = driver_class( + device_id=self._config.device_id or BOOTED_SIMULATOR_ID, + workspace_path=getattr(self._config, "ios_workspace_path", None), + ) + self._ios_driver = driver + try: + # Resolve the device without booting it or opening a native UI + # session. Mutating setup waits for run_task's execution lease. + await driver.resolve_device() + except (OSError, ValueError, RuntimeError, TimeoutError, asyncio.CancelledError): + await driver.disconnect() + self._ios_driver = None + raise + width, height = driver.screen_size + self._device_context = DeviceContext( + host_platform="DARWIN", + mobile_platform=DevicePlatform.IOS, + device_id=driver.device_id, + device_width=width, + device_height=height, + ) + # Android read-only ADB probes have no iOS equivalent yet; native + # simctl recording is supported and honors the configured flag. + self._config = self._config.model_copy(update={"disable_device_probes": True}) + publish_startup_progress("device_ready", "iOS device selected", session_id=self._session_id) + asyncio.create_task(self._prewarm_llm_connections(api_key)) + self._initialized = True + return True + + @contextlib.asynccontextmanager + async def _ios_operation(self): + """Serialize one public iOS SDK call behind the device execution lease. + + Called from ``run_task`` itself (internal app installation), the helper + reuses the lease and native session the task already holds. Called + publicly, it acquires the same FIFO device lease a task would, then + opens a short-lived native session that is always closed on exit. + """ + driver = self._ios_driver + if not self._initialized or driver is None: + raise AgentNotInitializedError() + if asyncio.current_task() is getattr(self, "_current_task", None): + yield driver + return + async with self._task_lock: + if not self._initialized or self._ios_driver is not driver: + raise AgentNotInitializedError() + device_lock = DeviceExecutionLock( + driver.device_id, + description="Artemis iOS device operation", + concurrency_mode=getattr(self._config, "concurrency_mode", "per_device"), + max_concurrency=getattr(self._config, "max_concurrency", None), + session_id=self._session_id, + ingress="sdk", + lock_scope=IOS_LOCK_SCOPE, + ) + queue_cancel_event = threading.Event() + acquire_task = asyncio.create_task( + asyncio.to_thread(device_lock.acquire, cancel_event=queue_cancel_event) + ) + connect_attempted = False + try: + try: + await asyncio.shield(acquire_task) + except asyncio.CancelledError: + queue_cancel_event.set() + try: + await asyncio.shield(acquire_task) + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + # Draining the lock acquisition after cancellation is best effort. + logger.debug( + f"Device lock acquisition drain after cancel failed: {exc}", + exc_info=True, + ) + raise + connect_attempted = True + await driver.connect() + ( + self._device_context.device_width, + self._device_context.device_height, + ) = driver.screen_size + yield driver + finally: + try: + if connect_attempted: + try: + await driver.disconnect() + except (OSError, ValueError, RuntimeError, TimeoutError) as exc: + logger.debug( + f"iOS driver disconnect after SDK operation failed: {exc}", + exc_info=True, + ) + finally: + await asyncio.to_thread(device_lock.release) + + async def _install_app_internal(self, app_path: str | Path) -> str | None: + if self._config.device_platform != DevicePlatform.IOS: + return await super()._install_app_internal(app_path) + async with self._ios_operation() as driver: + return await driver.install_app(Path(app_path)) + + async def _prepare_app_lock(self, task: Task, context: ArtemisContext): + if context.device.mobile_platform == DevicePlatform.IOS and task.request.locked_app_package: + raise AgentError( + "iOS app locking is unavailable because foreground ownership cannot be verified." + ) + return await super()._prepare_app_lock(task, context) + + async def get_screenshot(self): + if self._config.device_platform != DevicePlatform.IOS: + return await super().get_screenshot() + async with self._ios_operation() as driver: + data = await driver.get_screen_data() + with Image.open(BytesIO(data.screenshot_bytes)) as image: + return image.copy() async def _prewarm_llm_connections(self, api_key: str | None = None): """Pre-warms the HTTP2/gRPC connection pools for both Native GenAI and LangChain clients in the background.""" @@ -200,6 +348,10 @@ def _on_cancel() -> None: logger.debug(f"[{task_name}] Cancel watcher stopped: {exc}") async def clean(self, force: bool = False): + driver = getattr(self, "_ios_driver", None) + if driver is not None: + await driver.disconnect() + self._ios_driver = None if not self._initialized and not force: return @@ -210,6 +362,14 @@ async def clean(self, force: bool = False): async def _ensure_device_unlocked(self) -> None: """Reject secure keyguard instead of allowing an agent to guess credentials.""" + driver = getattr(self, "_ios_driver", None) + if driver is not None: + # A previous task may have closed its native transport. + await driver.connect() + self._device_context.device_width, self._device_context.device_height = ( + driver.screen_size + ) + return if self._adb_client is None: raise AgentError("ADB client is not initialized.") @@ -234,6 +394,8 @@ async def _ensure_device_unlocked(self) -> None: async def _prepare_device_environment(self, context: ArtemisContext): """Prepare device environment flags (like forcing Web Accessibility) before the task runs.""" + if context.device.mobile_platform == DevicePlatform.IOS: + return if not self._config.force_web_accessibility: logger.info( "Forcing web accessibility is disabled in AgentConfig. Skipping" @@ -282,6 +444,12 @@ async def _prepare_device_environment(self, context: ArtemisContext): def _prepare_tracing(self, task: Task, context: ArtemisContext): """Prepare tracing and data engine setup.""" + driver = getattr(self, "_ios_driver", None) + if driver is not None: + from artemis.mcp.actuators.ios import IosActuator + + context._active_driver = driver + context.actuator = IosActuator(context) task_name = self._prepare_trace_paths(task) context.execution_setup = ExecutionSetup( diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 16893864..d6d641aa 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -15,6 +15,7 @@ """Builder for AgentConfig objects using a fluent interface.""" import os +from pathlib import Path from typing import Any, cast from artemis.config import ( @@ -23,6 +24,7 @@ load_agent_config, settings, ) +from artemis.drivers.ios.discovery import BOOTED_SIMULATOR_ID from artemis.context import DevicePlatform from artemis.sdk.types.agent import AgentConfig, ServerConfig from artemis.utils.video import detect_video_tools_enabled @@ -37,11 +39,13 @@ class AgentConfigBuilder(AgentConfigBuilderBase): def __init__(self): """Initialize an empty AgentConfigBuilder with Artemis defaults.""" super().__init__(servers=get_default_servers()) + self._video_recording_tools_explicit: bool = False self._video_recording_tools_enabled: bool = detect_video_tools_enabled() self._force_web_accessibility: bool = False self._disable_checker: bool = False self._concurrency_mode: str = "per_device" self._max_concurrency: int | None = None + self._ios_workspace_path: Path | None = None agent_cfg = load_agent_config() self._explorer = agent_cfg.explorer @@ -50,9 +54,8 @@ def __init__(self): self._video_analyzer = agent_cfg.video_analyzer self._enable_video_ledger = agent_cfg.video_analyzer.enable_ledger if agent_cfg.video_analyzer.enabled is not None: + self._video_recording_tools_explicit = True self._video_recording_tools_enabled = agent_cfg.video_analyzer.enabled - else: - self._video_recording_tools_enabled = detect_video_tools_enabled() self._disable_planner_validation = not agent_cfg.planner_validation.enabled self._enable_committee = agent_cfg.committee.enabled self._committee_debate_rounds = agent_cfg.committee.debate_rounds @@ -76,6 +79,24 @@ def for_device_serial(self, device_serial: str) -> "AgentConfigBuilder": """Target a specific Android device by its ADB serial number.""" return self.for_device(DevicePlatform.ANDROID, device_serial) + def for_ios_device( + self, device_id: str = BOOTED_SIMULATOR_ID, *, workspace_path: str | Path | None = None + ) -> "AgentConfigBuilder": + """Target an iOS device UDID (simulator or paired physical), or the booted simulator.""" + if workspace_path is not None: + self.with_ios_workspace(workspace_path) + return self.for_device(DevicePlatform.IOS, device_id) + + # Backward-compatible alias from when iOS support was simulator-only. + for_ios_simulator = for_ios_device + + def with_ios_workspace(self, workspace_path: str | Path | None) -> "AgentConfigBuilder": + """Set an existing Xcode project/workspace for iOS first-run approval.""" + self._ios_workspace_path = ( + Path(workspace_path).expanduser().resolve() if workspace_path is not None else None + ) + return self + def with_concurrency_mode(self, mode: str) -> "AgentConfigBuilder": """Configure concurrency mode: 'global' (1 task globally) or 'per_device' (1 task per device).""" self._concurrency_mode = str(mode).strip().lower() @@ -92,6 +113,7 @@ def with_video_recording_tools(self, enabled: bool = True) -> "AgentConfigBuilde Args: enabled: Whether to enable video recording tools """ + self._video_recording_tools_explicit = True self._video_recording_tools_enabled = enabled return self @@ -425,7 +447,15 @@ def _extra_config_fields(self) -> dict[str, Any]: or os.environ.get("ARTEMIS_DEVICE_ID") or os.environ.get("ADB_DEVICE_SERIAL") ), - "video_recording_tools_enabled": self._video_recording_tools_enabled, + "video_recording_tools_enabled": ( + self._video_recording_tools_enabled + if self._video_recording_tools_explicit + else detect_video_tools_enabled( + self._device_platform.value + if self._device_platform is not None + else DevicePlatform.ANDROID.value + ) + ), "force_web_accessibility": self._force_web_accessibility, "disable_checker": self._disable_checker, "disable_midway_checks": self._disable_midway_checks, @@ -454,6 +484,7 @@ def _extra_config_fields(self) -> dict[str, Any]: ), "concurrency_mode": self._concurrency_mode, "max_concurrency": self._max_concurrency, + "ios_workspace_path": self._ios_workspace_path, } def build(self, validate_profiles: bool = True) -> AgentConfig: diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 10f4e1d9..ccff0a98 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Literal +from pathlib import Path +from typing import Any, Literal from artemis.config import ( ExplorerConfig, @@ -34,6 +35,13 @@ __all__ = ["AgentConfig", "AgentProfile", "ApiBaseUrl", "DevicePlatform", "ServerConfig"] +def _default_video_recording_tools(data: dict[str, Any]) -> bool: + """Detect the recording toolchain for the configured device platform.""" + platform = data.get("device_platform") + platform_name = getattr(platform, "value", platform) or DevicePlatform.ANDROID.value + return detect_video_tools_enabled(str(platform_name)) + + class AgentConfig(AgentConfigBase): """ARTEMIS agent configuration. @@ -41,7 +49,9 @@ class AgentConfig(AgentConfigBase): settings to :class:`AgentConfigBase`. """ - video_recording_tools_enabled: bool = Field(default_factory=detect_video_tools_enabled) + video_recording_tools_enabled: bool = Field( + default_factory=lambda data: _default_video_recording_tools(data) + ) force_web_accessibility: bool = False disable_checker: bool = False disable_midway_checks: bool = True @@ -71,6 +81,10 @@ class AgentConfig(AgentConfigBase): video_analyzer: VideoAnalyzerConfig = Field(default_factory=VideoAnalyzerConfig) concurrency_mode: Literal["global", "per_device"] = "per_device" max_concurrency: int | None = None + ios_workspace_path: Path | None = Field( + default=None, + description="Optional existing Xcode project/workspace for local iOS first-run approval; no permission grants.", + ) model_config = {"arbitrary_types_allowed": True} diff --git a/artemis/services/llm.py b/artemis/services/llm.py index ec91e6fa..2a46b362 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -929,6 +929,9 @@ def _get_val(obj, attr, expected_type): return ModelEndpoint( provider=ModelProvider.from_string(provider_val), model_name=str(model_val), + api_base=_get_val(cfg, "api_base", str), + api_key=_get_val(cfg, "api_key", str), + max_tokens=_get_val(cfg, "max_tokens", int), temperature=_get_val(cfg, "temperature", (int, float)) or 0.0, timeout_seconds=_get_val(cfg, "timeout", (int, float)) or 60.0, thinking_budget=_get_val(cfg, "thinking_budget", int), diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 28735c90..e5dca5b5 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -55,6 +55,7 @@ get_recent_subgoal_hashes, ) from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -283,7 +284,7 @@ async def run_agent_turn(llm, system_prompt, tools, agent_name): human_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) @@ -377,7 +378,7 @@ async def run_tool(tc): final_human_content.append( { "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, + "image_url": {"url": image_data_uri(screenshot_b64)}, } ) diff --git a/artemis/tools/explorer_tool.py b/artemis/tools/explorer_tool.py index 14a95b94..93fc7a72 100644 --- a/artemis/tools/explorer_tool.py +++ b/artemis/tools/explorer_tool.py @@ -62,6 +62,7 @@ from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.visualization import draw_dots from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -480,7 +481,7 @@ def render_operator_blocks( return text return [ {"type": "text", "text": text}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}, + {"type": "image_url", "image_url": {"url": image_data_uri(img_b64)}}, ] diff --git a/artemis/tools/history/screenshot.py b/artemis/tools/history/screenshot.py index 94f55d34..3548e7e6 100644 --- a/artemis/tools/history/screenshot.py +++ b/artemis/tools/history/screenshot.py @@ -32,6 +32,7 @@ from artemis.core.tool_failure import ToolFailure from artemis.utils.visualization import overlay_action_on_screenshot from third_party.mobile_use.utils.logger import get_logger +from artemis.utils.image_mime import image_data_uri logger = get_logger(__name__) @@ -65,7 +66,7 @@ def to_content_blocks(self) -> list[dict[str, Any]] | str: encoded = base64.b64encode(self.image_bytes).decode("utf-8") return [ {"type": "text", "text": self.description}, - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded}"}}, + {"type": "image_url", "image_url": {"url": image_data_uri(encoded)}}, ] diff --git a/artemis/tools/index.py b/artemis/tools/index.py index cf1762ce..6d71ecb6 100644 --- a/artemis/tools/index.py +++ b/artemis/tools/index.py @@ -27,9 +27,16 @@ def get_tools_from_wrappers( wrappers: list[ToolWrapper], ) -> list[BaseTool]: """Instantiate the available tools of ``wrappers``, traced into the data engine.""" - return build_tools_from_wrappers( + tools = build_tools_from_wrappers( ctx, wrappers, wrap_tool=lambda t: trace_langchain_tool(t, ctx) ) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + tools = [ + t + for t in tools + if t.name not in {"run_adb_command", "manage_task", "analyze_task_output"} + ] + return tools def get_tool_by_name(name: str, tools: list[BaseTool]) -> BaseTool | None: diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 23cde2c7..ca47d9fb 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -18,17 +18,35 @@ from langchain_core.tools import BaseTool from pydantic import BaseModel, Field -from artemis.context import ArtemisContext +from artemis.context import ArtemisContext, DevicePlatform from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper from artemis.utils.cython_compat import CyFunctionDetector -from third_party.mobile_use.tools.mobile.launch_app import LAUNCH_APP_DOCSTRING, find_package +from third_party.mobile_use.tools.mobile.launch_app import ( + LAUNCH_APP_DOCSTRING, + find_package as find_android_package, +) from third_party.mobile_use.utils.app_launch_utils import launch_app_with_retries +async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = True) -> str | None: + """Resolve an installed Android package or iOS bundle identifier.""" + if getattr(getattr(ctx, "device", None), "mobile_platform", None) != DevicePlatform.IOS: + return await find_android_package(ctx, app_name, use_fallback=use_fallback) + from artemis.drivers.factory import get_driver + + apps = await get_driver(ctx).list_apps() + if app_name in apps: + return app_name + matches = [ + bundle_id for bundle_id, name in apps.items() if name.casefold() == app_name.casefold() + ] + return matches[0] if len(matches) == 1 else None + + class LaunchAppArgs(BaseModel): """Arguments schema for launching an application.""" @@ -82,11 +100,24 @@ async def execute( outcome = f"Failed to launch app '{app}': Package not found." error_msg = "Package not found." else: - success, error_msg = await launch_app_with_retries( - ctx=ctx, app_package=package_name - ) + if ( + getattr(getattr(ctx, "device", None), "mobile_platform", None) + == DevicePlatform.IOS + ): + from artemis.drivers.factory import get_driver + + success = await get_driver(ctx).launch_app(package_name) + error_msg = None if success else "Launch failed." + else: + success, error_msg = await launch_app_with_retries( + ctx=ctx, app_package=package_name + ) outcome = ( - f"Launched app '{app}' ({package_name}); foreground confirmed." + f"Launched app '{app}' ({package_name})." + if success + and getattr(getattr(ctx, "device", None), "mobile_platform", None) + == DevicePlatform.IOS + else f"Launched app '{app}' ({package_name}); foreground confirmed." if success else f"Failed to launch app '{app}': {error_msg}" ) diff --git a/artemis/tools/mobile/read_hierarchy.py b/artemis/tools/mobile/read_hierarchy.py index 02930e82..229bd7ce 100644 --- a/artemis/tools/mobile/read_hierarchy.py +++ b/artemis/tools/mobile/read_hierarchy.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect +import json from typing import Any from langchain_core.tools import BaseTool @@ -76,6 +77,8 @@ async def execute( xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None) if xml_hierarchy: return str(xml_hierarchy) + if getattr(screen_data, "platform", None) == "ios": + return json.dumps(screen_data.ui_elements, ensure_ascii=False) return ToolFailure("Error retrieving UI hierarchy: No UI hierarchy in screen data.") if driver is not None and hasattr(driver, "get_ui_hierarchy"): xml_hierarchy = await driver.get_ui_hierarchy() @@ -98,7 +101,10 @@ async def execute( def get_ui_hierarchy_tool(ctx: ArtemisContext) -> BaseTool: """Exports get_ui_hierarchy as a LangChain BaseTool.""" - return trace_langchain_tool(get_ui_hierarchy.to_langchain_tool(ctx), ctx) + tool = get_ui_hierarchy.to_langchain_tool(ctx) + if getattr(getattr(ctx, "device", None), "mobile_platform", None) == "ios": + tool.description = "[DIAGNOSTIC] Retrieves the current iOS accessibility elements as JSON." + return trace_langchain_tool(tool, ctx) ui_hierarchy_wrapper = ToolWrapper( diff --git a/artemis/utils/image_mime.py b/artemis/utils/image_mime.py new file mode 100644 index 00000000..20706ec0 --- /dev/null +++ b/artemis/utils/image_mime.py @@ -0,0 +1,42 @@ +"""Image media-type detection for LLM message payloads. + +Screenshots flow through the codebase as raw bytes or base64 strings and are +embedded in OpenAI-style ``image_url`` data URIs and provider ``Part`` objects. +Historically every site hardcoded ``image/jpeg`` while the bytes were actually +PNG, which strict OpenAI-compatible endpoints reject. Sniff the magic bytes +instead of trusting the label. +""" + +import base64 + +_PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +_JPEG_MAGIC = b"\xff\xd8" + + +def image_mime_type(data: bytes | str) -> str: + """Return ``image/png`` or ``image/jpeg`` for raw or base64 image bytes.""" + if isinstance(data, str): + # The base64 alphabet can't hold the magic bytes; decode just enough + # to sniff the header. + if data.startswith("data:"): + data = data.rsplit(",", 1)[-1] + chunk = data[:20] + chunk += "=" * (-len(chunk) % 4) + data = base64.b64decode(chunk) + if data[:2] == _JPEG_MAGIC: + return "image/jpeg" + if data[:8] == _PNG_MAGIC: + return "image/png" + # Every screenshot path in the project emits PNG; keep PNG as the honest + # default for unrecognized-but-image bytes. + return "image/png" + + +def image_data_uri(data: bytes | str) -> str: + """Return a ``data:image/;base64,`` URI with the correct type.""" + if isinstance(data, bytes): + mime = image_mime_type(data) + data = base64.b64encode(data).decode("utf-8") + else: + mime = image_mime_type(data) + return f"data:{mime};base64,{data}" diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 76afb5df..3743b2a2 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -21,11 +21,13 @@ import asyncio import importlib.util import json +import math from pathlib import Path import re import shutil -import time import subprocess +import sys +import time from typing import Any import cv2 @@ -289,21 +291,74 @@ async def remux_recording_to_mp4(source_path: Path, output_path: Path) -> bool: return False -async def probe_video_segment(video_path: Path) -> dict[str, float | int]: - """Read duration and coded dimensions for a finalized segment.""" - process = await asyncio.create_subprocess_exec( - get_ffprobe_path(), - "-v", - "error", - "-show_entries", - "stream=width,height,duration,codec_type:format=duration", - "-of", - "json", - str(video_path), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, _stderr = await process.communicate() +def _probe_video_segment_cv2(video_path: Path) -> dict[str, float | int]: + """Read metadata from a finalized CFR video when ffprobe is unavailable.""" + capture = cv2.VideoCapture(str(video_path)) + try: + width = capture.get(cv2.CAP_PROP_FRAME_WIDTH) + height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + fps = capture.get(cv2.CAP_PROP_FPS) + frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) + finally: + capture.release() + if not ( + math.isfinite(width) + and math.isfinite(height) + and math.isfinite(fps) + and math.isfinite(frame_count) + and width > 0 + and height > 0 + and fps > 0 + and frame_count > 0 + ): + return {} + return { + "duration": frame_count / fps, + "width": int(width), + "height": int(height), + } + + +async def probe_video_segment( + video_path: Path, *, timeout_seconds: float | None = None +) -> dict[str, float | int]: + """Read duration and coded dimensions for a finalized segment. + + ``timeout_seconds`` bounds the owned ffprobe child: on timeout or + cancellation the subprocess is killed and drained before propagating. + """ + try: + process = await asyncio.create_subprocess_exec( + get_ffprobe_path(), + "-v", + "error", + "-show_entries", + "stream=width,height,duration,codec_type:format=duration", + "-of", + "json", + str(video_path), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + except FileNotFoundError: + return await asyncio.to_thread(_probe_video_segment_cv2, video_path) + if timeout_seconds is None: + # Default callers keep the original unbounded communicate behaviour. + stdout, _stderr = await process.communicate() + else: + try: + stdout, _stderr = await asyncio.wait_for(process.communicate(), timeout_seconds) + except (TimeoutError, asyncio.CancelledError): + if process.returncode is None: + try: + process.kill() + except ProcessLookupError: + pass + try: + await process.communicate() + except (OSError, RuntimeError, TimeoutError) as drain_error: + logger.debug(f"ffprobe drain after kill failed: {drain_error}") + raise if process.returncode != 0: return {} try: @@ -364,6 +419,7 @@ async def write_recording_manifest( output_dir: Path, mp4_paths: list[Path], segment_offsets: dict[Path, float] | None = None, + probe_timeout_seconds: float | None = None, ) -> Path | None: """Write the browser playlist used for orientation-aware playback. @@ -385,7 +441,13 @@ async def write_recording_manifest( for path in mp4_paths: if not path.exists(): continue - metadata = await probe_video_segment(path) + # Forward the timeout only when supplied so default callers keep the + # historical probe signature. + metadata = await ( + probe_video_segment(path, timeout_seconds=probe_timeout_seconds) + if probe_timeout_seconds is not None + else probe_video_segment(path) + ) duration = float(metadata.get("duration", 0)) width = int(metadata.get("width", 0)) height = int(metadata.get("height", 0)) @@ -564,8 +626,16 @@ def is_scrcpy_installed() -> bool: return shutil.which("scrcpy") is not None -def detect_video_tools_enabled() -> bool: - """Check if both scrcpy and ffmpeg are available to enable automated video features.""" +def detect_video_tools_enabled(platform: str = "android") -> bool: + """Check whether the platform's recording toolchain is available locally. + + iOS uses native ``xcrun simctl io recordVideo`` capture plus the bundled + FFmpeg for post-processing; Android requires scrcpy plus FFmpeg. + """ + if str(platform).strip().lower() == "ios": + return ( + sys.platform == "darwin" and shutil.which("xcrun") is not None and is_ffmpeg_installed() + ) return is_ffmpeg_installed() and is_scrcpy_installed() diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py index c6c2ddd2..ba398d20 100644 --- a/artemis/utils/visualization.py +++ b/artemis/utils/visualization.py @@ -13,6 +13,7 @@ # limitations under the License. import io +import math import os import re from typing import Any @@ -170,6 +171,38 @@ def get_center_coordinates(left: int, top: int, right: int, bottom: int) -> tupl return (left + right) // 2, (top + bottom) // 2 +def get_hit_point( + node: dict[str, Any], width: int | None = None, height: int | None = None +) -> tuple[int, int] | None: + """Return a valid native interaction point, when the driver supplied one.""" + point = node.get("hit_point") + if not isinstance(point, (list, tuple)) or len(point) != 2: + return None + try: + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + for value in point + ): + return None + except OverflowError: + return None + x, y = map(round, point) + if x < 0 or y < 0 or (width is not None and x >= width) or (height is not None and y >= height): + return None + return x, y + + +def _native_interaction_metadata(node: dict[str, Any], width: int, height: int) -> dict[str, Any]: + metadata: dict[str, Any] = {} + if (point := get_hit_point(node, width, height)) is not None: + metadata["hit_point"] = list(point) + if isinstance(bundle := node.get("activation_bundle_id"), str) and bundle: + metadata["activation_bundle_id"] = bundle + return metadata + + def _inject_mutual_occlusion_warnings( lines: list[str], items_bounds: list[tuple[int, tuple[int, int, int, int]]] ) -> list[str]: @@ -298,7 +331,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: ocr_bounds = parse_bounds(ocr.get("bounds")) if ocr_bounds: left, top, right, bottom = ocr_bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(ocr["text"], cx, cy): continue @@ -314,7 +349,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: if not registered and text.strip() and bounds: left, top, right, bottom = bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(text, cx, cy): continue @@ -374,7 +411,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: ocr_bounds = parse_bounds(ocr.get("bounds")) if ocr_bounds: left, top, right, bottom = ocr_bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(ocr["text"], cx, cy): continue @@ -389,8 +428,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: "text": ocr["text"].strip(), "bounds": [left, top, right, bottom], "class": node.get("class"), - "resource_id": node.get("resource-id"), + "resource_id": node.get("resource-id") or node.get("resource_id"), "is_ocr": True, + **_native_interaction_metadata(node, width, height), } ) labels.append(str(idx)) @@ -406,7 +446,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: shown = text.strip() or hint if not registered and shown and bounds: left, top, right, bottom = bounds - cx, cy = get_center_coordinates(left, top, right, bottom) + cx, cy = get_hit_point(node, width, height) or get_center_coordinates( + left, top, right, bottom + ) if is_duplicate(shown, cx, cy): continue @@ -424,8 +466,9 @@ def is_duplicate(text_val: str, cx: int, cy: int) -> bool: "text": shown, "bounds": [left, top, right, bottom], "class": node.get("class"), - "resource_id": node.get("resource-id"), + "resource_id": node.get("resource-id") or node.get("resource_id"), "is_ocr": False, + **_native_interaction_metadata(node, width, height), } if not text.strip(): element["is_hint"] = True @@ -513,8 +556,13 @@ def _resolve_coordinates(coord: Any, width: int, height: int) -> tuple[int, int] if 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0: px = int(round(x * width)) py = int(round(y * height)) - elif 0 <= x <= 1000 and 0 <= y <= 1000 and (width > 1000 or height > 1000): - # Android normalized 1000 coordinate space + elif ( + 0 <= x <= 1000 + and 0 <= y <= 1000 + and (width > 1000 or height > 1000 or x > width or y > height) + ): + # Normalized 0-1000 space — also on small (logical-size) iOS + # screenshots where the values exceed the pixel bounds anyway. px = int(round(x * width / 1000.0)) py = int(round(y * height / 1000.0)) else: diff --git a/docs/ios.md b/docs/ios.md new file mode 100644 index 00000000..0ddbb1b1 --- /dev/null +++ b/docs/ios.md @@ -0,0 +1,425 @@ +# iOS support + +Artemis can run Flash and Pro tasks on iOS simulators and on paired physical +iPhones/iPads on macOS with Xcode 27 or newer. Select iOS explicitly: the +existing Android defaults still apply on a Mac. iOS tasks work through the CLI, +embedded Python SDK, Artemis Daemon task queue, Admin Console, and the Artemis +MCP tools — including native screen recording, video analysis/replay, device +discovery, live screen streaming, and per-device locking that cannot collide +with Android targets. + +The iOS drivers use tools included with Xcode and Artemis's existing Python MCP +dependency. Simulators communicate with `xcrun mcpbridge` using an initialized +MCP session for screen observation and input, and `xcrun simctl` for lifecycle +operations. Physical devices use `xcrun devicectl` (CoreDevice) for lifecycle +plus **WebDriverAgent (WDA)** for observation and input — Xcode's +`DeviceInteraction*` MCP tools accept simulators only. WDA is the same +XCUITest HTTP bridge Appium uses and must be built and installed on the device +once (see below). Apple added native agent device interactions in Xcode 27; +see the +[Xcode 27 release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). + +## Prepare Xcode + +1. Install Xcode 27 or newer, complete its first-launch setup, and install an iOS + simulator runtime. Confirm that `xcodebuild -version` reports the intended + version and that `xcode-select -p` points to that Xcode installation. You can + also set `DEVELOPER_DIR` for the Artemis process when multiple Xcode versions + are installed. +2. Enable external agent access using Apple's + [Xcode MCP access instructions](https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode) + and review any Xcode approval prompts. For the workspace-independent MCP + server introduced in Xcode 27, follow Apple's enablement instructions in the + [release notes](https://developer.apple.com/documentation/xcode-release-notes/xcode-27-release-notes). + Artemis does not enable the server or grant permissions automatically. +3. For physical devices: connect the iPhone/iPad over USB (or enable network + pairing), tap **Trust** on the device pairing prompt, and enable **Developer + Mode** in Settings > Privacy & Security on iOS 16+. `xcrun devicectl list + devices` should show the device `connected` (or `available (paired)`); the + JSON fields are `pairingState: paired` and `tunnelState: connected`. +4. Check the toolchain and available simulators from the repository root: + + ```bash + bash scripts/setup_ios_env.sh + ``` + + This check reads the Xcode version, tool locations, simulator and + physical-device (`devicectl`) inventories, and `xcrun mcp-server status`. It + does not install dependencies, boot a simulator, or change access settings. A passing check confirms prerequisites; the first + driver connection checks native MCP tool availability and device access. +5. Install Artemis's Python dependencies and configure a model provider using + the normal project configuration: + + ```bash + uv sync --dev + cp .env.example .env + ``` + + Fill in the provider credentials in `.env` and select the desired models in + `config/artemis.jsonc`. The one-click Android startup and dependency installer + are not needed for this iOS workflow. + +## Run a task + +List the simulators to find the UDID for an available iOS device: + +```bash +xcrun simctl list devices available +``` + +Run a standalone task on that simulator: + +```bash +uv run artemis run "Open Settings and inspect the General screen" \ + --platform ios --standalone --device-serial --profile flash +``` + +An explicitly selected simulator is booted if necessary, and the driver waits +for it to finish booting. To use a simulator that is already running, omit +`--device-serial` or pass `--device-serial booted`. Exactly one available iOS +simulator must be booted; with zero or multiple booted devices, choose a UDID. +The driver resolves `booted` once and pins every later command to that UDID. + +Use `--profile pro` to run the planner, operator, and verification workflow on +the same driver. Prompts and app identifiers should refer to iOS apps and bundle +identifiers, such as `com.apple.Preferences`. + +To install a simulator build before the task, add +`--app-path /absolute/path/MyApp.app`. The bundle's `Info.plist` must provide +`CFBundleIdentifier`. + +### Physical devices + +A paired physical device is selected the same way — by UDID: + +```bash +xcrun devicectl list devices # find the device UDID +uv run artemis run "Open Settings" \ + --platform ios --standalone --device-serial --profile flash +``` + +Physical UDIDs are routed to a separate driver: `devicectl` handles install, +launch, terminate, app listing, URL opening, and screenshot capture, while +the UI hierarchy and input (taps, swipes, text, buttons) go through +**WebDriverAgent**. `booted` remains a simulator-only selector — physical +tasks always require an explicit `--device-serial`. + +#### WebDriverAgent setup (one time per device) + +WDA is a signed XCTest runner app that exposes an HTTP automation endpoint on +port 8100. Build and install it once per device — Appium users can reuse an +existing WDA install: + +```bash +git clone https://github.com/appium/WebDriverAgent.git +cd WebDriverAgent +xcodebuild build-for-testing -project WebDriverAgent.xcodeproj \ + -scheme WebDriverAgentRunner -destination id= \ + -allowProvisioningUpdates DEVELOPMENT_TEAM= \ + PRODUCT_BUNDLE_IDENTIFIER=com.example.WebDriverAgentRunner +xcrun devicectl device install app --device \ + /Build/Products/Debug-iphoneos/WebDriverAgentRunner-Runner.app +``` + +Any Apple development team works; a free Personal Team profile must be +re-signed every 7 days while paid-program profiles last a year. + +When Artemis connects, it finds an installed `*WebDriverAgent*` runner on the +device, launches it via `devicectl device process launch`, and probes its HTTP +endpoint on the CoreDevice tunnel address and `127.0.0.1:8100` (for +`iproxy`/`pymobiledevice3` port forwards). Endpoint overrides: + +- `ARTEMIS_IOS_WDA_URL` — full base URL, e.g. `http://127.0.0.1:8100` or + `http://:8100` +- `ARTEMIS_IOS_WDA_HOST` — host only; port 8100 assumed +- `ARTEMIS_IOS_WDA_BUNDLE_ID` — nonstandard runner bundle identifier +- `ARTEMIS_IOS_WDA_XCTESTRUN` — path to a `WebDriverAgentRunner_*.xctestrun` + bundle from `xcodebuild build-for-testing`; Artemis hosts it via + `xcodebuild test-without-building` (the canonical WDA session — a bare + runner app launch does not start the HTTP server) + +Endpoint ownership limits: an explicitly configured or forwarded WDA +endpoint (`ARTEMIS_IOS_WDA_URL`/`ARTEMIS_IOS_WDA_HOST`) must belong to the +selected physical device. Before opening a session, Artemis reads WDA's +sessionless `GET /wda/device/info` and refuses to connect when it reports a +simulator or a device name different from the selected device — the name +corroborates the selection but is not a cryptographic proof of unique-device +identity (WDA's `uuid` is `identifierForVendor`, not the device UDID). Artemis +also refuses to replace a WDA session that belongs to another client, since +WDA's `POST /session` unconditionally kills the active session; close the +existing session first. + +Protocol details verified against WebDriverAgent 16.14.0 (commit `d177824`, +checked 2026-10-07): +[FBSessionCommands.m](https://github.com/appium/WebDriverAgent/blob/d177824/WebDriverAgentLib/Commands/FBSessionCommands.m), +[FBResponsePayload.m](https://github.com/appium/WebDriverAgent/blob/d177824/WebDriverAgentLib/Routing/FBResponsePayload.m), +[FBCustomCommands.m](https://github.com/appium/WebDriverAgent/blob/d177824/WebDriverAgentLib/Commands/FBCustomCommands.m). + +Simulator bridging note: `xcrun mcpbridge` is spawned with a minimal +environment; `DEVELOPER_DIR` and `MCP_XCODE_PID` are forwarded when set so a +specific Xcode toolchain can be pinned. + +Differences from simulators: + +- The device must already be paired, trusted, and connected; Artemis never + boots or unlocks it — unlock the device before connecting, since WDA cannot + inject touches while it is locked. The first XCTest attach may also show an + on-device "Enable UI Automation" passcode prompt; approve it once. +- `--app-path` expects a device-signed artifact: an `.app` built for an arm64 + device destination (signed with a valid provisioning profile) or a `.ipa`. + Simulator `.app` bundles are x86_64/arm64-simulator builds and cannot be + installed on hardware. +- Screen recording polls `devicectl device capture screenshot` and assembles + timestamped MP4 segments (~1–3 fps). There is no `recordVideo`-equivalent + stream on hardware, so motion fidelity is lower than simulator captures and + brief gaps between frames are expected. + +### First-run Xcode approval + +The very first time a new Python interpreter asks Xcode for device access, +Xcode requires user approval for that interpreter and, if one is supplied, the +selected project folder. Point Artemis at an existing Xcode project or +workspace so the request can be recorded: + +```bash +uv run artemis run "Open Settings" --platform ios --standalone \ + --ios-workspace /absolute/path/MyApp.xcodeproj +``` + +If approval is still pending, the run stops with an "Xcode Approval Required" +panel instead of retrying. Approve the interpreter and the selected folder from +the Xcode MCP menu bar icon, choosing **Always Allow** there if offered and you +want later runs to skip the prompt; then rerun the task. Advanced users can instead +inspect pending request IDs with `xcrun mcp-server status` and approve only +those entries via `sudo xcrun mcp-server approve --always` from +their own terminal; the CLI path requires admin rights and is performed by the +user, never by Artemis. + +Granted approvals persist across fresh bridge processes; there is no need to +keep a bridge alive. A different interpreter path or build, a different project +folder, or an expiring grant can require approval again. The Always/persistent +choice belongs to you and Xcode; Artemis requests only scoped approval for its +interpreter and the folder you select and never enables global access. + +Grant duration is Xcode's decision, not Artemis's: Xcode binds each agent +approval to the requesting binary's code signature. A binary signed with a +real signing identity can hold a persistent **Always Allow** grant, but an +unsigned or adhoc-signed interpreter — the common case for `python3` from uv, +Homebrew, or a virtual environment — receives a grant that expires after +roughly 24 hours, and upgrading or replacing that interpreter binary requires +re-approval either way. `xcrun mcp-server status` lists every grant with its +expiry. On headless or CI machines an administrator can run +`sudo xcrun mcp-server enable` to keep the service reachable while Xcode is +closed; per-agent approval still applies unless the administrator also passes +`--unsafe-always-allow-all-agents`, which lets any local process drive +reachable projects and should stay disabled outside CI. + +## Embedded Python SDK + +Configure iOS through the embedded SDK's builder: + +```python +from artemis.sdk import Agent +from artemis.sdk.builders import AgentConfigBuilder + +config = AgentConfigBuilder().for_ios_device("").build() +agent = Agent(config=config) +``` + +To run the first-run approval flow against a specific project, pass the +existing project or workspace through the builder: + +```python +config = ( + AgentConfigBuilder() + .for_ios_device( + "", + workspace_path="/absolute/path/MyApp.xcodeproj", + ) + .build() +) +``` + +`with_ios_workspace(path)` applies the same setting, and `None` clears it. +The path is used only if Xcode refuses the initial session with an approval +error; already-approved runs never open a workspace. + +The generic builder also accepts +`for_device(DevicePlatform.IOS, "")`, with `DevicePlatform` +imported from `artemis.context`. Supplying only a `device_serial` without an iOS +configuration retains the existing Android selection behavior. + +The dependency-free remote client submits iOS tasks to an iOS-capable host — +the remote host needs macOS, Xcode, and (for hardware) WDA set up as above; +`ios_workspace` is an optional host-side path, not resolved locally: + +```python +from artemis_client import ArtemisClient + +client = ArtemisClient( + base_url="http://mac-host:8000", + device_serial="", + platform="ios", + # optional: ios_workspace="/abs/host/path/MyApp.xcworkspace", +) +result = await client.run("Open Settings and inspect the General page") +``` + +iOS submissions first check `GET /api/v1/capabilities` and refuse to POST +when the host does not advertise `platform.ios` — upgrade the host instead +of silently running on Android. + +## Daemon, web console, and batch submission + +Without `--standalone`, `artemis run --platform ios` forwards +`platform`/`ios_workspace`/`device_serial` to the running Artemis Daemon, which +queues the task against the selected iOS device under the shared `ios` +lock scope and spawns the worker with `--platform ios` — no ADB endpoint or Android readiness +probe is involved. `artemis batch` accepts the same `--platform`, +`--device-serial`, and `--ios-workspace` flags for goal lists. + +The Admin Console `/api/run` accepts `platform: "ios"`, an iOS device UDID +(simulator or paired physical hardware) in `device_serial`, and an optional +`ios_workspace`; `/api/devices` lists Android +devices and iOS devices (simulators and paired physical hardware) together, +each tagged with its `platform`. The live screen view +(`/api/stream/device-live`) streams frames captured with `simctl io +screenshot` for simulators or `devicectl capture screenshot` for physical +devices when an iOS task holds the lock; `/api/stream/device-state` reports +the `platform` of the streamed device. Replay preserves the recorded session's +`mobile_platform`, so an iOS trace replays through the native iOS driver (the +device picker retargets iOS replays to a chosen UDID — simulator or paired +physical). + +`mobile_run_task` accepts `platform="ios"`, `device_serial=`, and +`ios_workspace=`; it validates the UDID against the `simctl` and +`devicectl` inventories rather than ADB and queues the runner under the `ios` +lock scope. `mobile_get_device_state` and `mobile_diagnose` accept the same +`platform` switch — the latter runs a native screenshot/hierarchy smoke test +on the selected iOS device. The legacy +`Android_ADB_Controller` actuator server (tap/swipe/type tools) remains +Android-only. + +A minimal run looks like: + +```python +agent = Agent(config=config) +try: + await agent.init() + result = await agent.run_task(goal="Open Settings", profile="flash") + screenshot = await agent.get_screenshot() +finally: + await agent.clean() +``` + +`init` resolves the simulator without opening a native session. Public +`get_screenshot()` and `install_app()` calls acquire the device execution +lease and own a short-lived native session that is closed before they return, +so they work both right after `init` and between tasks. An app installation +requested inside `run_task` reuses the task's existing lease and session +instead of acquiring a second one. + +## Supported operations and limits + +| Operation | Simulator | Physical device | +| --- | --- | --- | +| Device selection and readiness | `simctl` inventory, `boot`, `bootstatus` | `devicectl` inventory; must be paired and connected | +| Screenshot and accessibility hierarchy | Xcode native device-interaction MCP session | WDA `/source` and `/screenshot` (devicectl screenshot fallback) | +| Tap, long press, and swipe | Native synthesized touch events | WDA W3C pointer actions | +| Text entry | Native keyboard synthesis with `clear_exist=false` | WDA `/wda/keys`, `clear_exist=false` | +| Enter, Home, Power, volume, and app switcher | Native keyboard and button synthesis | WDA `/wda/keys` (Enter), `/wda/homescreen`, and `/wda/pressButton` | +| App install, launch, and terminate | `simctl` with simulator `.app` bundles | `devicectl` with signed `.app`/`.ipa`; terminate re-verifies the running executable against the installed app URL (a cached launch PID is only a hint — recycled PIDs are never trusted) | +| Screen recording | `simctl io recordVideo` (VFR H.264) | `devicectl` screenshot polling assembled to timestamped MP4 | + +The native hierarchy is normalized into the element tree used by Artemis's +perception and action tools. Screenshots and touch coordinates are kept in the +same coordinate space. Custom UI without accessible elements still relies on +visual targeting. + +watchOS, tvOS, and visionOS are outside this implementation. Install a +simulator build on simulators and a device-signed build on hardware; neither +artifact is interchangeable with the other or with an Android `.apk`. The +drivers do not build Xcode projects. + +iOS has no system Back button. Use the app's visible navigation controls. +The native driver supports `enter`, `home`, `power`, `volume_up`, `volume_down`, +and `app_switch`. The `erase_one_char` and `focus_and_clear_text` actions are +not exposed. The `back` and `delete` keys and automatic replacement of existing +text fail with an explicit unsupported-operation error. When calling the `input_text` action, +pass `clear_exist=false`; the default requests text replacement. In direct +driver or controller calls, the equivalent argument is `clear_existing=False`. +To replace text, clear the field using its visible UI first. +Android shell commands, Logcat, Android resource identifiers, Android package +discovery, and the Android Accessibility Helper are unavailable on iOS. +Platform-specific operations fail explicitly when unsupported. + +Screen recording on simulators uses the native `xcrun simctl io recordVideo` +capture with no third-party device automation. Each segment is a +variable-frame-rate H.264 `.mov` anchored to its first captured frame; after +recording stops (or when a rotation or the duration limit rolls a segment), +the bundled FFmpeg post-processing finalizes a browser-safe 30 fps MP4 and a +`recording.json` manifest mapping every segment to its session-time offset. +On physical devices, recording polls `devicectl device capture screenshot` +into timestamped PNG frames and assembles the same manifest/MP4 output shape +at the real capture cadence. The video analyzer can clip the sealed portion of +an in-progress recording; request ranges that reach past the sealed boundary +are clipped with a warning. Limitations: iOS capture is silent (no audio), +pre-first-frame startup time is not captured, physical capture runs at ~1–3 +fps rather than continuous video, and recorder restarts leave a brief gap in +the timeline rather than stretching recorded frames. + +## Troubleshooting + +- **Xcode version or tools are incorrect:** inspect `xcodebuild -version`, + `xcode-select -p`, and any `DEVELOPER_DIR` override. Command Line Tools alone + do not provide the full Xcode simulator and device-interaction environment. +- **MCP tool missing or access denied:** follow Apple's MCP access instructions, + review approval prompts, and check `xcrun mcp-server status`. Xcode 27 release + notes mention that some settings may require relaunching Xcode or restarting + the Mac. If Xcode requests workspace approval before allowing a session, open + and approve the intended workspace through Xcode's normal access flow. +- **CoreSimulator inventory fails:** run `xcrun simctl list devices available` + in a local terminal. Sandboxed processes need access to the simulator service + and its device data. Install a runtime and create a simulator if the iOS + inventory is empty. +- **Multiple booted simulators:** supply `--device-serial` with the intended + simulator's UDID. +- **Physical device not found:** confirm `xcrun devicectl list devices` lists + it as `paired`; if pairing is absent, reconnect and approve the Trust prompt. + A device shown as `disconnected`/offline needs USB reattachment or reachable + network pairing. +- **Physical install fails:** the artifact must be signed for the device — + check the signing team and provisioning profile, or build an `.ipa` with + `xcodebuild -exportArchive` for the device destination. +- **Physical input fails on a locked device:** unlock the device; WDA cannot + inject touches while locked. +- **"No WebDriverAgent server answers":** no WDA endpoint responded on the + CoreDevice tunnel or `127.0.0.1:8100`, and no installed runner matching + `*WebDriverAgent*` was found to launch. Build and install the WDA runner + (one-time steps above), or point `ARTEMIS_IOS_WDA_URL` at a forwarded or + LAN-reachable server. +- **WDA runner launches but never answers:** unlock the device first — UI + Automation, and therefore WDA's HTTP server, cannot start while it is + locked. Otherwise the runner may be crashing on launch — check provisioning + (`get-task-allow`, matching certificate) and whether a free-team profile + expired; also try forwarding port 8100 (`iproxy 8100 8100`) and setting + `ARTEMIS_IOS_WDA_URL`. +- **A queued iOS task runs on the wrong surface:** confirm the submission + carried `platform: "ios"` (CLI `--platform ios`, web request `platform`, + or the MCP `platform` argument); tasks default to Android. + +## Contributing and validation + +Keep native MCP and subprocess interactions behind the iOS driver. Unit tests +should use fake MCP sessions and command runners so the normal deterministic +suite remains usable on Linux and without Xcode or model credentials. Run +`make test`, `make lint`, and `make typecheck` before submitting a change. + +For live acceptance on a configured Mac, run the prerequisite script and an +explicit standalone CLI task. Verify that the chosen simulator receives taps, +swipes, and text, and that the screenshot and hierarchy describe the same +screen. Test an ambiguous `booted` selection and denied MCP access as well as a +successful session, and confirm that the session is released when the task +ends or is cancelled. Record the Xcode version, simulator runtime, UDID, and +observed results in the pull request; unit tests alone do not establish live +device compatibility. diff --git a/mcp_server/background/task_runner.py b/mcp_server/background/task_runner.py index a6f377ce..cad7d71b 100644 --- a/mcp_server/background/task_runner.py +++ b/mcp_server/background/task_runner.py @@ -42,6 +42,7 @@ load_dotenv(os.path.join(PROJECT_ROOT, ".env")) from artemis.runtime import trace_store +from artemis.runtime.adb_endpoint import normalize_device_platform from mcp_server.notifiers import notify from mcp_server.utils import device_utils @@ -121,6 +122,8 @@ async def run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_pro_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ): """Executes the mobile automation agent task and logs all actions/results. @@ -154,7 +157,8 @@ async def run_task( print("--------------------------------------------------") agent = None - adb_path = device_utils.resolve_adb_path() + is_ios = normalize_device_platform(platform, strict=False) == "ios" + adb_path = None if is_ios else device_utils.resolve_adb_path() target_serial = device_serial try: @@ -168,9 +172,14 @@ async def run_task( from artemis.sdk.builders import Builders from artemis.sdk.types import AgentProfile - connected_devices = device_utils.get_connected_devices(adb_path) + connected_devices = [] if is_ios else device_utils.get_connected_devices(adb_path) - if device_serial: + if is_ios: + # The iOS driver resolves "booted" against `simctl` at connect + # time; an explicit UDID was already validated at admission. + target_serial = device_serial or "booted" + print(f"✅ Targeting iOS device: '{target_serial}'.") + elif device_serial: target_serial = device_serial if connected_devices and device_serial not in connected_devices: print( @@ -221,7 +230,12 @@ async def run_task( if settings.ADB_HOST: config_builder.with_adb_server(host=settings.ADB_HOST, port=settings.ADB_PORT) - if target_serial: + if is_ios: + config_builder.for_ios_device( + device_id=target_serial or "booted", + workspace_path=ios_workspace, + ) + elif target_serial: from artemis.context import DevicePlatform config_builder.for_device(DevicePlatform.ANDROID, target_serial) @@ -434,6 +448,16 @@ async def run_task( "--explorer-pro-mode", help="Pro-profile Explorer perception version: 'flash', 'pro' or 'ultra'", ) + parser.add_argument( + "--platform", + default="android", + choices=["android", "ios"], + help="Target platform ('android' default; 'ios' targets simulators or physical devices via Xcode/devicectl)", + ) + parser.add_argument( + "--ios-workspace", + help="Xcode project/workspace path for first-run iOS approval (iOS only)", + ) args = parser.parse_args() @@ -449,5 +473,7 @@ async def run_task( device_serial=args.device_serial, verification_level=args.verification_level, explorer_pro_mode=args.explorer_pro_mode, + platform=args.platform, + ios_workspace=args.ios_workspace, ) ) diff --git a/mcp_server/tools/device_state.py b/mcp_server/tools/device_state.py index 76b682bf..916f6cda 100644 --- a/mcp_server/tools/device_state.py +++ b/mcp_server/tools/device_state.py @@ -14,11 +14,14 @@ """MCP Tool: mobile_get_device_state.""" +import asyncio import base64 import os from mcp_server.base import mcp from artemis.mcp.adb_server import _get_controller +from artemis.runtime.adb_endpoint import normalize_device_platform +from artemis.runtime.ios_observation import observe_ios_controller from mcp_server.utils import env_utils from artemis.utils.ocr_xml_fusion import ( fuse_ocr_with_xml, @@ -31,7 +34,11 @@ @mcp.tool() -async def mobile_get_device_state(view_type: str, device_serial: str | None = None) -> str: +async def mobile_get_device_state( + view_type: str, + device_serial: str | None = None, + platform: str | None = None, +) -> str: """Real-time mobile device state observer (for debugging and validation). Retrieves a real-time screenshot or a simplified UI element tree from the @@ -44,20 +51,40 @@ async def mobile_get_device_state(view_type: str, device_serial: str | None = No returns the image's local file URI. - "hierarchy": returns the simplified text-labeled element list — exactly what the automation subagent sees when making decisions. - device_serial: Optional device serial (e.g. "emulator-5554") to inspect - a specific device; omitted → the default connected device. With - several devices attached, confirm the target with the user - (`adb devices -l` lists serials). + device_serial: Optional device serial (e.g. "emulator-5554", or an iOS + device UDID with platform="ios") to inspect a specific device; + omitted → the default connected device. + platform: "android" (default) or "ios". For iOS, device_serial is a + device UDID (simulator or paired physical device); omit it to use the single booted simulator. """ try: - controller = _get_controller(device_serial=device_serial) - device_width = controller.ctx.device.device_width - device_height = controller.ctx.device.device_height + # Strict at the public boundary: a misspelled platform must never fall + # through to the Android controller. None/empty stays Android. + canonical_platform = normalize_device_platform(platform) + except ValueError as e: + return f"Error: Failed to initialize/lock device controller: {e}" + try: + if canonical_platform == "ios": + # iOS resolution enumerates simctl/devicectl synchronously; offload + # the blocking factory so the MCP loop stays responsive. + controller = await asyncio.to_thread( + _get_controller, device_serial=device_serial, target_platform="ios" + ) + else: + controller = _get_controller(device_serial=device_serial) + is_ios = str(getattr(controller.ctx.device, "mobile_platform", "android")).lower() == "ios" except Exception as e: - return f"Error: Failed to initialize/lock Android device controller: {e}" + return f"Error: Failed to initialize/lock device controller: {e}" try: - device_data = await controller.get_screen_data() + if is_ios: + # Owns the whole one-shot lifecycle — device lease, native connect, + # capture, disconnect, release — all cancellation-safe. + device_data = await observe_ios_controller(controller) + else: + device_data = await controller.get_screen_data() + device_width = controller.ctx.device.device_width + device_height = controller.ctx.device.device_height latest_screenshot_b64 = device_data.base64 xml_hierarchy = device_data.elements @@ -69,7 +96,8 @@ async def mobile_get_device_state(view_type: str, device_serial: str | None = No safe_device_id = "".join( [c if c.isalnum() or c in ("-", "_") else "_" for c in device_id] ) - screenshot_filename = f"live_screenshot_{safe_device_id}.jpg" + ext = "png" if is_ios else "jpg" + screenshot_filename = f"live_screenshot_{safe_device_id}.{ext}" screenshot_path = os.path.join(project_root, screenshot_filename) with open(screenshot_path, "wb") as f: diff --git a/mcp_server/tools/diagnose.py b/mcp_server/tools/diagnose.py index 4f336463..305bac52 100644 --- a/mcp_server/tools/diagnose.py +++ b/mcp_server/tools/diagnose.py @@ -44,6 +44,7 @@ ) from artemis.core.diagnostics.schema import ProbeResult, ProbeStatus, SystemReadinessReport from artemis.runtime import DeviceExecutionLock, trace_store +from artemis.runtime.adb_endpoint import normalize_device_platform from artemis.runtime.helper_manager import helper_manager from artemis.utils.credentials_validator import validate_api_key from third_party.mobile_use.utils.logger import get_logger @@ -467,14 +468,94 @@ def _probe_unavailable(serial: str | None, error: str) -> dict[str, Any]: } -async def _device_smoke_test(device_serial: str | None) -> dict[str, Any]: +async def _device_smoke_test( + device_serial: str | None, platform: str = "android" +) -> dict[str, Any]: """Drive the device end to end (screenshot + UI hierarchy) through the shared smoke test.""" - return await smoke_test_device(device_serial) + return await smoke_test_device(device_serial, platform=platform) + + +def _ios_requested_device( + ios_result: ProbeResult | None, requested_device: str | None +) -> dict[str, Any] | None: + """Canonical iOS selection from probe metadata, or None when ambiguous. + + Mirrors the driver/pool contract: UDID compare is case-insensitive, names + match exactly, exactly one match is required, and physical hardware is + never picked implicitly. Returns the selected entry as + ``{serial, name, state, is_emulator}``. + """ + metadata = ios_result.metadata if ios_result else {} + usable_sims = [ + d + for d in (metadata.get("simulators") or []) + if d.get("udid") and d.get("state") in ("Booted", "Shutdown") + ] + physical = [d for d in (metadata.get("connected_physical_devices") or []) if d.get("udid")] + booted = [d for d in usable_sims if d.get("state") == "Booted"] + + def _entry(device: dict[str, Any], *, is_emulator: bool) -> dict[str, Any]: + state = device.get("state") or "" + return { + "serial": str(device.get("udid")), + "name": device.get("name"), + "state": ("device" if not is_emulator or state == "Booted" else state or "unknown"), + "is_emulator": is_emulator, + } + + if requested_device: + if requested_device.strip().lower() == "booted": + return _entry(booted[0], is_emulator=True) if len(booted) == 1 else None + needle = requested_device.lower() + matches = [ + (d, True) + for d in usable_sims + if str(d.get("udid")).lower() == needle or d.get("name") == requested_device + ] + matches += [ + (d, False) + for d in physical + if str(d.get("udid")).lower() == needle or d.get("name") == requested_device + ] + if len(matches) != 1: + return None + device, is_emulator = matches[0] + return _entry(device, is_emulator=is_emulator) + if len(booted) == 1: + return _entry(booted[0], is_emulator=True) + if len(usable_sims) == 1: + return _entry(usable_sims[0], is_emulator=True) + return None async def _run_device_probe( - adb_result: ProbeResult | None, requested_device: str | None + adb_result: ProbeResult | None, + requested_device: str | None, + ios_result: ProbeResult | None = None, + platform: str = "android", ) -> dict[str, Any]: + if platform == "ios": + selected = _ios_requested_device(ios_result, requested_device) + if selected is None: + if requested_device: + return _probe_unavailable( + requested_device, + f"requested iOS device '{requested_device}' does not resolve to exactly " + "one usable iOS device; pass the simulator or paired physical device " + "UDID listed in the ios_simulators check facts", + ) + return _probe_unavailable( + None, + "no single unambiguous iOS device is available; pass device_serial with an " + "explicit simulator or paired physical device UDID", + ) + try: + return await _device_smoke_test(selected["serial"], platform="ios") + except Exception as exc: + return _probe_unavailable( + selected["serial"], f"iOS smoke test raised {exc.__class__.__name__}: {exc}" + ) + devices = (adb_result.metadata.get("devices") if adb_result else None) or [] ready = [str(d.get("serial")) for d in devices if d.get("state") == "device"] if requested_device and requested_device not in ready: @@ -732,6 +813,7 @@ def _next_steps( tasks: dict[str, list[dict[str, Any]]], device_probe: dict[str, Any] | None, accessibility_helper: dict[str, Any] | None = None, + platform: str = "android", ) -> list[str]: steps: list[str] = [] needs_restart = False @@ -773,16 +855,20 @@ def _next_steps( steps.extend(credential_lines) needs_restart = needs_restart or credential_restart - steps.extend( - _device_steps( - adb_result, - attempt_fix=attempt_fix, - requested_device=requested_device, - emulator=emulator, - launch_requested=bool(launch_steps), - tasks=tasks, + if platform != "ios": + # ADB-specific guidance (busy devices, AVD hints, authorization) does + # not apply to simulator targets; the ios_simulators probe's own + # steps cover iOS findings. + steps.extend( + _device_steps( + adb_result, + attempt_fix=attempt_fix, + requested_device=requested_device, + emulator=emulator, + launch_requested=bool(launch_steps), + tasks=tasks, + ) ) - ) steps.extend(launch_steps) steps.extend(_device_probe_steps(device_probe)) steps.extend(_helper_steps(accessibility_helper, attempt_fix=attempt_fix)) @@ -975,19 +1061,24 @@ async def _run_extras( launch_avd: str | None, verify_credentials: bool, probe_device: bool, + platform: str = "android", ) -> dict[str, Any]: """Optional, slower work: emulator launch, live key checks, device smoke test, lock state.""" adb_result = _find(results, "android_adb") + ios_result = _find(results, "ios_simulators") launch_steps: list[str] = [] if launch_avd: emulator, launch_steps = await _handle_launch_avd(launch_avd, adb_result) else: emulator = _emulator_status() - helper_serial = _helper_target(adb_result, requested_device) + # The on-device Accessibility Helper is Android-only; iOS observes natively. + helper_serial = _helper_target(adb_result, requested_device) if platform != "ios" else None credentials, device_probe, accessibility_helper = await asyncio.gather( _verify_credentials(_find(results, "gemini_api_key")) if verify_credentials else _none(), - _run_device_probe(adb_result, requested_device) if probe_device else _none(), + _run_device_probe(adb_result, requested_device, ios_result, platform) + if probe_device + else _none(), asyncio.to_thread(_helper_status, helper_serial) if helper_serial else _none(), ) return { @@ -1000,10 +1091,19 @@ async def _run_extras( } -def _requested_device_ready(results: list[ProbeResult], requested_device: str | None) -> bool: - """A caller-named device counts as a blocker: attached and authorized, or not ready.""" +def _requested_device_ready( + results: list[ProbeResult], + requested_device: str | None, + platform: str = "android", +) -> bool: + """A caller-named device counts as a blocker: attached and usable, or not ready.""" if not requested_device: return True + if platform == "ios": + ios_result = _find(results, "ios_simulators") + if ios_result is None: + return False + return _ios_requested_device(ios_result, requested_device) is not None adb_result = _find(results, "android_adb") if adb_result is None: return False @@ -1018,9 +1118,19 @@ def _verdict( credentials: list[dict[str, Any]] | None, device_probe: dict[str, Any] | None, accessibility_helper: dict[str, Any] | None = None, + platform: str = "android", ) -> str: - verdict = base_verdict(results) - if verdict == "blocked" or not _requested_device_ready(results, requested_device): + if platform == "ios": + # iOS tasks never touch ADB: the iOS simulator probe takes the device + # gate and android_adb drops out of the blocker set for this call. + ios_result = _find(results, "ios_simulators") + adjusted = [r for r in results if r.id != "android_adb"] + verdict = base_verdict(adjusted) + if verdict == "blocked" or ios_result is None or ios_result.status is not ProbeStatus.PASS: + return "blocked" + else: + verdict = base_verdict(results) + if verdict == "blocked" or not _requested_device_ready(results, requested_device, platform): return "blocked" primary = _primary_credential(credentials) if primary is not None and not primary.get("valid"): @@ -1096,6 +1206,7 @@ async def mobile_diagnose( launch_avd: str | None = None, verify_credentials: bool = False, probe_device: bool = False, + platform: str | None = None, ) -> dict[str, Any]: """Diagnoses why ARTEMIS cannot run tasks from this IDE and returns the fixes. @@ -1177,10 +1288,23 @@ async def mobile_diagnose( really start. Use it when the checks pass but tasks still fail on the device, or the screen stays black. A failed probe makes the verdict "blocked" and lists the fix. + platform: 'android' (default) or 'ios'. With 'ios', `device_serial` + is a simulator or paired physical UDID and `probe_device` exercises + the native path (Xcode MCP for simulators, WebDriverAgent for + hardware); ADB-only extras (AVD launch, accessibility helper) are skipped. """ fixes_applied: list[dict[str, Any]] = [] + try: + platform_name = normalize_device_platform(platform) + except ValueError: + return { + "verdict": "blocked", + "error": f"Invalid platform '{platform}'. Expected 'android' or 'ios'.", + } requested_device = device_serial.strip() if device_serial and device_serial.strip() else None - avd_name = launch_avd.strip() if launch_avd and launch_avd.strip() else None + avd_name = ( + launch_avd.strip() if launch_avd and launch_avd.strip() and platform_name != "ios" else None + ) try: report, host = await asyncio.wait_for( collect_readiness(), timeout=DIAGNOSIS_TIMEOUT_SECONDS @@ -1201,6 +1325,7 @@ async def mobile_diagnose( launch_avd=avd_name, verify_credentials=verify_credentials, probe_device=probe_device, + platform=platform_name, ), timeout=DIAGNOSIS_TIMEOUT_SECONDS, ) @@ -1216,10 +1341,26 @@ async def mobile_diagnose( credentials=credentials, device_probe=device_probe, accessibility_helper=accessibility_helper, + platform=platform_name, ) - device = _compact_device(report) - if device is not None and accessibility_helper is not None: - device["accessibility_helper"] = accessibility_helper + if platform_name == "ios": + # The response device is the iOS target diagnose would smoke/probe, + # not the Android active_device from the readiness report. + selected_ios = _ios_requested_device(_find(results, "ios_simulators"), requested_device) + device = ( + { + "serial": selected_ios["serial"], + "model": selected_ios["name"], + "platform": "ios", + "is_emulator": selected_ios["is_emulator"], + } + if selected_ios + else None + ) + else: + device = _compact_device(report) + if device is not None and accessibility_helper is not None: + device["accessibility_helper"] = accessibility_helper env_file = host.metadata.get("env_file") return { "verdict": verdict, @@ -1236,6 +1377,7 @@ async def mobile_diagnose( tasks=extras["tasks"], device_probe=device_probe, accessibility_helper=accessibility_helper, + platform=platform_name, ), "checks": [_render_check(r) for r in sort_by_fix_order(results)], "host": _compact_host(host.metadata), diff --git a/mcp_server/tools/task_runner.py b/mcp_server/tools/task_runner.py index d30c42eb..8d045e54 100644 --- a/mcp_server/tools/task_runner.py +++ b/mcp_server/tools/task_runner.py @@ -29,12 +29,14 @@ from artemis.config import ExplorerVersion, checker_overrides_for_level from artemis.config.runtime import read_ipc_port from artemis.runtime import ( + IOS_LOCK_SCOPE, DeviceExecutionLock, - device_pool, + IosTarget, ensure_daemon_running, submit_task_to_daemon, trace_store, ) +from artemis.runtime.adb_endpoint import device_pool_for, normalize_device_platform # Seconds the spawned runner gets to finish its imports and open its log files # before the spawn is declared dead. Normal startup creates stdout.log within a @@ -144,17 +146,23 @@ def _start_spawn_watchdog( ).start() -def _validate_device_serial(device_serial: str) -> dict[str, Any] | None: - """Reject a task whose explicitly requested device is not attached and authorized. +def _validate_device_serial(device_serial: str, platform: str = "android") -> dict[str, Any] | None: + """Reject a task whose explicitly requested device is not attached and usable. Returns a failure response dict when the serial must be rejected, or None when - the device is usable. If device enumeration itself fails (no adb available), + the device is usable. If device enumeration itself fails (no adb/xcrun), validation is skipped rather than blocking task submission. """ try: - # The shared validator fails open on an indeterminate/empty enumeration: + # The shared validators fail open on an indeterminate/empty enumeration: # the task proceeds and fails downstream with a clear no-device error. - detail = device_pool.validate_explicit_serial(device_serial) + detail = device_pool_for(platform).validate_explicit_serial(device_serial) + inspect_hint = ( + "Run `xcrun simctl list devices` (simulators) or " + "`xcrun devicectl list devices` (physical devices)" + if platform == "ios" + else "Run `adb devices -l` to inspect attached hardware" + ) except Exception: return None if detail is None: @@ -164,8 +172,8 @@ def _validate_device_serial(device_serial: str) -> dict[str, Any] | None: "error": detail, "message": ( f"{detail} Task rejected to prevent execution on an unintended device. " - "Run `adb devices -l` to inspect attached hardware, then resubmit with a " - "valid serial (or omit device_serial for automatic selection)." + f"{inspect_hint}, then resubmit with a " + "valid identifier (or omit device_serial for automatic selection)." ), } @@ -212,8 +220,10 @@ def mobile_run_task( device_serial: str | None = None, verification_level: str | None = None, explorer_mode: str | None = None, + platform: str = "android", + ios_workspace: str | None = None, ) -> dict[str, Any]: - """Starts an autonomous mobile UI automation subagent on a connected Android device. + """Starts an autonomous mobile UI automation subagent on a connected mobile device. Delegates a mobile workflow to a background agent. Non-blocking: returns immediately with `trace_id` (for `mobile_manage_task` / `mobile_inspect_trace`), @@ -259,14 +269,20 @@ def mobile_run_task( model: `"Flash"` or `"Pro"` — see model selection above. locked_app_package: Optional package name to lock execution to; the agent auto-launches it and restricts actions to that app. - app_path: Optional local APK path to install before running. + app_path: Optional local APK (Android) or signed .app/.ipa (iOS) to install before running. expected_output_desc: Optional, Pro only. If set, a summarization agent writes a report to `output.md` in `notes_dir`. Ignored for Flash. - device_serial: Optional device serial (e.g. "emulator-5554") to bind - execution to a specific device; distinct devices run concurrently. - If omitted, an available device is selected automatically. When + device_serial: Optional device serial (e.g. "emulator-5554" or an iOS + device UDID) to bind execution to a specific device; distinct devices + run concurrently. If omitted, an available device is selected + automatically (physical iOS hardware is never auto-selected). When several devices are attached, confirm the target with the user first - (`adb devices -l` lists serials and authorization states). + (`adb devices -l` lists Android serials; `xcrun devicectl list + devices` lists physical iOS UDIDs). + platform: `"android"` (default) or `"ios"` — iOS runs locally against + a simulator or paired physical device. + ios_workspace: Optional Xcode workspace path for iOS; scopes the + agent-access approval grant (simulator interaction only). verification_level: Optional, Pro only. Coarse Checker preset: `"off"` (no audit; the Operator self-reports), `"final"` (one exit review against the goal, the default), `"checkpoints"` (every plan @@ -282,6 +298,15 @@ def mobile_run_task( if model.lower() not in ("flash", "pro"): raise ValueError(f"Invalid model '{model}'. Must be either 'Flash' or 'Pro'.") canonical_model = "Flash" if model.lower() == "flash" else "Pro" + # 0a. Validate and normalize platform: 'android' (ADB) or 'ios' (Xcode). + platform = normalize_device_platform(platform) + if platform == "ios" and locked_app_package: + return { + "trace_id": None, + "status": "failed", + "error": "locked_app_package is Android-only; iOS tasks cannot lock to an app package.", + "message": "Task rejected: 'locked_app_package' requires the Android platform.", + } # 0b. Validate the Pro tuning knobs before any trace exists so a typo is a # plain tool error rather than a failed trace on disk. verification_level, explorer_mode = _normalize_pro_tuning(verification_level, explorer_mode) @@ -303,7 +328,7 @@ def mobile_run_task( # different device than the caller asked for. Runs after init_trace so the # rejection carries a trace_id like every other response of this tool. if device_serial: - rejection = _validate_device_serial(device_serial) + rejection = _validate_device_serial(device_serial, platform) if rejection: trace_store.update_trace_status(trace_id, "failed", error=rejection["error"]) return {"trace_id": trace_id, **rejection} @@ -325,6 +350,8 @@ def mobile_run_task( conversation_id=conversation_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, base_url=base_url, ) if resp and resp.get("status") == "rejected": @@ -442,6 +469,10 @@ def mobile_run_task( } if device_serial: reserve_kwargs["device_id"] = device_serial + if platform == "ios": + # iOS workers queue under the ios__ scope so they never collide + # with an Android device sharing the same textual identifier. + reserve_kwargs["lock_scope"] = IOS_LOCK_SCOPE queue_ticket = DeviceExecutionLock.reserve(**reserve_kwargs) # 5. Spawn the background task runner as an independent subprocess @@ -458,6 +489,8 @@ def mobile_run_task( canonical_model, "--conversation-id", conversation_id or "", + "--platform", + platform, ] if locked_app_package: cmd.extend(["--locked-app-package", locked_app_package]) @@ -467,6 +500,8 @@ def mobile_run_task( cmd.extend(["--expected-output-desc", expected_output_desc]) if device_serial: cmd.extend(["--device-serial", device_serial]) + if platform == "ios" and ios_workspace: + cmd.extend(["--ios-workspace", ios_workspace]) if verification_level: cmd.extend(["--verification-level", verification_level]) if explorer_mode: @@ -475,7 +510,13 @@ def mobile_run_task( env = os.environ.copy() env["ARTEMIS_SESSION_ID"] = trace_id env["ARTEMIS_TASK_INGRESS"] = "mcp" - if device_serial: + if platform == "ios": + # iOS workers run under the ios lock scope; the target scrubs + # stale ADB serials so an Android id can never leak in. + IosTarget(serial=device_serial).apply_to_environment(env) + if device_serial: + env["ARTEMIS_DEVICE_ID"] = device_serial + elif device_serial: env["ADB_DEVICE_SERIAL"] = device_serial env["ARTEMIS_DEVICE_ID"] = device_serial env[DeviceExecutionLock.QUEUE_TICKET_ENV] = queue_ticket @@ -499,6 +540,8 @@ def mobile_run_task( "session_id": trace_id, "ingress": "mcp", } + if platform == "ios": + transfer_kwargs["lock_scope"] = IOS_LOCK_SCOPE if device_serial: transfer_kwargs["device_id"] = device_serial diff --git a/packages/artemis-client/src/artemis_client/client.py b/packages/artemis-client/src/artemis_client/client.py index d1d3e2d4..b4525583 100644 --- a/packages/artemis-client/src/artemis_client/client.py +++ b/packages/artemis-client/src/artemis_client/client.py @@ -38,6 +38,8 @@ def request( #: Pro-profile Explorer perception versions accepted by ``/api/run`` (``explorer_mode``). ExplorerMode = Literal["flash", "pro", "ultra"] EXPLORER_MODES: tuple[str, ...] = ("flash", "pro", "ultra") +#: Device platforms accepted by ``/api/run`` (``platform``). +PLATFORMS: tuple[str, ...] = ("android", "ios") def _normalize_choice(value: str | None, name: str, choices: tuple[str, ...]) -> str | None: @@ -84,9 +86,15 @@ def __init__( concurrency_mode: str = "per_device", max_concurrency: int | None = None, standalone: bool = False, + platform: str | None = None, + ios_workspace: str | None = None, ) -> None: if poll_interval <= 0: raise ValueError("poll_interval must be greater than zero") + normalized_platform = _normalize_choice(platform, "platform", PLATFORMS) + workspace = str(ios_workspace).strip() if ios_workspace is not None else "" + if workspace and normalized_platform != "ios": + raise ValueError("ios_workspace requires platform='ios'") resolved_base_url = base_url or os.environ.get("ARTEMIS_BASE_URL") if not resolved_base_url: daemon_host = os.environ.get("ARTEMIS_DAEMON_HOST", "127.0.0.1") @@ -100,6 +108,9 @@ def __init__( self.concurrency_mode = str(concurrency_mode).strip().lower() self.max_concurrency = max_concurrency self.standalone = standalone + self._platform = normalized_platform + # Host-side path only: it is never resolved against the local disk. + self._ios_workspace = workspace or None self._transport = transport or JsonTransport( resolved_base_url, token=token, @@ -186,6 +197,8 @@ async def submit( task_id: str | None = None, verification_level: VerificationLevel | None = None, explorer_mode: ExplorerMode | None = None, + platform: str | None = None, + ios_workspace: str | None = None, options: Mapping[str, Any] | None = None, ) -> TaskHandle: """Submit one task and return immediately after scheduler admission. @@ -205,6 +218,13 @@ async def submit( verification_level, "verification_level", VERIFICATION_LEVELS ) resolved_mode = _normalize_choice(explorer_mode, "explorer_mode", EXPLORER_MODES) + resolved_platform = _normalize_choice(platform, "platform", PLATFORMS) or self._platform + call_workspace = str(ios_workspace).strip() if ios_workspace is not None else "" + if call_workspace and resolved_platform != "ios": + raise ValueError("ios_workspace requires platform='ios'") + resolved_workspace = call_workspace or ( + self._ios_workspace if resolved_platform == "ios" else None + ) if task_id is None: resolved_task_id = str(uuid.uuid4()) else: @@ -229,10 +249,22 @@ async def submit( "conversation_id": conversation_id, "verification_level": resolved_level, "explorer_mode": resolved_mode, + "platform": resolved_platform, + "ios_workspace": resolved_workspace, "options": dict(options) if options is not None else None, } payload.update({key: value for key, value in optional_values.items() if value is not None}) + if resolved_platform == "ios": + # A legacy host silently drops unknown platform fields — refuse + # before POSTing unless it advertises iOS wire support. + advertised = await self.capabilities() + if not advertised.supports("platform.ios"): + raise TaskRejectedError( + "This Artemis host does not advertise 'platform.ios' " + "support; upgrade the host before submitting iOS tasks." + ) + response = self._mapping( await self._request("POST", "/api/run", json_body=payload), endpoint="/api/run", @@ -306,6 +338,8 @@ async def run( task_id: str | None = None, verification_level: VerificationLevel | None = None, explorer_mode: ExplorerMode | None = None, + platform: str | None = None, + ios_workspace: str | None = None, options: Mapping[str, Any] | None = None, timeout: float = 1800.0, poll_interval: float | None = None, @@ -323,6 +357,8 @@ async def run( task_id=task_id, verification_level=verification_level, explorer_mode=explorer_mode, + platform=platform, + ios_workspace=ios_workspace, options=options, ) return await self.wait_for_task( @@ -341,6 +377,8 @@ async def run_task(self, task: Any, **overrides: Any) -> TaskResult: "device_serial": getattr(task, "device_serial", None) or getattr(task, "device_id", None), "locked_app_package": getattr(task, "locked_package", None), + "platform": getattr(task, "platform", None), + "ios_workspace": getattr(task, "ios_workspace", None), } values.update(overrides) return await self.run(goal, **values) diff --git a/packages/artemis-client/src/artemis_client/models.py b/packages/artemis-client/src/artemis_client/models.py index e9c4e2d4..05378a21 100644 --- a/packages/artemis-client/src/artemis_client/models.py +++ b/packages/artemis-client/src/artemis_client/models.py @@ -52,6 +52,24 @@ def _device_from_payload(payload: Mapping[str, Any]) -> str | None: return None +def _platform_from_payload(payload: Mapping[str, Any]) -> str: + """Normalized platform ('android'|'ios'|...) from raw or device_info.""" + direct = _string(payload.get("platform")) + if direct: + return direct.lower() + device_info = payload.get("device_info") + if isinstance(device_info, str): + try: + device_info = json.loads(device_info) + except (TypeError, ValueError, json.JSONDecodeError): + device_info = None + if isinstance(device_info, Mapping): + nested = _string(device_info.get("mobile_platform")) + if nested: + return nested.lower() + return "android" + + @dataclass(frozen=True, slots=True) class TaskHandle: """A task accepted by the remote Artemis scheduler.""" @@ -66,6 +84,11 @@ def session_id(self) -> str: """Compatibility alias for servers that call a task a session.""" return self.task_id + @property + def platform(self) -> str: + """Recorded device platform; defaults to 'android' when absent.""" + return _platform_from_payload(self.raw) + @classmethod def from_payload(cls, payload: Mapping[str, Any]) -> TaskHandle: task_id = _string(payload.get("task_id") or payload.get("session_id") or payload.get("id")) @@ -111,6 +134,11 @@ def done(self) -> bool: def succeeded(self) -> bool: return self.status in SUCCESS_TASK_STATUSES + @property + def platform(self) -> str: + """Recorded device platform; defaults to 'android' when absent.""" + return _platform_from_payload(self.raw) + @classmethod def from_payload( cls, @@ -160,6 +188,11 @@ class Device: busy: bool = False raw: Mapping[str, Any] = field(default_factory=dict, repr=False, compare=False) + @property + def platform(self) -> str: + """Recorded device platform; defaults to 'android' when absent.""" + return _platform_from_payload(self.raw) + @classmethod def from_payload(cls, payload: Mapping[str, Any]) -> Device: serial = _string( diff --git a/packages/artemis-client/tests/test_client.py b/packages/artemis-client/tests/test_client.py index a5480a7a..c3cceb63 100644 --- a/packages/artemis-client/tests/test_client.py +++ b/packages/artemis-client/tests/test_client.py @@ -15,9 +15,12 @@ from artemis_client import ( ArtemisClient, + Device, NotFoundError, ProtocolError, + TaskHandle, TaskRejectedError, + TaskResult, TaskTimeoutError, ) @@ -285,5 +288,186 @@ async def test_stop_targets_one_session(self) -> None: ) +class IosPlatformSubmissionTests(unittest.IsolatedAsyncioTestCase): + """Platform/workspace forwarding and iOS capability preflight.""" + + CAPABLE = { + "api_version": "1", + "features": ["tasks.submit", "platform.ios"], + } + + def _client(self, **kwargs) -> ArtemisClient: + self.transport = FakeTransport() + return ArtemisClient( + "https://artemis.example.test", + poll_interval=0.001, + transport=self.transport, + **kwargs, + ) + + @staticmethod + def _admitted(platform=None): + task = {"session_id": "task-1", "status": "pending"} + if platform: + task["platform"] = platform + return {"status": "started", "tasks": [task]} + + async def test_ios_submit_preflights_capabilities_then_posts_platform(self): + client = self._client(device_serial="SIM-UDID", platform="ios") + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + + handle = await client.submit("Open Settings") + + get_call, post_call = self.transport.calls + self.assertEqual(get_call[0], "GET") + self.assertEqual(post_call[0], "POST") + body = post_call[2] + self.assertEqual(body["platform"], "ios") + self.assertNotIn("ios_workspace", body) + self.assertEqual(handle.platform, "ios") + + async def test_android_submit_sends_no_platform_and_no_preflight(self): + client = self._client() + self.transport.add("POST", "/api/run", self._admitted()) + + await client.submit("Open Settings") + + self.assertEqual(len(self.transport.calls), 1) + method, path, body = self.transport.calls[0] + self.assertEqual((method, path), ("POST", "/api/run")) + self.assertNotIn("platform", body) + self.assertNotIn("ios_workspace", body) + + async def test_ios_workspace_is_forwarded_only_for_ios(self): + client = self._client(platform="ios", ios_workspace="/host/MyApp.xcworkspace") + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + + await client.submit("Open Settings") + + body = self.transport.calls[1][2] + self.assertEqual(body["platform"], "ios") + self.assertEqual(body["ios_workspace"], "/host/MyApp.xcworkspace") + + # A blank per-call token must inherit, never erase, the ctor choice. + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + await client.submit("Open Settings", platform=" ") + self.assertEqual(self.transport.calls[3][2]["platform"], "ios") + + def test_constructor_workspace_requires_ios_platform(self): + with self.assertRaises(ValueError): + self._client(ios_workspace="/host/MyApp.xcworkspace") + with self.assertRaises(ValueError): + self._client(platform="android", ios_workspace="/host/MyApp.xcworkspace") + + async def test_workspace_on_non_ios_call_rejects_before_transport(self): + client = self._client() + with self.assertRaises(ValueError): + await client.submit("goal", ios_workspace="/host/ws.xcworkspace") + self.assertEqual(self.transport.calls, []) + + async def test_invalid_platform_rejects_before_transport(self): + client = self._client() + with self.assertRaises(ValueError): + await client.submit("goal", platform="tvos") + self.assertEqual(self.transport.calls, []) + + async def test_legacy_server_blocks_ios_before_posting(self): + client = self._client(platform="ios") + self.transport.add("GET", "/api/v1/capabilities", NotFoundError(404, "not implemented")) + + with self.assertRaises(TaskRejectedError): + await client.submit("Open Settings") + + methods = [call[0] for call in self.transport.calls] + self.assertEqual(methods, ["GET"]) # no POST /api/run attempted + + async def test_per_call_platform_override_beats_constructor_default(self): + client = self._client(platform="android") + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add("POST", "/api/run", self._admitted("ios")) + + await client.submit("goal", platform="ios") + + self.assertEqual(self.transport.calls[1][2]["platform"], "ios") + + async def test_run_and_run_task_forward_platform(self): + task_id = "00000000-0000-4000-8000-000000000199" + client = self._client() + self.transport.add("GET", "/api/v1/capabilities", self.CAPABLE) + self.transport.add( + "POST", + "/api/run", + { + "status": "started", + "tasks": [{"session_id": task_id, "status": "pending", "platform": "ios"}], + }, + ) + self.transport.add( + "GET", + f"/api/sessions/{task_id}", + {"session_id": task_id, "status": "completed"}, + ) + task = type( + "T", + (), + { + "goal": "Do it", + "platform": "ios", + "ios_workspace": "/host/ws.xcworkspace", + }, + )() + result = await client.run_task(task, task_id=task_id) + + self.assertTrue(result.done) + body = self.transport.calls[1][2] + self.assertEqual(body["platform"], "ios") + self.assertEqual(body["ios_workspace"], "/host/ws.xcworkspace") + + async def test_platform_defaults_to_android_everywhere(self): + client = self._client() + self.transport.add("POST", "/api/run", self._admitted()) + handle = await client.submit("goal") + self.assertEqual(handle.platform, "android") + + self.transport.add( + "GET", "/api/sessions/task-1", {"session_id": "task-1", "status": "completed"} + ) + result = await client.get_task("task-1") + self.assertEqual(result.platform, "android") + + self.transport.add("GET", "/api/devices", {"devices": [{"serial": "x", "state": "device"}]}) + (device,) = await client.list_devices() + self.assertEqual(device.platform, "android") + + def test_models_read_platform_from_top_level_or_device_info(self): + self.assertEqual( + TaskHandle.from_payload({"task_id": "t", "platform": " iOS "}).platform, + "ios", + ) + self.assertEqual( + TaskResult.from_payload( + { + "task_id": "t", + "status": "completed", + "device_info": {"mobile_platform": "IOS"}, + } + ).platform, + "ios", + ) + self.assertEqual( + Device.from_payload( + { + "serial": "s", + "device_info": '{"mobile_platform": "ios"}', + } + ).platform, + "ios", + ) + self.assertEqual(Device.from_payload({"serial": "s", "platform": "ios"}).platform, "ios") + + if __name__ == "__main__": unittest.main() diff --git a/pyright-core.json b/pyright-core.json index 3ae152e7..2d9ede52 100644 --- a/pyright-core.json +++ b/pyright-core.json @@ -1,11 +1,15 @@ { "venvPath": ".", "venv": ".venv", + "pythonVersion": "3.12", "include": [ "artemis/graph/state.py", "artemis/graph/visibility.py", "artemis/runtime/adb_endpoint.py", "artemis/runtime/process_probe.py", + "artemis/drivers/ios", + "artemis/runtime/ios_device_pool.py", + "artemis/mcp/actuators/ios.py", "artemis/mcp/action_types.py", "artemis/data_engine/models.py", "artemis/agents/explorer/native_runner.py", diff --git a/scripts/setup_ios_env.sh b/scripts/setup_ios_env.sh new file mode 100755 index 00000000..028077f4 --- /dev/null +++ b/scripts/setup_ios_env.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Read-only prerequisite checks. This script does not install tools, boot +# devices, select Xcode, or change Xcode's MCP permissions. +set -euo pipefail + +fail() { + printf 'Error: %s\n' "$1" >&2 + exit 1 +} + +[[ "$(uname -s)" == "Darwin" ]] || fail "iOS support requires macOS." + +command -v xcodebuild >/dev/null 2>&1 || fail "Install Xcode 27 or newer." +command -v xcrun >/dev/null 2>&1 || fail "Xcode's xcrun is unavailable." + +if ! version_output=$(xcodebuild -version); then + fail "Cannot read the selected Xcode version. Check xcode-select -p." +fi +version=$(printf '%s\n' "$version_output" | awk '/^Xcode / {print $2; exit}') +[[ "$version" =~ ^([0-9]+)(\.[0-9]+)*$ ]] || fail "Cannot parse Xcode version: $version" +(( BASH_REMATCH[1] >= 27 )) || fail "Xcode 27 or newer is required (selected: $version)." + +if ! simctl_path=$(xcrun --find simctl); then + fail "The selected Xcode installation does not provide simctl." +fi +if ! bridge_path=$(xcrun --find mcpbridge); then + fail "The selected Xcode installation does not provide mcpbridge." +fi + +printf 'Xcode %s\nsimctl: %s\nmcpbridge: %s\n\n' "$version" "$simctl_path" "$bridge_path" + +if ! devices=$(xcrun simctl list devices available); then + fail "Cannot query CoreSimulator. Run this check in a local terminal with access to Xcode's simulator services." +fi +printf '%s\n\n' "$devices" +if ! printf '%s\n' "$devices" | awk ' + /^-- iOS / { ios = 1; next } + /^-- / { ios = 0 } + ios && /\([[:xdigit:]-]+\) \((Booted|Shutdown)\)/ { found = 1 } + END { exit !found } +'; then + fail "No available iOS simulator found. Install an iOS runtime and create a simulator in Xcode." +fi + +printf 'Xcode MCP server status:\n' +if ! xcrun mcp-server status; then + printf 'Could not read MCP server status. Check Xcode MCP access manually.\n' >&2 +fi + +if ! devicectl_path=$(xcrun --find devicectl); then + fail "The selected Xcode installation does not provide devicectl (needed for physical devices)." +fi +printf 'devicectl: %s\n\n' "$devicectl_path" + +printf 'Physical devices (devicectl inventory):\n' +if physical=$(xcrun devicectl list devices 2>/dev/null); then + printf '%s\n' "$physical" | awk 'NR > 2 && NF' + if ! printf '%s\n' "$physical" | grep -q 'paired'; then + printf 'No paired physical devices found. Pair over USB and tap Trust to use hardware.\n' + fi +else + printf 'Could not query CoreDevice. Physical-device runs need a paired, connected device.\n' >&2 +fi + +printf '\nToolchain and simulator inventory checks passed.\n' +printf 'Before running Artemis, review and grant Xcode MCP access for your agent.\n' +printf 'This check does not verify that device-interaction permission is granted.\n' +printf 'Physical devices additionally need a WebDriverAgent runner installed; see docs/ios.md.\n' +printf 'See docs/ios.md and https://developer.apple.com/documentation/xcode/giving-external-agents-access-to-xcode\n' diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index 6f3e6c81..19ebb364 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -1,11 +1,15 @@ { "click": { - "type": "function", "function": { - "name": "click", "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "click", "parameters": { "properties": { + "delay_ms": { + "default": 100, + "description": "Delay in milliseconds between consecutive clicks. Default is 100.", + "type": "integer" + }, "target": { "anyOf": [ { @@ -36,11 +40,6 @@ "default": 1, "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", "type": "integer" - }, - "delay_ms": { - "default": 100, - "description": "Delay in milliseconds between consecutive clicks. Default is 100.", - "type": "integer" } }, "required": [ @@ -48,18 +47,19 @@ ], "type": "object" } - } + }, + "type": "function" }, "input_text": { - "type": "function", "function": { - "name": "input_text", "description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').", + "name": "input_text", "parameters": { "properties": { - "text": { - "description": "The text content to input. Supports multi-line content with '\\n'.", - "type": "string" + "clear_exist": { + "default": true, + "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", + "type": "boolean" }, "target": { "anyOf": [ @@ -87,10 +87,9 @@ "default": null, "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." }, - "clear_exist": { - "default": true, - "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", - "type": "boolean" + "text": { + "description": "The text content to input. Supports multi-line content with '\\n'.", + "type": "string" } }, "required": [ @@ -99,13 +98,115 @@ ], "type": "object" } - } + }, + "type": "function" + }, + "long_press": { + "function": { + "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "long_press", + "parameters": { + "properties": { + "duration": { + "default": 1000, + "description": "Long press duration in milliseconds (default 1000).", + "type": "integer" + }, + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + }, + "target_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + } + }, + "required": [ + "target" + ], + "type": "object" + } + }, + "type": "function" + }, + "manage_app": { + "function": { + "description": "[ACTION] Launch or force stop a specified application.", + "name": "manage_app", + "parameters": { + "properties": { + "action": { + "description": "The action type.", + "enum": [ + "launch", + "stop" + ], + "type": "string" + }, + "app_name": { + "description": "Display name or package name of the application.", + "type": "string" + } + }, + "required": [ + "action", + "app_name" + ], + "type": "object" + } + }, + "type": "function" + }, + "press_key": { + "function": { + "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", + "name": "press_key", + "parameters": { + "properties": { + "key": { + "description": "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", + "enum": [ + "ENTER", + "BACK", + "HOME", + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN" + ], + "type": "string" + } + }, + "required": [ + "key" + ], + "type": "object" + } + }, + "type": "function" }, "swipe": { - "type": "function", "function": { + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n\u2022 Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n\u2022 Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "parameters": { "properties": { "direction": { @@ -126,20 +227,17 @@ "default": null, "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." }, - "start": { + "duration": { "anyOf": [ { - "items": { - "type": "integer" - }, - "type": "array" + "type": "integer" }, { "type": "null" } ], "default": null, - "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." + "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." }, "end": { "anyOf": [ @@ -156,10 +254,16 @@ "default": null, "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." }, - "target": { + "gesture": { "anyOf": [ { - "type": "integer" + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" }, { "items": { @@ -167,27 +271,15 @@ }, "type": "array" }, - { - "type": "string" - }, { "type": "null" } ], "default": null, - "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." }, - "gesture": { + "start": { "anyOf": [ - { - "enum": [ - "up", - "down", - "left", - "right" - ], - "type": "string" - }, { "items": { "type": "integer" @@ -199,10 +291,19 @@ } ], "default": null, - "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." }, - "target_description": { + "target": { "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, { "type": "string" }, @@ -211,83 +312,30 @@ } ], "default": null, - "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." }, - "duration": { + "target_description": { "anyOf": [ { - "type": "integer" + "type": "string" }, { "type": "null" } ], "default": null, - "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." - } - }, - "type": "object" - } - } - }, - "press_key": { - "type": "function", - "function": { - "name": "press_key", - "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).", - "parameters": { - "properties": { - "key": { - "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", - "enum": [ - "ENTER", - "BACK", - "HOME", - "APP_SWITCH" - ], - "type": "string" - } - }, - "required": [ - "key" - ], - "type": "object" - } - } - }, - "manage_app": { - "type": "function", - "function": { - "name": "manage_app", - "description": "[ACTION] Launch or force stop a specified application.", - "parameters": { - "properties": { - "action": { - "description": "The action type.", - "enum": [ - "launch", - "stop" - ], - "type": "string" - }, - "app_name": { - "description": "Display name or package name of the application.", - "type": "string" + "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." } }, - "required": [ - "action", - "app_name" - ], "type": "object" } - } + }, + "type": "function" }, "wait_for_delay": { - "type": "function", "function": { + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse\u2014whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "name": "wait_for_delay", - "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "parameters": { "properties": { "time_in_ms": { @@ -300,52 +348,7 @@ ], "type": "object" } - } - }, - "long_press": { - "type": "function", - "function": { - "name": "long_press", - "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", - "parameters": { - "properties": { - "target": { - "anyOf": [ - { - "type": "integer" - }, - { - "items": { - "type": "integer" - }, - "type": "array" - } - ], - "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." - }, - "target_description": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." - }, - "duration": { - "default": 1000, - "description": "Long press duration in milliseconds (default 1000).", - "type": "integer" - } - }, - "required": [ - "target" - ], - "type": "object" - } - } + }, + "type": "function" } } diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 33f5ac85..d1527c33 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -1,131 +1,92 @@ { - "CLICK_TOOL": { - "type": "function", - "function": { - "name": "click", - "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", - "parameters": { - "type": "object", - "properties": { - "target": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "array", - "items": { - "type": "integer" - } - } - ], - "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." - }, - "target_description": { - "type": "string", - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." - }, - "times": { - "type": "integer", - "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1." - }, - "delay_ms": { - "type": "integer", - "description": "Delay in milliseconds between consecutive clicks. Default is 100." - } - }, - "required": [ - "target" - ] - } - } - }, "CLICK_SEQUENCE_TOOL": { - "type": "function", "function": { - "name": "click_sequence", "description": "[ACTION] Executes a sequence of taps one by one in order on the specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks in the sequence have completed.", + "name": "click_sequence", "parameters": { - "type": "object", "properties": { + "delay_ms": { + "description": "Delay between consecutive taps in milliseconds (default 50ms).", + "type": "integer" + }, "sequence": { - "type": "array", + "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]].", "items": { - "type": "array", + "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]).", "items": { "type": "integer" }, - "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])." + "type": "array" }, - "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." + "type": "array" }, "target_descriptions": { - "type": "array", + "description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself.", "items": { "type": "string" }, - "description": "What each target is, in a few words, one entry per sequence entry in the same order (e.g. ['video body', 'skip button']). Required: a coordinate names nothing by itself." - }, - "delay_ms": { - "type": "integer", - "description": "Delay between consecutive taps in milliseconds (default 50ms)." + "type": "array" } }, "required": [ "sequence", "target_descriptions" - ] + ], + "type": "object" } - } + }, + "type": "function" }, - "LONG_PRESS_TOOL": { - "type": "function", + "CLICK_TOOL": { "function": { - "name": "long_press", - "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "click", "parameters": { - "type": "object", "properties": { + "delay_ms": { + "description": "Delay in milliseconds between consecutive clicks. Default is 100.", + "type": "integer" + }, "target": { "anyOf": [ { "type": "integer" }, { - "type": "array", "items": { "type": "integer" - } + }, + "type": "array" } ], - "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { - "type": "string", - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index.", + "type": "string" }, - "duration": { - "type": "integer", - "description": "Long press duration in milliseconds (default 1000)." + "times": { + "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", + "type": "integer" } }, "required": [ "target" - ] + ], + "type": "object" } - } + }, + "type": "function" }, "INPUT_TEXT_TOOL": { - "type": "function", "function": { - "name": "input_text", "description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').", + "name": "input_text", "parameters": { - "type": "object", "properties": { - "text": { - "type": "string", - "description": "The text content to input. Supports multi-line content with '\\n'." + "clear_exist": { + "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", + "type": "boolean" }, "target": { "anyOf": [ @@ -133,61 +94,41 @@ "type": "integer" }, { - "type": "array", "items": { "type": "integer" - } + }, + "type": "array" } ], "description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { - "type": "string", - "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index." + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index.", + "type": "string" }, - "clear_exist": { - "type": "boolean", - "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content." + "text": { + "description": "The text content to input. Supports multi-line content with '\\n'.", + "type": "string" } }, "required": [ "text", "target" - ] + ], + "type": "object" } - } + }, + "type": "function" }, - "SWIPE_TOOL": { - "type": "function", + "LONG_PRESS_TOOL": { "function": { - "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", + "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "name": "long_press", "parameters": { - "type": "object", "properties": { - "direction": { - "type": "string", - "enum": [ - "up", - "down", - "left", - "right" - ], - "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." - }, - "start": { - "type": "array", - "items": { - "type": "integer" - }, - "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." - }, - "end": { - "type": "array", - "items": { - "type": "integer" - }, - "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." + "duration": { + "description": "Long press duration in milliseconds (default 1000).", + "type": "integer" }, "target": { "anyOf": [ @@ -195,120 +136,182 @@ "type": "integer" }, { - "type": "array", "items": { "type": "integer" - } - }, - { - "type": "string" + }, + "type": "array" } ], - "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." - }, - "gesture": { - "anyOf": [ - { - "type": "string", - "enum": [ - "up", - "down", - "left", - "right" - ] - }, - { - "type": "array", - "items": { - "type": "integer" - } - } - ], - "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." }, "target_description": { - "type": "string", - "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling." + "description": "What the target is, in a few words (e.g. 'play button', 'search input', 'video body'). REQUIRED when target is a coordinate pair; ignored for an element index.", + "type": "string" + } + }, + "required": [ + "target" + ], + "type": "object" + } + }, + "type": "function" + }, + "MANAGE_APP_TOOL": { + "function": { + "description": "[ACTION] Launch or force stop a specified application.", + "name": "manage_app", + "parameters": { + "properties": { + "action": { + "description": "The action type.", + "enum": [ + "launch", + "stop" + ], + "type": "string" }, - "duration": { - "type": "integer", - "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes)." + "app_name": { + "description": "Display name or package name of the application.", + "type": "string" } }, - "required": [] + "required": [ + "action", + "app_name" + ], + "type": "object" } - } + }, + "type": "function" }, "PRESS_KEY_TOOL": { - "type": "function", "function": { + "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", "name": "press_key", - "description": "[ACTION] Press a physical or virtual system button (e.g. ENTER, BACK, HOME, APP_SWITCH).", "parameters": { - "type": "object", "properties": { "key": { - "type": "string", + "description": "Standard system button name (ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN).", "enum": [ "ENTER", "BACK", "HOME", - "APP_SWITCH" + "APP_SWITCH", + "POWER", + "VOLUME_UP", + "VOLUME_DOWN" ], - "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)." + "type": "string" } }, "required": [ "key" - ] + ], + "type": "object" } - } + }, + "type": "function" }, - "MANAGE_APP_TOOL": { - "type": "function", + "SWIPE_TOOL": { "function": { - "name": "manage_app", - "description": "[ACTION] Launch or force stop a specified application.", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n\u2022 Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n\u2022 Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", + "name": "swipe", "parameters": { - "type": "object", "properties": { - "action": { - "type": "string", + "direction": { + "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left).", "enum": [ - "launch", - "stop" + "up", + "down", + "left", + "right" ], - "description": "The action type." + "type": "string" }, - "app_name": { - "type": "string", - "description": "Display name or package name of the application." + "duration": { + "description": "Optional swipe/drag duration in milliseconds (default 800; computed automatically for directional swipes).", + "type": "integer" + }, + "end": { + "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", + "items": { + "type": "integer" + }, + "type": "array" + }, + "gesture": { + "anyOf": [ + { + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + }, + "start": { + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", + "items": { + "type": "integer" + }, + "type": "array" + }, + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "string" + } + ], + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + }, + "target_description": { + "description": "What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.", + "type": "string" } }, - "required": [ - "action", - "app_name" - ] + "required": [], + "type": "object" } - } + }, + "type": "function" }, "WAIT_FOR_DELAY_TOOL": { - "type": "function", "function": { + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse\u2014whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "name": "wait_for_delay", - "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", "parameters": { - "type": "object", "properties": { "time_in_ms": { - "type": "integer", - "description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes)." + "description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes).", + "type": "integer" } }, "required": [ "time_in_ms" - ] + ], + "type": "object" } - } + }, + "type": "function" } } diff --git a/tests/unit/admin_console/test_task_queue_service.py b/tests/unit/admin_console/test_task_queue_service.py index 24a572cd..9be986c0 100644 --- a/tests/unit/admin_console/test_task_queue_service.py +++ b/tests/unit/admin_console/test_task_queue_service.py @@ -1111,3 +1111,83 @@ async def test_enqueue_tasks_debounces_rapid_identical_submissions(): ) assert len(state.queue_items) == 1 assert res2["enqueued_count"] == 0 + + +@pytest.mark.asyncio +async def test_enqueue_tasks_platforms_do_not_dedupe_against_each_other(): + """Identical goal+serial text on Android vs iOS are distinct submissions.""" + state.queue_items = [] + state.active_session_id = None + + with ( + patch("apps.admin_console.services.task_queue_service.session_repo"), + patch( + "apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", + return_value="ticket-x", + ), + patch( + "artemis.runtime.device_pool.device_pool.try_list_devices_async", + new=AsyncMock(return_value=[]), + ), + patch( + "artemis.runtime.device_pool.device_pool.validate_explicit_serial_async", + new=AsyncMock(return_value=None), + ), + patch( + "artemis.runtime.ios_device_pool.ios_device_pool.validate_explicit_serial_async", + new=AsyncMock(return_value=None), + ), + patch.object(TaskQueueService, "ensure_worker_running"), + ): + res_android = await task_queue_service.enqueue_tasks( + ["shared goal"], device_serial="shared-serial", platform="android" + ) + res_ios = await task_queue_service.enqueue_tasks( + ["shared goal"], device_serial="shared-serial", platform="ios" + ) + # Same platform + goal + serial still debounces. + res_ios_again = await task_queue_service.enqueue_tasks( + ["shared goal"], device_serial="shared-serial", platform="ios" + ) + + assert res_android["enqueued_count"] == 1 + assert res_ios["enqueued_count"] == 1 + assert res_ios_again["enqueued_count"] == 0 + platforms = sorted(i["platform"] for i in state.queue_items) + assert platforms == ["android", "ios"] + ios_item = next(i for i in state.queue_items if i["platform"] == "ios") + assert ios_item["device_serial"] == "shared-serial" + + +def test_task_target_ios_skips_a_malformed_adb_endpoint(): + """iOS items never parse their stale/malformed Android endpoint snapshot.""" + from artemis.runtime.adb_endpoint import IosTarget + + target = TaskQueueService._task_target( + { + "platform": "ios", + "device_serial": "AAAA-1111", + "adb_endpoint": {"host": "", "port": "not-a-port"}, + } + ) + assert isinstance(target, IosTarget) + assert target.serial == "AAAA-1111" + + +def test_task_target_android_still_validates_its_endpoint(): + from artemis.runtime.adb_endpoint import AdbTarget + + with pytest.raises((ValueError, TypeError)): + TaskQueueService._task_target( + { + "platform": "android", + "device_serial": "emulator-5554", + "adb_endpoint": {"host": "", "port": "not-a-port"}, + } + ) + target = TaskQueueService._task_target( + {"device_serial": "emulator-5554", "adb_endpoint": AdbEndpoint.local().to_dict()} + if hasattr(AdbEndpoint.local(), "to_dict") + else {"device_serial": "emulator-5554"} + ) + assert isinstance(target, AdbTarget) diff --git a/tests/unit/agents/test_explorer_tiers.py b/tests/unit/agents/test_explorer_tiers.py index e8c400d3..1f84c6ca 100644 --- a/tests/unit/agents/test_explorer_tiers.py +++ b/tests/unit/agents/test_explorer_tiers.py @@ -251,7 +251,7 @@ async def test_universal_dispatch_appends_image_message(tmp_path): image_blocks = human.content[1:] assert len(image_blocks) == 2 # the missing file is skipped assert all(b["type"] == "image_url" for b in image_blocks) - assert image_blocks[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert image_blocks[0]["image_url"]["url"].startswith("data:image/png;base64,") @pytest.mark.asyncio diff --git a/tests/unit/agents/test_operator.py b/tests/unit/agents/test_operator.py index 5fcfb7c2..cf512642 100644 --- a/tests/unit/agents/test_operator.py +++ b/tests/unit/agents/test_operator.py @@ -1499,13 +1499,19 @@ async def test_operator_press_key_translation_is_case_insensitive(): assert err is None assert actions == [{"action": "press_key", "keycode": "KEYCODE_APP_SWITCH"}] - # Unsupported keys get a self-explanatory error that lists the accepted names. actions, err = node._translate_and_validate_tool( {"name": "press_key", "args": {"key": "volume_up"}}, mock_state ) + assert err is None + assert actions == [{"action": "press_key", "keycode": "KEYCODE_VOLUME_UP"}] + + # Unsupported keys get a self-explanatory error that lists the accepted names. + actions, err = node._translate_and_validate_tool( + {"name": "press_key", "args": {"key": "delete"}}, mock_state + ) assert actions == [] assert err == ( - "Error: Unsupported key 'volume_up'. Supported keys: ENTER, BACK, HOME, APP_SWITCH." + "Error: Unsupported key 'delete'. Supported keys: ENTER, BACK, HOME, APP_SWITCH, POWER, VOLUME_UP, VOLUME_DOWN." ) diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index 8def19fa..6cf135a7 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -151,7 +151,7 @@ def __init__(self, messages, initial_goal, operator_raw_data): assert "Step 1: Opened browser" in human_content[0]["text"] assert "Plan: Find capital" in human_content[0]["text"] assert human_content[1]["type"] == "image_url" - assert human_content[1]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data" + assert human_content[1]["image_url"]["url"] == "data:image/png;base64,mock_b64_data" @patch("artemis.agents.outputter.outputter.get_llm") diff --git a/tests/unit/core/test_device_smoke.py b/tests/unit/core/test_device_smoke.py index 8fd6e367..ef42b244 100644 --- a/tests/unit/core/test_device_smoke.py +++ b/tests/unit/core/test_device_smoke.py @@ -48,10 +48,13 @@ def _install(get_screen_data=None, factory=None): controller.get_screen_data = get_screen_data calls: list[str | None] = [] - def _fake_get_controller(device_serial=None): + def _fake_get_controller(device_serial=None, target_platform=None): calls.append(device_serial) if factory is not None: - return factory(device_serial) + if target_platform is None: + # Preserve the Android call shape: no platform kwarg. + return factory(device_serial) + return factory(device_serial, target_platform=target_platform) return controller monkeypatch.setattr("artemis.mcp.adb_server._get_controller", _fake_get_controller) @@ -358,6 +361,123 @@ def test_fix_for_error_generic_fallback_uses_serial(): assert "" in device_smoke.fix_for_error("timeout", None)[1] +# --------------------------------------------------------------------------- # +# iOS path +# --------------------------------------------------------------------------- # + +IOS_UDID = "AAAA-1111-0000" + + +def _ios_screen_data(): + return SimpleNamespace( + base64=base64.b64encode(FAKE_JPEG).decode("ascii"), + elements='', + width=1170, + height=2532, + ) + + +def _ios_controller(device_id=IOS_UDID): + controller = MagicMock() + controller.ctx = SimpleNamespace( + device=SimpleNamespace(device_id=device_id, mobile_platform="ios") + ) + controller._driver = SimpleNamespace() # not a PhysicalIosDriver -> xcode + return controller + + +@pytest.mark.asyncio +async def test_ios_smoke_runs_observation_helper_and_reports_xcode(patch_controller, monkeypatch): + """iOS observes through the lease-owning helper on the caller's loop.""" + observe = AsyncMock(return_value=_ios_screen_data()) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + captured: dict = {} + + def factory(device_serial, target_platform=None): + captured["device_serial"] = device_serial + captured["target_platform"] = target_platform + return _ios_controller() + + patch_controller(factory=factory) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is True + assert result["error"] is None + assert result["fix"] == [] + assert result["serial"] == IOS_UDID + assert result["hierarchy_backend"] == "xcode" + assert captured == {"device_serial": IOS_UDID, "target_platform": "ios"} + observe.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ios_smoke_ignores_adb_serial_env(patch_controller, monkeypatch): + """ADB_DEVICE_SERIAL must never seed an iOS target.""" + monkeypatch.setenv("ADB_DEVICE_SERIAL", "emulator-5554") + observe = AsyncMock(return_value=_ios_screen_data()) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + + def factory(device_serial, target_platform=None): + return _ios_controller(device_id=device_serial or "booted") + + patch_controller(factory=factory) + + result = await smoke_test_device(platform="ios") + + assert result["ok"] is True + # Serial comes from ctx after canonical resolution — never the ADB serial. + assert result["serial"] == "booted" + assert "emulator-5554" not in str(result) + + +@pytest.mark.asyncio +async def test_ios_smoke_skips_unscoped_android_busy_guard(patch_controller, monkeypatch): + """The helper performs scope-correct admission; an Android owner on the + same serial text must not veto the iOS smoke.""" + monkeypatch.setattr( + DeviceExecutionLock, + "get_active_owners", + classmethod(lambda cls: {IOS_UDID: _owner(IOS_UDID, "android task")}), + ) + observe = AsyncMock(return_value=_ios_screen_data()) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + patch_controller(factory=lambda device_serial, target_platform=None: _ios_controller()) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is True + observe.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ios_smoke_failure_maps_to_ios_fixes(patch_controller, monkeypatch): + observe = AsyncMock(side_effect=RuntimeError("WebDriverAgent did not answer within 45s")) + monkeypatch.setattr(device_smoke, "observe_ios_controller", observe) + patch_controller(factory=lambda device_serial, target_platform=None: _ios_controller()) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is False + assert "WebDriverAgent" in result["error"] + assert any("WebDriverAgent" in step for step in result["fix"]) + assert not any("adb " in step for step in result["fix"]) + + +@pytest.mark.asyncio +async def test_ios_smoke_init_failure_reports_ios_target(patch_controller): + def _boom(device_serial, target_platform=None): + raise Exception("iOS support requires macOS and Xcode 27 or later") + + patch_controller(factory=_boom) + + result = await smoke_test_device(device_serial=IOS_UDID, platform="ios") + + assert result["ok"] is False + assert "Failed to initialize iOS device controller" in result["error"] + assert any("Xcode" in step for step in result["fix"]) + + def test_smoke_test_is_exported_from_diagnostics_package(): import artemis.core.diagnostics as diagnostics diff --git a/tests/unit/core/test_readiness.py b/tests/unit/core/test_readiness.py index 0cfb0c77..37c93037 100644 --- a/tests/unit/core/test_readiness.py +++ b/tests/unit/core/test_readiness.py @@ -131,6 +131,7 @@ def test_sort_by_fix_order_follows_check_order_and_appends_unknown_ids(): "integration_host", "gemini_api_key", "android_adb", + "ios_simulators", "toolchain", "vision_ocr_key", ] diff --git a/tests/unit/drivers/test_ios_bridge.py b/tests/unit/drivers/test_ios_bridge.py new file mode 100644 index 00000000..ceb7d06e --- /dev/null +++ b/tests/unit/drivers/test_ios_bridge.py @@ -0,0 +1,428 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MCP bridge ownership and failure behavior without launching Xcode.""" + +import asyncio +from contextlib import asynccontextmanager +import json +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +from mcp.types import CallToolResult, TextContent +import pytest + +from artemis.drivers.ios import bridge +from artemis.drivers.ios.bridge import ( + XcodeApprovalRequiredError, + XcodeBridge, + xcode_approval_guidance, +) + + +def response(data=None, *, text=None, error=False): + return CallToolResult( + content=[TextContent(type="text", text=text)] if text is not None else [], + structuredContent=data, + isError=error, + ) + + +class NativeSession: + def __init__(self): + self.events = [] + self.tools_calls = [] + self.tool_calls = [] + self.initialize_hook = None + self.call_hook = None + self.pages = { + None: (["DeviceInteractionStartSession"], "second"), + "second": (["DeviceInteractionSynthesize", "DeviceInteractionEndSession"], None), + } + + async def __aenter__(self): + self.events.append(("session-enter", asyncio.current_task())) + return self + + async def __aexit__(self, *args): + self.events.append(("session-exit", asyncio.current_task())) + + async def initialize(self): + self.events.append(("initialize", asyncio.current_task())) + if self.initialize_hook: + await self.initialize_hook() + + async def list_tools(self, cursor=None): + self.tools_calls.append(cursor) + names, next_cursor = self.pages[cursor] + return SimpleNamespace( + tools=[SimpleNamespace(name=name) for name in names], nextCursor=next_cursor + ) + + async def call_tool(self, name, arguments): + self.tool_calls.append((name, arguments)) + if self.call_hook: + return await self.call_hook(name, arguments) + return response({"userMessage": "OK"}) + + +@pytest.fixture +def native_transport(monkeypatch): + session = NativeSession() + events = [] + parameters = [] + + @asynccontextmanager + async def stdio(params): + parameters.append(params) + events.append(("stdio-enter", asyncio.current_task())) + try: + yield object(), object() + finally: + events.append(("stdio-exit", asyncio.current_task())) + + monkeypatch.setattr(bridge, "stdio_client", stdio) + monkeypatch.setattr(bridge, "ClientSession", lambda reader, writer: session) + return SimpleNamespace(session=session, events=events, parameters=parameters) + + +def assert_resources_closed_in_owner(native): + assert [event for event, task in native.events] == ["stdio-enter", "stdio-exit"] + assert [event for event, task in native.session.events if event.startswith("session-")] == [ + "session-enter", + "session-exit", + ] + owners = {task for event, task in native.events + native.session.events} + assert len(owners) == 1 + + +@pytest.mark.asyncio +async def test_initializes_discovers_paginated_tools_and_preserves_xcode_selection( + native_transport, monkeypatch +): + monkeypatch.setenv("DEVELOPER_DIR", "/Applications/Selected Xcode.app/Contents/Developer") + monkeypatch.setenv("MCP_XCODE_PID", "1234") + client = XcodeBridge() + await client.start() + assert client.connected + assert client.tools == { + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", + } + assert native_transport.session.tools_calls == [None, "second"] + params = native_transport.parameters[0] + assert params.command == "xcrun" and params.args == ["mcpbridge"] + assert params.env["DEVELOPER_DIR"] == "/Applications/Selected Xcode.app/Contents/Developer" + assert params.env["MCP_XCODE_PID"] == "1234" + assert await client.call("DeviceInteractionEndSession", {"interactionSessionKey": "key"}) == { + "userMessage": "OK" + } + await client.close() + assert not client.connected + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "native_response", + [ + response({"screenshotPath": "/tmp/native.png"}), + response(text=json.dumps({"screenshotPath": "/tmp/native.png"})), + ], +) +async def test_reads_structured_and_legacy_json_results(native_transport, native_response): + async def call(name, arguments): + return native_response + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + assert await client.call("DeviceInteractionSynthesize", {}) == { + "screenshotPath": "/tmp/native.png" + } + await client.close() + + +@pytest.mark.asyncio +async def test_native_tool_errors_propagate_without_retrying_or_destroying_connection( + native_transport, +): + results = iter( + [response(text="Agent is not approved", error=True), response({"userMessage": "Closed"})] + ) + + async def call(name, arguments): + return next(results) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises( + XcodeApprovalRequiredError, + match="DeviceInteractionStartSession failed: Agent is not approved", + ): + await client.call("DeviceInteractionStartSession", {}) + assert await client.call("DeviceInteractionEndSession", {}) == {"userMessage": "Closed"} + assert [name for name, arguments in native_transport.session.tool_calls] == [ + "DeviceInteractionStartSession", + "DeviceInteractionEndSession", + ] + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "native_message", + [ + "This agent isn't approved to use Xcode's tools yet. Call XcodeOpenWorkspace first.", + "Agent is not approved", + "Xcode is waiting for the user to approve this request; it has been recorded.", + ], +) +async def test_native_approval_refusals_become_typed_errors(native_transport, native_message): + async def call(name, arguments): + return response(text=native_message, error=True) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises(XcodeApprovalRequiredError) as caught: + await client.call("DeviceInteractionStartSession", {}) + error = caught.value + assert error.tool_name == "DeviceInteractionStartSession" + assert error.native_message == native_message + assert str(error) == f"Xcode tool DeviceInteractionStartSession failed: {native_message}" + await client.close() + + +@pytest.mark.asyncio +async def test_unrelated_tool_errors_stay_generic_and_keep_bridge_alive(native_transport): + results = iter( + [response(text="Simulator is busy", error=True), response({"userMessage": "OK"})] + ) + + async def call(name, arguments): + return next(results) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises(RuntimeError, match="Simulator is busy") as caught: + await client.call("DeviceInteractionSynthesize", {}) + assert not isinstance(caught.value, XcodeApprovalRequiredError) + assert await client.call("DeviceInteractionEndSession", {}) == {"userMessage": "OK"} + await client.close() + + +def test_approval_guidance_names_interpreter_and_workspace_without_subprocess( + tmp_path, monkeypatch +): + def forbidden(*args, **kwargs): + raise AssertionError("guidance must never launch a subprocess") + + monkeypatch.setattr(subprocess, "run", forbidden) + monkeypatch.setattr(subprocess, "Popen", forbidden) + monkeypatch.setattr(bridge.asyncio, "create_subprocess_exec", forbidden) + guidance = xcode_approval_guidance(workspace_path=tmp_path) + assert str(Path(sys.executable).resolve()) in guidance + assert str(tmp_path) in guidance + assert "Always Allow" in guidance + assert "--always" in guidance + assert "not supplied" not in guidance + bare = xcode_approval_guidance() + assert "Workspace: not supplied" in bare + assert isinstance(XcodeApprovalRequiredError("tool", "msg").guidance, str) + scoped = XcodeApprovalRequiredError("tool", "msg", workspace_path=tmp_path) + assert scoped.workspace_path == tmp_path + assert str(tmp_path) in scoped.guidance + + +@pytest.mark.asyncio +async def test_unstructured_success_is_rejected(native_transport): + async def call(name, arguments): + return response(text="not JSON") + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + with pytest.raises(RuntimeError, match="no structured result"): + await client.call("DeviceInteractionSynthesize", {}) + await client.close() + + +@pytest.mark.asyncio +async def test_simulator_calls_are_serialized(native_transport): + first_started, release_first = asyncio.Event(), asyncio.Event() + + async def call(name, arguments): + if name == "first": + first_started.set() + await release_first.wait() + return response({"name": name}) + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + first = asyncio.create_task(client.call("first", {})) + await first_started.wait() + second = asyncio.create_task(client.call("second", {})) + await asyncio.sleep(0) + assert [name for name, arguments in native_transport.session.tool_calls] == ["first"] + release_first.set() + assert await asyncio.gather(first, second) == [{"name": "first"}, {"name": "second"}] + await client.close() + + +@pytest.mark.asyncio +async def test_initialization_timeout_releases_all_resources(native_transport): + async def initialize(): + await asyncio.Event().wait() + + native_transport.session.initialize_hook = initialize + client = XcodeBridge(timeout_seconds=0.01) + with pytest.raises(TimeoutError): + await client.start() + assert_resources_closed_in_owner(native_transport) + with pytest.raises(RuntimeError, match="not connected"): + await client.call("DeviceInteractionSynthesize", {}) + + +@pytest.mark.asyncio +async def test_initialization_cancellation_releases_all_resources(native_transport): + started = asyncio.Event() + + async def initialize(): + started.set() + await asyncio.Event().wait() + + native_transport.session.initialize_hook = initialize + client = XcodeBridge() + task = asyncio.create_task(client.start()) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_initialization_failure_releases_all_resources(native_transport): + async def initialize(): + raise OSError("Cannot open native service") + + native_transport.session.initialize_hook = initialize + client = XcodeBridge() + with pytest.raises( + RuntimeError, match="Cannot connect to Xcode MCP: Cannot open native service" + ): + await client.start() + assert not client.connected + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_input_timeout_retires_bridge_without_repeating_possibly_executed_action( + native_transport, +): + async def call(name, arguments): + await asyncio.Event().wait() + + native_transport.session.call_hook = call + client = XcodeBridge(timeout_seconds=0.01) + await client.start() + with pytest.raises(TimeoutError): + await client.call("DeviceInteractionSynthesize", {"interactionCommand": "t 10 20"}) + assert len(native_transport.session.tool_calls) == 1 + assert_resources_closed_in_owner(native_transport) + with pytest.raises(RuntimeError, match="not connected"): + await client.call("DeviceInteractionSynthesize", {}) + + +@pytest.mark.asyncio +async def test_cancelled_input_retires_bridge_and_fails_queued_action(native_transport): + started = asyncio.Event() + + async def call(name, arguments): + started.set() + await asyncio.Event().wait() + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + first = asyncio.create_task(client.call("first", {})) + await started.wait() + second = asyncio.create_task(client.call("second", {})) + await asyncio.sleep(0) + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + with pytest.raises(RuntimeError, match="bridge has closed"): + await second + assert [name for name, arguments in native_transport.session.tool_calls] == ["first"] + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_transport_failure_retires_bridge_and_fails_queued_action(native_transport): + started, fail = asyncio.Event(), asyncio.Event() + + async def call(name, arguments): + started.set() + await fail.wait() + raise OSError("pipe disconnected") + + native_transport.session.call_hook = call + client = XcodeBridge() + await client.start() + first = asyncio.create_task(client.call("first", {})) + await started.wait() + second = asyncio.create_task(client.call("second", {})) + await asyncio.sleep(0) + fail.set() + with pytest.raises(RuntimeError, match="connection failed: pipe disconnected"): + await first + with pytest.raises(RuntimeError, match="bridge has closed"): + await second + await client.close() + assert_resources_closed_in_owner(native_transport) + + +@pytest.mark.asyncio +async def test_bridge_can_reconnect_after_timeout_for_session_cleanup(native_transport): + async def call(name, arguments): + await asyncio.Event().wait() + + native_transport.session.call_hook = call + client = XcodeBridge(timeout_seconds=0.01) + await client.start() + with pytest.raises(TimeoutError): + await client.call("DeviceInteractionSynthesize", {"interactionCommand": "t 10 20"}) + assert not client.connected + native_transport.session.call_hook = None + await client.start() + assert client.connected + assert await client.call( + "DeviceInteractionEndSession", {"interactionSessionKey": "known-key"} + ) == {"userMessage": "OK"} + assert [name for name, arguments in native_transport.session.tool_calls] == [ + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", + ] + await client.close() + assert [event for event, task in native_transport.events] == [ + "stdio-enter", + "stdio-exit", + "stdio-enter", + "stdio-exit", + ] diff --git a/tests/unit/drivers/test_ios_driver.py b/tests/unit/drivers/test_ios_driver.py new file mode 100644 index 00000000..e20ba36e --- /dev/null +++ b/tests/unit/drivers/test_ios_driver.py @@ -0,0 +1,957 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native iOS behavior tested without Xcode, CoreSimulator, or a device.""" + +import asyncio +import base64 +import json +from pathlib import Path +import plistlib +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from PIL import Image +import pytest + +from artemis.drivers.base import KeyCode +from artemis.drivers.ios import xcode_driver +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError +from artemis.drivers.ios.hierarchy import application_bundle, parse_hierarchy +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver, run_xcrun +from third_party.mobile_use.controllers.types import ElementQuery + + +IOS_A = "DE345DD3-5792-4DAD-B863-144682629565" +IOS_B = "F81B6533-B1EC-40CE-8A21-9CD4F2994E55" +WATCH = "362C24CC-751F-457F-B8BF-9FFAC1F855A0" +NATIVE_TOOLS = { + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + "DeviceInteractionEndSession", +} +# Apple's packaged device-interaction skill uses these frame/hitPoint formats. +APPLE_HIERARCHY = """Device orientation: Portrait +------------------------ +Application bundle identifier: com.example.app +Application UI orientation: Portrait +Application, pid: 123, label: 'Example' + Window, {{0.0, 0.0}, {200.0, 400.0}}, hitPoint: {100.0, 200.0} + Other, {{0.0, 0.0}, {200.0, 400.0}}, hitPoint: {100.0, 200.0} + Button, {{10.0, 20.0}, {60.0, 30.0}}, identifier: 'login', label: 'Login', hitPoint: {15.0, 25.0}, activationBundleId: com.example.app + StaticText, {{20.0, 70.0}, {100.0, 20.0}}, label: 'Welcome' +""" + + +def device(udid=IOS_A, state="Booted", available=True): + return {"udid": udid, "name": "iPhone", "state": state, "isAvailable": available} + + +class NativeBridge: + def __init__(self, observation): + self.tools = NATIVE_TOOLS.copy() + self.connected = False + self.start = AsyncMock(side_effect=self.open) + self.close = AsyncMock(side_effect=self.finish) + self.call = AsyncMock(side_effect=self.respond) + self.observation = observation + self.hooks = {} + self.start_result = { + "interactionSessionKey": "test-session", + "deviceUUID": IOS_A, + "deviceIsSimulator": True, + } + + async def open(self): + self.connected = True + + async def finish(self): + self.connected = False + + async def respond(self, name, arguments): + if name in self.hooks: + return await self.hooks[name](arguments) + if name == "DeviceInteractionStartSession": + return self.start_result + if name == "DeviceInteractionSynthesize": + return self.observation + if name == "DeviceInteractionEndSession": + return {"userMessage": "Closed"} + if name == "XcodeOpenWorkspace": + return {"workspaceIdentifier": "workspace-1"} + raise AssertionError(f"Unexpected native tool {name}") + + +@pytest.fixture +def simulator(tmp_path, monkeypatch): + screenshot = tmp_path / "native.png" + Image.new("RGB", (200, 400)).save(screenshot) + hierarchy = tmp_path / "native-hierarchy.txt" + hierarchy.write_text(APPLE_HIERARCHY, encoding="utf-8") + observation = { + "screenshotPath": str(screenshot), + "hierarchyPath": str(hierarchy), + "thumbnailScreenshotPath": str(screenshot), + "logsPath": str(tmp_path / "logs.txt"), + "applicationState": "NotRun", + } + native = NativeBridge(observation) + inventory = {"devices": {"com.apple.CoreSimulator.SimRuntime.iOS-27-0": [device()]}} + + async def command(*arguments, timeout=30): + if arguments == ("xcodebuild", "-version"): + return b"Xcode 27.0\nBuild version 27A266a\n" + if arguments == ("simctl", "list", "devices", "--json"): + return json.dumps(inventory).encode() + if arguments[:2] == ("simctl", "listapps"): + # Real simctl emits an OpenStep/ASCII plist that plistlib cannot + # parse — the driver must route it through plutil. + return ( + b'{ "com.example.app" = { CFBundleDisplayName = Example; ' + b'CFBundleIdentifier = "com.example.app"; }; }' + ) + return b"" + + async def fake_plutil(payload: bytes, timeout: float = 30.0) -> bytes: + return json.dumps({"com.example.app": {"CFBundleDisplayName": "Example"}}).encode() + + commands = AsyncMock(side_effect=command) + monkeypatch.setattr(xcode_driver.sys, "platform", "darwin") + monkeypatch.setattr(xcode_driver, "run_xcrun", commands) + monkeypatch.setattr(xcode_driver, "plist_to_json", fake_plutil) + monkeypatch.setattr(xcode_driver, "XcodeBridge", lambda: native) + return SimpleNamespace( + driver=XcodeSimulatorDriver(), + native=native, + commands=commands, + inventory=inventory, + screenshot=screenshot, + hierarchy=hierarchy, + ) + + +@pytest.mark.asyncio +async def test_booted_selection_pins_every_app_operation_to_one_ios_udid(simulator, tmp_path): + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"].append( + device(IOS_B, state="Shutdown") + ) + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.watchOS-27-0"] = [ + device(WATCH) + ] + driver = simulator.driver + await driver.connect() + assert driver.device_id == IOS_A + await driver.launch_app("com.example.app") + await driver.stop_app("com.example.app") + await driver.open_url("example://item?q=$literal") + assert await driver.list_apps() == {"com.example.app": "Example"} + app = tmp_path / "Example.app" + app.mkdir() + (app / "Info.plist").write_bytes(plistlib.dumps({"CFBundleIdentifier": "com.example.app"})) + assert await driver.install_app(app) == "com.example.app" + + native_start = simulator.native.call.await_args_list[0] + assert native_start.args[0] == "DeviceInteractionStartSession" + assert native_start.args[1]["deviceIdentifier"] == IOS_A + assert all( + call.args[2] == IOS_A + for call in simulator.commands.await_args_list + if len(call.args) >= 3 + and call.args[:2] + in { + ("simctl", "launch"), + ("simctl", "terminate"), + ("simctl", "openurl"), + ("simctl", "listapps"), + ("simctl", "install"), + } + ) + assert not any( + call.args[:2] == ("simctl", "boot") for call in simulator.commands.await_args_list + ) + await driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("booted_count", [0, 2]) +async def test_booted_selector_rejects_ambiguous_or_missing_devices(simulator, booted_count): + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"] = [ + device(udid) for udid in (IOS_A, IOS_B)[:booted_count] + ] + with pytest.raises(ValueError, match="exactly one booted iOS simulator"): + await simulator.driver.connect() + simulator.native.start.assert_not_awaited() + simulator.native.call.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_booted_selector_tolerates_mixed_case_and_whitespace(simulator): + """The reserved 'booted' token normalizes before comparison.""" + simulator.driver._device_id = " BoOtEd " + await simulator.driver.connect() + assert simulator.driver.device_id == IOS_A + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("requested, available", [(WATCH, True), (IOS_A, False)]) +async def test_explicit_selection_rejects_non_ios_and_unavailable_devices( + simulator, requested, available +): + simulator.driver._device_id = requested + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"][0][ + "isAvailable" + ] = available + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.watchOS-27-0"] = [ + device(WATCH) + ] + with pytest.raises(ValueError, match="Unavailable iOS simulator"): + await simulator.driver.connect() + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_explicit_shutdown_device_is_booted_and_selected_case_insensitively(simulator): + simulator.driver._device_id = IOS_A.lower() + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"][0]["state"] = ( + "Shutdown" + ) + await simulator.driver.connect() + simulator.commands.assert_any_await("simctl", "boot", IOS_A) + simulator.commands.assert_any_await("simctl", "bootstatus", IOS_A, "-b", timeout=180.0) + assert simulator.driver.device_id == IOS_A + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_repeated_connect_reuses_session(simulator): + await simulator.driver.connect() + await simulator.driver.connect() + simulator.native.start.assert_awaited_once() + starts = [ + c for c in simulator.native.call.await_args_list if c.args[0].endswith("StartSession") + ] + assert len(starts) == 1 + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "result", + [ + {"interactionSessionKey": "wrong-target", "deviceUUID": IOS_B, "deviceIsSimulator": True}, + { + "interactionSessionKey": "physical-target", + "deviceUUID": IOS_A, + "deviceIsSimulator": False, + }, + ], +) +async def test_native_wrong_target_is_closed_before_any_interaction(simulator, result): + simulator.native.start_result = result + with pytest.raises(RuntimeError, match="different device"): + await simulator.driver.connect() + simulator.native.call.assert_any_await( + "DeviceInteractionEndSession", {"interactionSessionKey": result["interactionSessionKey"]} + ) + assert not any( + c.args[0] == "DeviceInteractionSynthesize" for c in simulator.native.call.await_args_list + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_missing_native_tools_fail_before_session_start(simulator): + simulator.native.tools.remove("DeviceInteractionSynthesize") + with pytest.raises(RuntimeError, match="tools are unavailable"): + await simulator.driver.connect() + simulator.native.call.assert_not_awaited() + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_non_macos_host_is_rejected_before_launching_native_tools(simulator, monkeypatch): + monkeypatch.setattr(xcode_driver.sys, "platform", "linux") + with pytest.raises(RuntimeError, match="requires macOS"): + await simulator.driver.connect() + simulator.commands.assert_not_awaited() + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_old_xcode_is_rejected_before_device_boot(simulator): + simulator.commands.side_effect = None + simulator.commands.return_value = b"Xcode 26.3\nBuild version 17C529\n" + with pytest.raises(RuntimeError, match="requires Xcode 27"): + await simulator.driver.connect() + simulator.commands.assert_awaited_once_with("xcodebuild", "-version") + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_start_session_error_closes_bridge_without_fake_session_cleanup(simulator): + simulator.native.call.side_effect = RuntimeError("Agent approval required") + with pytest.raises(RuntimeError, match="Agent approval required"): + await simulator.driver.connect() + assert [c.args[0] for c in simulator.native.call.await_args_list] == [ + "DeviceInteractionStartSession" + ] + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_missing_session_key_is_rejected_and_bridge_closed(simulator): + simulator.native.start_result.pop("interactionSessionKey") + with pytest.raises(RuntimeError, match="session key"): + await simulator.driver.connect() + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connect_preserves_device_rejection_when_cleanup_also_fails(simulator): + simulator.native.start_result["deviceUUID"] = IOS_B + + async def respond(name, arguments): + if name == "DeviceInteractionEndSession": + raise RuntimeError("Cleanup unavailable") + return await simulator.native.respond(name, arguments) + + simulator.native.call.side_effect = respond + with pytest.raises(RuntimeError, match="different device"): + await simulator.driver.connect() + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_failed_initial_observation_releases_native_session(simulator): + async def fail_capture(name, arguments): + if name == "DeviceInteractionSynthesize": + raise RuntimeError("Accessibility service failed") + return await simulator.native.respond(name, arguments) + + simulator.native.call.side_effect = fail_capture + with pytest.raises(RuntimeError, match="Accessibility service failed"): + await simulator.driver.connect() + simulator.native.call.assert_any_await( + "DeviceInteractionEndSession", {"interactionSessionKey": "test-session"} + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_disconnect_closes_bridge_even_when_session_end_fails(simulator): + await simulator.driver.connect() + simulator.native.call.side_effect = RuntimeError("Native session expired") + with pytest.raises(RuntimeError, match="expired"): + await simulator.driver.disconnect() + simulator.native.close.assert_awaited_once() + with pytest.raises(RuntimeError, match="Connect"): + await simulator.driver.get_screen_data() + + +@pytest.mark.asyncio +async def test_cancelled_initial_capture_releases_session_and_preserves_cancellation(simulator): + started = asyncio.Event() + + async def respond(name, arguments): + if name == "DeviceInteractionSynthesize": + started.set() + await asyncio.Event().wait() + return await simulator.native.respond(name, arguments) + + simulator.native.call.side_effect = respond + task = asyncio.create_task(simulator.driver.connect()) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + simulator.native.call.assert_any_await( + "DeviceInteractionEndSession", {"interactionSessionKey": "test-session"} + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_disconnect_reopens_retired_transport_only_to_close_known_session(simulator): + await simulator.driver.connect() + simulator.native.connected = False + simulator.native.call.reset_mock() + await simulator.driver.disconnect() + assert simulator.native.start.await_count == 2 + simulator.native.call.assert_awaited_once_with( + "DeviceInteractionEndSession", {"interactionSessionKey": "test-session"} + ) + simulator.native.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scale", [1, 3]) +async def test_observation_and_coordinate_actions_share_screenshot_geometry(simulator, scale): + Image.new("RGB", (200 * scale, 400 * scale)).save(simulator.screenshot) + await simulator.driver.connect() + data = await simulator.driver.get_screen_data() + assert data.platform == "ios" + assert (data.width, data.height) == (200 * scale, 400 * scale) + assert base64.b64decode(data.screenshot_base64) == data.screenshot_bytes + button = next(element for element in data.ui_elements if element["resource_id"] == "login") + assert button["hit_point"] == [15 * scale, 25 * scale] + assert button["parsed_bounds"] == { + "left": 10 * scale, + "top": 20 * scale, + "right": 70 * scale, + "bottom": 50 * scale, + } + + await simulator.driver.tap(15 * scale, 25 * scale, duration_ms=600) + arguments = simulator.native.call.await_args.args[1] + assert arguments["interactSessionKey"] == "test-session" + command = arguments["interactionCommand"].split() + assert command[0] == "t" + assert list(map(float, command[1:])) == [15, 25, 0.6] + + await simulator.driver.swipe(150 * scale, 300 * scale, 50 * scale, 100 * scale, 250) + command = simulator.native.call.await_args.args[1]["interactionCommand"].split() + assert command[0] == "t" and command[3] == "f" + assert list(map(float, command[1:3] + command[4:])) == [150, 300, 50, 100, 0.25] + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_element_action_uses_native_hit_point_and_app_activation(simulator): + await simulator.driver.connect() + assert await simulator.driver.tap_element(ElementQuery(resource_id="login")) + arguments = simulator.native.call.await_args.args[1] + assert arguments["activationBundleId"] == "com.example.app" + assert list(map(float, arguments["interactionCommand"].split()[1:])) == [15, 25, 0.1] + assert not await simulator.driver.tap_element(ElementQuery(text="Missing")) + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_text_preserves_unicode_whitespace_and_literal_native_escape_sequences(simulator): + await simulator.driver.connect() + text = "한글🙂 'quoted'\n\t" + r"\u{000A} b h $(literal)" + assert await simulator.driver.input_text(text, clear_existing=False) + command = simulator.native.call.await_args.args[1]["interactionCommand"] + assert command.startswith("sender keyboard kbd ") + encoded = command.removeprefix("sender keyboard kbd ") + # The native grammar decodes escapes once. Literal user escape syntax must + # survive that decode, while actual controls and supplementary Unicode work. + decoded = re.sub(r"\\u\{([0-9A-Fa-f]+)\}", lambda m: chr(int(m[1], 16)), encoded) + assert decoded == text + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_unverified_text_replacement_is_explicitly_rejected(simulator): + with pytest.raises(NotImplementedError, match="replace-text"): + await simulator.driver.input_text("replacement", clear_existing=True) + simulator.native.call.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key, command", + [ + (KeyCode.HOME, "b h"), + (KeyCode.APP_SWITCH, "b h b h"), + (KeyCode.ENTER, r"sender keyboard kbd \u{000A}"), + ], +) +async def test_supported_native_hardware_and_enter_keys(simulator, key, command): + await simulator.driver.connect() + assert await simulator.driver.press_key(key) + assert simulator.native.call.await_args.args[1]["interactionCommand"] == command + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +async def test_android_back_and_shell_are_not_reported_as_success(simulator): + with pytest.raises(NotImplementedError, match="not supported"): + await simulator.driver.press_key(KeyCode.BACK) + with pytest.raises(NotImplementedError, match="unavailable"): + await simulator.driver.execute_shell("input keyevent BACK") + simulator.native.call.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_missing_hierarchy_prevents_coordinate_input(simulator): + await simulator.driver.connect() + simulator.native.observation.pop("hierarchyPath") + before = len(simulator.native.call.await_args_list) + with pytest.raises(RuntimeError, match="no accessibility hierarchy"): + await simulator.driver.tap(15, 25) + new_calls = simulator.native.call.await_args_list[before:] + assert len(new_calls) == 1 + assert new_calls[0].args[1]["interactionCommand"] == "" + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "operation, arguments", + [ + ("launch_app", ("com.example.app",)), + ("stop_app", ("com.example.app",)), + ("open_url", ("example://",)), + ("list_apps", ()), + ("install_app", (Path("Untrusted.app"),)), + ], +) +async def test_app_operations_require_connected_pinned_device(simulator, operation, arguments): + with pytest.raises(RuntimeError, match="[Cc]onnect"): + await getattr(simulator.driver, operation)(*arguments) + simulator.commands.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_directional_swipe_uses_current_screen_after_rotation(simulator): + await simulator.driver.connect() + Image.new("RGB", (400, 200)).save(simulator.screenshot) + simulator.hierarchy.write_text( + APPLE_HIERARCHY.replace("{200.0, 400.0}", "{400.0, 200.0}"), encoding="utf-8" + ) + assert await simulator.driver.swipe_direction("up") + command = simulator.native.call.await_args.args[1]["interactionCommand"].split() + assert list(map(float, command[1:3] + command[4:6])) == [200, 150, 200, 50] + await simulator.driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action, arguments", [("tap", (15, 25)), ("swipe", (15, 25, 100, 150))]) +async def test_stale_absolute_coordinates_are_rejected_after_rotation(simulator, action, arguments): + await simulator.driver.connect() + Image.new("RGB", (400, 200)).save(simulator.screenshot) + simulator.hierarchy.write_text( + APPLE_HIERARCHY.replace("{200.0, 400.0}", "{400.0, 200.0}"), encoding="utf-8" + ) + simulator.native.call.reset_mock() + with pytest.raises(ValueError, match="changed orientation or size"): + await getattr(simulator.driver, action)(*arguments) + simulator.native.call.assert_awaited_once_with( + "DeviceInteractionSynthesize", + {"interactSessionKey": "test-session", "interactionCommand": ""}, + ) + await simulator.driver.disconnect() + + +def test_apple_frame_formats_and_hitpoints_are_parsed_without_retina_assumptions(): + hierarchy = """Application bundle identifier: com.example.app +UIWindow {{0, 0}, {200, 400}}, hitPoint: {100, 200} + UIButton "Login" {{10, 20}, {60, 30}}, hitPoint: {15, 25} + UIView {{-10, 50}, {20, 20}}, hitPoint: {5, 60} + UIButton "Offscreen" {{300, 500}, {60, 30}} + UIView {{10, 10}, {0, 5}} +""" + elements, scale = parse_hierarchy(hierarchy, 200, 400) + assert scale == (1, 1) + login = next(element for element in elements if element["text"] == "Login") + assert login["class"] == "UIButton" + assert login["bounds"] == "[10,20][70,50]" + assert login["hit_point"] == [15, 25] + assert not any(element["text"] == "Offscreen" for element in elements) + assert len(elements) == 3 + + +def test_known_rotated_window_geometry_is_not_guessed(): + with pytest.raises(RuntimeError, match="matching the screenshot"): + parse_hierarchy("Window {{0, 0}, {400, 200}}", 200, 400) + + +def test_inaccessible_canvas_can_use_native_logical_screenshot_space(): + elements, scale = parse_hierarchy("AX hierarchy is unavailable", 200, 400) + assert elements == [] + assert scale == (1, 1) + + +EMPTY_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "placeholderValue: 'Habit to avoid', Keyboard Focused, hitPoint: {201.0, 210.0}" +) +FILLED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "placeholderValue: 'Habit to avoid', value: caf\u00e9 \U0001f642, " + "Keyboard Focused, hitPoint: {201.0, 210.0}" +) +TRUNCATED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "placeholderValue: 'Habit to avoid', value: Artemis iOS direct..., " + "Keyboard Focused, hitPoint: {201.0, 210.0}" +) +COMMA_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "value: hello, world, Keyboard Focused, hitPoint: {201.0, 210.0}" +) +QUOTED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, " + "value: 'quoted text', Keyboard Focused, hitPoint: {201.0, 210.0}" +) +LABELED_FIELD = ( + "TextField, {{32.0, 199.0}, {338.0, 22.0}}, label: 'Title', " + "value: 'typed', placeholderValue: 'Habit to avoid', hitPoint: {201.0, 210.0}" +) + + +def test_empty_textfield_exposes_placeholder_as_text_and_metadata(): + elements, _ = parse_hierarchy(EMPTY_FIELD, 402, 874) + assert len(elements) == 1 + field = elements[0] + assert field["class"] == "TextField" + assert field["placeholder"] == "Habit to avoid" + assert field["text"] == "Habit to avoid" + assert "value" not in field + assert field["hit_point"] == [201, 210] + + +def test_populated_unlabeled_textfield_exposes_value_as_text_and_metadata(): + elements, _ = parse_hierarchy(FILLED_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "caf\u00e9 \U0001f642" + assert field["text"] == "caf\u00e9 \U0001f642" + assert field["placeholder"] == "Habit to avoid" + assert field["hit_point"] == [201, 210] + + +def test_elided_unquoted_value_is_preserved_verbatim(): + elements, _ = parse_hierarchy(TRUNCATED_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "Artemis iOS direct..." + assert field["text"] == "Artemis iOS direct..." + + +def test_unquoted_value_keeps_ordinary_commas_before_metadata(): + elements, _ = parse_hierarchy(COMMA_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "hello, world" + assert field["text"] == "hello, world" + + +def test_quoted_value_format_still_parsed(): + elements, _ = parse_hierarchy(QUOTED_FIELD, 402, 874) + field = elements[0] + assert field["value"] == "quoted text" + assert field["text"] == "quoted text" + + +def test_labeled_textfield_keeps_label_text_and_retains_field_metadata(): + elements, _ = parse_hierarchy(LABELED_FIELD, 402, 874) + field = elements[0] + assert field["text"] == "Title" + assert field["value"] == "typed" + assert field["placeholder"] == "Habit to avoid" + assert field["hit_point"] == [201, 210] + + +def test_unquoted_native_value_metadata_keeps_label_text(): + line = ( + "Other, {{369.0, 132.0}, {30.0, 414.0}}, " + "label: 'Vertical scroll bar, 1 page', value: 0%, hitPoint: {384.0, 339.0}" + ) + elements, _ = parse_hierarchy(line, 402, 874) + field = elements[0] + assert field["text"] == "Vertical scroll bar, 1 page" + assert field["value"] == "0%" + + +def test_foreground_package_is_unknown_when_multiple_apps_overlap(): + assert application_bundle(APPLE_HIERARCHY) == "com.example.app" + assert ( + application_bundle(APPLE_HIERARCHY + "Application bundle identifier: com.other.app") is None + ) + assert application_bundle("unavailable hierarchy") is None + + +class BlockingProcess: + def __init__(self): + self.returncode = None + self.started = asyncio.Event() + self.communications = 0 + self.kills = 0 + + async def communicate(self): + self.communications += 1 + if self.communications == 1: + self.started.set() + await asyncio.Event().wait() + return b"", b"" + + def kill(self): + self.kills += 1 + self.returncode = -9 + + +@pytest.mark.asyncio +async def test_simctl_timeout_kills_and_reaps_child(monkeypatch): + child = BlockingProcess() + spawn = AsyncMock(return_value=child) + monkeypatch.setattr(xcode_driver.asyncio, "create_subprocess_exec", spawn) + with pytest.raises(TimeoutError): + await run_xcrun("simctl", "bootstatus", IOS_A, "-b", timeout=0.01) + assert child.kills == 1 + assert child.communications == 2 + assert spawn.await_args.args == ("xcrun", "simctl", "bootstatus", IOS_A, "-b") + + +@pytest.mark.asyncio +async def test_simctl_cancellation_kills_and_reaps_child(monkeypatch): + child = BlockingProcess() + monkeypatch.setattr( + xcode_driver.asyncio, "create_subprocess_exec", AsyncMock(return_value=child) + ) + task = asyncio.create_task(run_xcrun("simctl", "launch", IOS_A, "com.example.app")) + await child.started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert child.kills == 1 + assert child.communications == 2 + + +@pytest.mark.asyncio +async def test_simctl_failures_report_stderr_and_preserve_literal_argv(monkeypatch): + child = SimpleNamespace(returncode=1, communicate=AsyncMock(return_value=(b"", b"No app\xff"))) + spawn = AsyncMock(return_value=child) + monkeypatch.setattr(xcode_driver.asyncio, "create_subprocess_exec", spawn) + url = "example://item?command=$(literal)&x='quote'" + with pytest.raises(RuntimeError, match="No app"): + await run_xcrun("simctl", "openurl", IOS_A, url) + assert spawn.await_args.args == ("xcrun", "simctl", "openurl", IOS_A, url) + + +DENIAL = "This agent isn't approved to use Xcode's tools yet. Call XcodeOpenWorkspace first." +PENDING = "Xcode is waiting for the user to approve this request; it has been recorded." + + +def _tool_names(native): + return [call.args[0] for call in native.call.call_args_list] + + +@pytest.mark.asyncio +async def test_initial_approval_refusal_opens_workspace_once_and_retries_start(simulator, tmp_path): + project = tmp_path / "Example App [2].xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + starts = [] + + async def start_hook(arguments): + starts.append(dict(arguments)) + if len(starts) == 1: + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + return native.start_result + + native.hooks["DeviceInteractionStartSession"] = start_hook + driver = XcodeSimulatorDriver(workspace_path=project) + await driver.connect() + assert _tool_names(native) == [ + "DeviceInteractionStartSession", + "XcodeOpenWorkspace", + "DeviceInteractionStartSession", + "DeviceInteractionSynthesize", + ] + assert native.call.call_args_list[1].args == ( + "XcodeOpenWorkspace", + {"path": str(project)}, + ) + assert starts[0] == starts[1] + assert starts[0]["deviceIdentifier"] == IOS_A + invoked = [call.args for call in simulator.commands.call_args_list] + assert ("simctl", "boot") not in [args[:2] for args in invoked] + assert not any("mcp-server" in args for args in invoked) + await driver.disconnect() + assert "DeviceInteractionEndSession" in _tool_names(native) + + +@pytest.mark.asyncio +async def test_pending_workspace_approval_propagates_without_retry(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + async def pending_open(arguments): + raise XcodeApprovalRequiredError("XcodeOpenWorkspace", PENDING) + + native.hooks["DeviceInteractionStartSession"] = fail_start + native.hooks["XcodeOpenWorkspace"] = pending_open + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(XcodeApprovalRequiredError, match="waiting for the user") as caught: + await driver.connect() + assert caught.value.workspace_path == project + assert str(project) in caught.value.guidance + assert "not supplied" not in caught.value.guidance + assert _tool_names(native) == ["DeviceInteractionStartSession", "XcodeOpenWorkspace"] + native.start.assert_awaited_once() + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("identifier", [None, "", 123]) +async def test_malformed_workspace_acceptance_fails_closed(simulator, tmp_path, identifier): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + async def open_workspace(arguments): + return {"workspaceIdentifier": identifier} + + native.hooks["DeviceInteractionStartSession"] = fail_start + native.hooks["XcodeOpenWorkspace"] = open_workspace + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(RuntimeError, match="no usable workspace identifier"): + await driver.connect() + assert _tool_names(native) == ["DeviceInteractionStartSession", "XcodeOpenWorkspace"] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_still_pending_after_workspace_open_stops_after_one_retry(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", PENDING) + + native.hooks["DeviceInteractionStartSession"] = fail_start + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(XcodeApprovalRequiredError, match="waiting for the user") as caught: + await driver.connect() + assert caught.value.workspace_path == project + assert _tool_names(native) == [ + "DeviceInteractionStartSession", + "XcodeOpenWorkspace", + "DeviceInteractionStartSession", + ] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_approval_refusal_without_workspace_propagates_guidance(simulator): + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + native.hooks["DeviceInteractionStartSession"] = fail_start + with pytest.raises(XcodeApprovalRequiredError) as caught: + await simulator.driver.connect() + assert _tool_names(native) == ["DeviceInteractionStartSession"] + assert caught.value.workspace_path is None + assert "Workspace: not supplied" in caught.value.guidance + assert "--ios-workspace" in caught.value.guidance + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_approved_sessions_never_open_workspace(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + simulator.native.tools.add("XcodeOpenWorkspace") + driver = XcodeSimulatorDriver(workspace_path=project) + await driver.connect() + await driver.connect() + names = _tool_names(simulator.native) + assert "XcodeOpenWorkspace" not in names + assert names.count("DeviceInteractionStartSession") == 1 + await driver.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("native boom"), TimeoutError("slow tool")]) +async def test_non_approval_failures_skip_workspace_and_retry(simulator, tmp_path, failure): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + native.tools.add("XcodeOpenWorkspace") + + async def fail(arguments): + raise failure + + native.hooks["DeviceInteractionStartSession"] = fail + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(type(failure)): + await driver.connect() + assert _tool_names(native) == ["DeviceInteractionStartSession"] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_missing_workspace_tool_reports_actionable_approval_error(simulator, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + native = simulator.native + + async def fail_start(arguments): + raise XcodeApprovalRequiredError("DeviceInteractionStartSession", DENIAL) + + native.hooks["DeviceInteractionStartSession"] = fail_start + driver = XcodeSimulatorDriver(workspace_path=project) + with pytest.raises(XcodeApprovalRequiredError, match="XcodeOpenWorkspace") as caught: + await driver.connect() + assert caught.value.workspace_path == project + assert str(project) in caught.value.guidance + assert _tool_names(native) == ["DeviceInteractionStartSession"] + native.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_workspace_must_be_existing_project_directory(simulator, tmp_path): + (tmp_path / "plain-folder").mkdir() + (tmp_path / "file.xcodeproj").write_text("x") + bad_paths = [ + tmp_path / "missing.xcodeproj", + tmp_path / "plain-folder", + tmp_path / "file.xcodeproj", + ] + for path in bad_paths: + driver = XcodeSimulatorDriver(workspace_path=path) + with pytest.raises(ValueError, match="xcodeproj or .xcworkspace"): + await driver.connect() + simulator.commands.assert_not_called() + simulator.native.start.assert_not_called() + + workspace = tmp_path / "Nested Dir [x].xcworkspace" + workspace.mkdir() + driver = XcodeSimulatorDriver(workspace_path=workspace) + await driver.connect() + await driver.disconnect() + + +@pytest.mark.asyncio +async def test_duplicate_simulator_names_reject_instead_of_first_matching(simulator): + """Two same-named sims must not silently bind — pin the UDID instead.""" + simulator.driver._device_id = "Office iPhone" + simulator.inventory["devices"]["com.apple.CoreSimulator.SimRuntime.iOS-27-0"] = [ + device(IOS_A) | {"name": "Office iPhone"}, + device(IOS_B, state="Shutdown") | {"name": "Office iPhone"}, + ] + with pytest.raises(ValueError, match="UDID"): + await simulator.driver.connect() + simulator.native.start.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_exact_simulator_name_resolves_to_its_udid(simulator): + simulator.driver._device_id = "iPhone" + await simulator.driver.connect() + assert simulator.driver.device_id == IOS_A + await simulator.driver.disconnect() diff --git a/tests/unit/drivers/test_ios_physical.py b/tests/unit/drivers/test_ios_physical.py new file mode 100644 index 00000000..0546c80e --- /dev/null +++ b/tests/unit/drivers/test_ios_physical.py @@ -0,0 +1,1657 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical iOS device behavior tested without attached hardware.""" + +import asyncio +import io +import json +from pathlib import Path +import plistlib +import threading +from types import SimpleNamespace + +import pytest + +from artemis.drivers.ios import discovery, physical_driver, physical_recording, wda +from artemis.drivers.ios.physical_driver import PhysicalIosDriver +from artemis.drivers.ios.physical_recording import PhysicalIosRecorder +from artemis.drivers.ios.wda import WdaClient + + +IPHONE_UDID = "00008130-0000ABCD1234AAAA" +IPAD_UDID = "00008103-0000ABCD1234BBBB" +SIM_UDID = "DE345DD3-5792-4DAD-B863-144682629565" +WATCH_UDID = "00008301-0000ABCD1234CCCC" + + +def _devicectl_device( + udid, + name="Device", + os_version="27.0", + platform="iOS", + reality="physical", + tunnel="connected", + pairing="paired", + visibility="default", +): + """One device entry in devicectl's deprecated split-properties shape.""" + return { + "deviceProperties": {"name": name, "osVersionNumber": os_version}, + "hardwareProperties": {"udid": udid, "platform": platform, "reality": reality}, + "connectionProperties": {"tunnelState": tunnel, "pairingState": pairing}, + "visibilityClass": visibility, + } + + +def _devicectl_payload(*devices): + return {"info": {}, "result": {"devices": list(devices)}} + + +def _parsed(*devices): + """Flattened entries as `parse_devicectl_devices` emits them.""" + return discovery.parse_devicectl_devices(_devicectl_payload(*devices)) + + +PHYSICAL_IPHONE = _devicectl_device(IPHONE_UDID, name="Jane's iPhone") +OFFLINE_IPHONE = _devicectl_device(IPHONE_UDID, name="iPhone", tunnel="disconnected") +UNPAIRED_IPHONE = _devicectl_device(IPHONE_UDID, name="iPhone", pairing="unpaired") +SIMULATOR = _devicectl_device( + SIM_UDID, name="iPhone 18 Pro", reality="simulated", visibility="simulators" +) +WATCH = _devicectl_device(WATCH_UDID, name="Watch", platform="watchOS") + + +def test_devicectl_devices_flatten_physical_and_simulator_entries(): + devices = discovery.parse_devicectl_devices( + _devicectl_payload(PHYSICAL_IPHONE, SIMULATOR, WATCH) + ) + by_udid = {device["udid"]: device for device in devices} + assert by_udid[IPHONE_UDID]["reality"] == "physical" + assert by_udid[IPHONE_UDID]["platform"] == "iOS" + assert by_udid[SIM_UDID]["reality"] == "simulated" + assert by_udid[WATCH_UDID]["platform"] == "watchOS" + + +def test_devicectl_devices_read_the_nested_properties_shape(): + """devicectl is migrating split property dicts into one `properties` map.""" + nested = { + "properties": { + "deviceProperties": {"name": "iPhone", "osVersionNumber": "27.0"}, + "hardwareProperties": {"udid": IPHONE_UDID, "platform": "iOS", "reality": "physical"}, + "connectionProperties": {"tunnelState": "connected", "pairingState": "paired"}, + } + } + devices = discovery.parse_devicectl_devices(_devicectl_payload(nested)) + assert devices == [ + { + "udid": IPHONE_UDID, + "name": "iPhone", + "os_version": "27.0", + "platform": "iOS", + "reality": "physical", + "product_type": None, + "connection_state": "connected", + "pairing_state": "paired", + "visibility": None, + } + ] + + +def test_devicectl_devices_skip_entries_without_udid(): + devices = discovery.parse_devicectl_devices( + _devicectl_payload({"hardwareProperties": {"platform": "iOS"}}, PHYSICAL_IPHONE) + ) + assert [device["udid"] for device in devices] == [IPHONE_UDID] + + +def test_physical_classification_requires_ios_and_hardware(): + assert discovery.is_physical_ios({"platform": "iOS", "reality": "physical"}) + assert not discovery.is_physical_ios({"platform": "iOS", "reality": "simulated"}) + assert not discovery.is_physical_ios({"platform": "watchOS", "reality": "physical"}) + assert not discovery.is_physical_ios({"platform": "iOS"}) + + +def test_find_physical_device_matches_udid_case_insensitively(monkeypatch): + monkeypatch.setattr( + discovery, + "list_core_devices_sync", + lambda force_refresh=False: _parsed(PHYSICAL_IPHONE, SIMULATOR), + ) + found = discovery.find_physical_ios_device_sync(IPHONE_UDID.lower()) + assert found is not None and found["udid"] == IPHONE_UDID + assert discovery.find_physical_ios_device_sync("Jane's iPhone") is not None + assert discovery.find_physical_ios_device_sync(SIM_UDID) is None + assert discovery.find_physical_ios_device_sync("missing") is None + + +def test_find_physical_device_survives_enumeration_failure(monkeypatch): + monkeypatch.setattr(discovery, "list_core_devices_sync", lambda force_refresh=False: None) + assert discovery.find_physical_ios_device_sync(IPHONE_UDID) is None + + +@pytest.fixture +def driver(monkeypatch): + instance = PhysicalIosDriver(device_id=IPHONE_UDID) + monkeypatch.setattr(instance, "_require_ios_host", _async_return(None)) + return instance + + +@pytest.fixture(autouse=True) +def _clean_wda_env(monkeypatch): + for key in ( + "ARTEMIS_IOS_WDA_URL", + "ARTEMIS_IOS_WDA_HOST", + "ARTEMIS_IOS_WDA_XCTESTRUN", + "ARTEMIS_IOS_WDA_BUNDLE_ID", + ): + monkeypatch.delenv(key, raising=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("serial", ["booted", " Booted ", "BOOTED", ""]) +async def test_physical_resolve_rejects_booted_and_empty_serials(driver, serial): + driver._device_id = serial + with pytest.raises(ValueError, match="device-serial"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_physical_resolve_pins_the_udid(driver, monkeypatch): + monkeypatch.setattr( + physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE, SIMULATOR)) + ) + candidate = await driver._resolve_device() + assert candidate["udid"] == IPHONE_UDID + assert driver.device_id == IPHONE_UDID + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_simulator_udids(driver, monkeypatch): + driver._device_id = SIM_UDID + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(SIMULATOR))) + with pytest.raises(ValueError, match="Simulator"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_non_ios_hardware(driver, monkeypatch): + driver._device_id = WATCH_UDID + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(_parsed(WATCH))) + with pytest.raises(ValueError, match="paired physical iOS"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_physical_resolve_reports_enumeration_failure(driver, monkeypatch): + monkeypatch.setattr(physical_driver, "list_core_devices", _async_devices(None)) + with pytest.raises(RuntimeError, match="devicectl"): + await driver._resolve_device() + + +@pytest.mark.asyncio +async def test_offline_device_fails_with_actionable_error(driver): + with pytest.raises(RuntimeError, match="not connected|Developer Mode"): + await driver._prepare_device(_parsed(OFFLINE_IPHONE)[0]) + + +@pytest.mark.asyncio +async def test_unpaired_device_fails_with_trust_guidance(driver): + with pytest.raises(RuntimeError, match="not paired|Trust"): + await driver._prepare_device(_parsed(UNPAIRED_IPHONE)[0]) + + +@pytest.mark.asyncio +async def test_connected_device_needs_no_boot(driver): + await driver._prepare_device(_parsed(PHYSICAL_IPHONE)[0]) + + +def _async_devices(result): + async def _list(force_refresh=False): + return result + + return _list + + +def _xcrun_payload_writer(payloads): + """Fake run_xcrun that honors ``--json-output `` vs ``-``. + + ``payloads`` maps a devicectl subcommand ("apps", "processes", "details", + "launch", ...) to the dict written into the output file. Anything else + lands on stdout so ``--json-output -`` callers get JSON bytes. + """ + + async def fake(*arguments, timeout=30.0): + args = list(arguments) + if "--json-output" in args: + target = args[args.index("--json-output") + 1] + keys = [a for a in args if a in payloads] + payload = payloads.get(keys[0], {}) if keys else {} + if target == "-": + return json.dumps(payload).encode() + Path(target).write_text(json.dumps(payload), encoding="utf-8") + return b"" + return b"" + + return fake + + +class _FakeWda: + """In-memory WebDriverAgent stand-in for driver interaction tests.""" + + def __init__( + self, + window=(100.0, 200.0), + tree=None, + device_name="Jane's iPhone", + is_simulator=False, + ): + self.session_id = None + self.base_url = "http://fake-wda:8100" + self.window = window + self.tree = tree if tree is not None else {"type": "Application", "children": []} + self.device_name = device_name + self.is_simulator = is_simulator + self.opened_sessions = 0 + self.tapped: list[tuple[float, float, int]] = [] + self.swiped: list[tuple[float, float, float, float, int]] = [] + self.typed: list[str] = [] + self.buttons: list[str] = [] + self.homescreen_calls = 0 + self._png = self._make_png() + self._closed = 0 + + @staticmethod + def _make_png(width: int = 300, height: int = 600) -> bytes: + from PIL import Image + + buffer = io.BytesIO() + Image.new("RGB", (width, height), color="red").save(buffer, format="PNG") + return buffer.getvalue() + + async def device_info(self, timeout=10.0): + return {"name": self.device_name, "isSimulator": self.is_simulator} + + async def open_session(self, adopt_existing=False): + self.adopt_requested = adopt_existing + self.opened_sessions += 1 + self.session_id = "wda-session" + return self.session_id + + async def close_session(self): + self._closed += 1 + self.session_id = None + + async def screenshot_png(self): + return self._png + + async def window_size(self): + return self.window + + async def source_json(self): + return self.tree + + async def active_app(self): + return "com.example.foreground" + + async def tap(self, x, y, hold_ms=0): + self.tapped.append((x, y, hold_ms)) + + async def swipe(self, sx, sy, ex, ey, duration_ms): + self.swiped.append((sx, sy, ex, ey, duration_ms)) + + async def type_text(self, text): + self.typed.append(text) + + async def press_button(self, name): + self.buttons.append(name) + return True + + async def homescreen(self): + self.homescreen_calls += 1 + + +@pytest.fixture +def connected_driver(): + """A driver with a stub WDA session and a known 300x600 observation.""" + driver = PhysicalIosDriver(device_id=IPHONE_UDID) + driver._session_key = "wda-session" + driver._wda = _FakeWda() + # Pretend one capture already ran: the orientation guard compares sizes. + driver._width, driver._height = 300, 600 + return driver + + +@pytest.mark.asyncio +async def test_launch_records_the_pid_from_devicectl_json(connected_driver, monkeypatch): + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return json.dumps( + { + "result": { + "process": { + "processIdentifier": 4242, + "executable": "file:///Apps/Example.app/Example", + } + } + } + ).encode() + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.launch_app("com.example.app") + assert calls[0][:5] == ("devicectl", "device", "process", "launch", "--device") + assert IPHONE_UDID in calls[0] + assert calls[0][-1] == "com.example.app" + assert connected_driver._launched_pids == {"com.example.app": 4242} + + +@pytest.mark.asyncio +async def test_stop_app_terminates_the_tracked_pid(connected_driver, monkeypatch): + """A cached launch pid wins only after it is re-verified against the app's URL.""" + connected_driver._launched_pids["com.example.app"] = 4242 + calls = [] + payloads = { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": { + "result": { + "runningProcesses": [ + {"processIdentifier": 7, "executable": "file:///usr/libexec/other"}, + { + "processIdentifier": 4242, + "executable": "file:///private/var/containers/X/Example.app/Example", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.stop_app("com.example.app") + terminate = next(c for c in calls if "terminate" in c) + assert terminate[:5] == ["devicectl", "device", "process", "terminate", "--device"] + assert terminate[terminate.index("--pid") + 1] == "4242" + assert connected_driver._launched_pids == {} + + +@pytest.mark.asyncio +async def test_stop_app_ignores_a_stale_cached_pid(connected_driver, monkeypatch): + """A recycled cached pid now owned by a different app must not be killed.""" + connected_driver._launched_pids["com.example.app"] = 4242 + calls = [] + payloads = { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": { + "result": { + "runningProcesses": [ + # pid 4242 was recycled into an unrelated binary. + { + "processIdentifier": 4242, + "executable": "file:///usr/sbin/otherd", + }, + { + "processIdentifier": 42, + "executable": "file:///var/containers/X/Example.app/Example", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.stop_app("com.example.app") + terminated = [c[c.index("--pid") + 1] for c in calls if "terminate" in c] + assert terminated == ["42"] + + +@pytest.mark.asyncio +async def test_stop_app_never_matches_sibling_app_prefixes(connected_driver, monkeypatch): + """'Example.appOther' must not match the 'Example.app' URL prefix.""" + calls = [] + payloads = { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": { + "result": { + "runningProcesses": [ + { + "processIdentifier": 99, + "executable": "file:///var/containers/X/Example.appOther/Other", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + with pytest.raises(ValueError, match="No running process"): + await connected_driver.stop_app("com.example.app") + assert not any("terminate" in c for c in calls) + + +@pytest.mark.asyncio +async def test_stop_app_rejects_a_missing_app_url(connected_driver, monkeypatch): + """An installed-app entry without a usable URL cannot verify a victim.""" + calls = [] + payloads = { + "apps": {"result": {"apps": [{"bundleIdentifier": "com.example.app", "url": None}]}}, + "processes": { + "result": { + "runningProcesses": [ + { + "processIdentifier": 42, + "executable": "file:///var/containers/X/Example.app/Example", + }, + ] + } + }, + } + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(list(arguments)) + args = list(arguments) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + keys = [a for a in args if a in payloads] + Path(args[args.index("--json-output") + 1]).write_text( + json.dumps(payloads.get(keys[0], {}) if keys else {}) + ) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + with pytest.raises(ValueError, match="not installed|could not be determined"): + await connected_driver.stop_app("com.example.app") + assert not any("terminate" in c for c in calls) + + +@pytest.mark.asyncio +async def test_stop_app_without_pid_scans_running_processes(connected_driver, monkeypatch): + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + args = list(arguments) + calls.append(args) + if "--json-output" in args and args[args.index("--json-output") + 1] != "-": + if "apps" in args: + payload = { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + } + else: + payload = { + "result": { + "runningProcesses": [ + { + "processIdentifier": 7, + "executable": "file:///usr/libexec/other", + }, + { + "processIdentifier": 42, + "executable": "file:///var/containers/X/Example.app/Example", + }, + ] + } + } + Path(args[args.index("--json-output") + 1]).write_text(json.dumps(payload)) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.stop_app("com.example.app") + assert calls[0][2:4] == ["info", "apps"] + assert calls[1][2:4] == ["info", "processes"] + terminate = calls[2] + assert terminate[:5] == ["devicectl", "device", "process", "terminate", "--device"] + assert terminate[terminate.index("--pid") + 1] == "42" + + +@pytest.mark.asyncio +async def test_stop_app_without_process_fails_clearly(connected_driver, monkeypatch): + monkeypatch.setattr( + physical_driver, + "run_xcrun", + _xcrun_payload_writer( + { + "apps": { + "result": { + "apps": [ + { + "bundleIdentifier": "com.example.app", + "url": "file:///var/containers/X/Example.app/", + } + ] + } + }, + "processes": {"result": {"runningProcesses": []}}, + } + ), + ) + with pytest.raises(ValueError, match="No running process"): + await connected_driver.stop_app("com.example.app") + + +@pytest.mark.asyncio +async def test_install_accepts_signed_app_directories(connected_driver, monkeypatch, tmp_path): + app = tmp_path / "Writer.app" + app.mkdir() + (app / "Info.plist").write_bytes(plistlib.dumps({"CFBundleIdentifier": "com.example.writer"})) + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.install_app(app) == "com.example.writer" + assert calls[0][:4] == ("devicectl", "device", "install", "app") + assert str(app) in calls[0] + + +@pytest.mark.asyncio +async def test_install_accepts_ipa_files(connected_driver, monkeypatch, tmp_path): + ipa = tmp_path / "Writer.ipa" + ipa.write_bytes(b"PK") + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.install_app(ipa) == "Writer" + + +@pytest.mark.asyncio +async def test_install_rejects_simulator_built_artifacts(connected_driver, tmp_path): + with pytest.raises(ValueError, match="signed .app"): + await connected_driver.install_app(tmp_path / "Writer.zip") + + +@pytest.mark.asyncio +async def test_open_url_uses_devicectl(connected_driver, monkeypatch): + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"{}" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + assert await connected_driver.open_url("example://open") + assert calls[0][:5] == ("devicectl", "device", "process", "openURL", "--device") + assert calls[0][-1] == "example://open" + + +@pytest.mark.asyncio +async def test_list_apps_parses_devicectl_json(connected_driver, monkeypatch): + payload = { + "apps": [ + {"bundleIdentifier": "com.example.writer", "name": "Writer"}, + {"bundleIdentifier": "com.example.other"}, + ] + } + + async def fake_xcrun(*arguments, timeout=30.0): + args = list(arguments) + if "--json-output" in args: + Path(args[args.index("--json-output") + 1]).write_text(json.dumps({"result": payload})) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + apps = await connected_driver.list_apps() + assert apps == {"com.example.writer": "Writer", "com.example.other": "com.example.other"} + + +@pytest.mark.asyncio +async def test_shell_commands_stay_rejected(connected_driver): + with pytest.raises(NotImplementedError): + await connected_driver.execute_shell("ls") + + +@pytest.mark.asyncio +async def test_recording_requires_connection(): + driver = PhysicalIosDriver(device_id=IPHONE_UDID) + with pytest.raises(RuntimeError, match="Connect"): + await driver.start_video_recording() + + +# --- Physical recorder ------------------------------------------------------ + + +def _png(path: Path, width: int = 100, height: int = 200) -> None: + from PIL import Image + + Image.new("RGB", (width, height), color="red").save(path) + + +@pytest.mark.asyncio +async def test_recorder_polls_frames_and_assembles_mp4(tmp_path, monkeypatch): + """Frames become one timestamped segment through the ffconcat demuxer.""" + ffmpeg_calls = [] + + async def fake_screenshot(device_id, destination, timeout=30.0): + # Pace like real devicectl round-trips; an instant fake lets the poll + # loop flood tmp_path with frames and stalls cleanup for minutes. + await asyncio.sleep(0.02) + _png(destination) + + async def fake_ffmpeg(arguments): + ffmpeg_calls.append(arguments) + output = Path(arguments[-1]) + output.write_bytes(b"mp4") + return 0, b"" + + async def fake_probe(path, timeout_seconds=None): + return {"duration": 1.0, "width": 100, "height": 200} + + async def fake_manifest(output_dir, paths, offsets, probe_timeout_seconds=None): + manifest = Path(output_dir) / "manifest.json" + manifest.write_text("{}") + return manifest + + monkeypatch.setattr(physical_recording, "devicectl_screenshot", fake_screenshot) + monkeypatch.setattr(physical_recording, "_run_ffmpeg", fake_ffmpeg) + monkeypatch.setattr(physical_recording, "probe_video_segment", fake_probe) + monkeypatch.setattr(physical_recording, "write_recording_manifest", fake_manifest) + recorder = PhysicalIosRecorder(IPHONE_UDID) + session = await recorder.start(tmp_path) + path = await recorder.stop() + assert path is not None and path.suffix == ".mp4" + assert any("frames.txt" in str(arg) for call in ffmpeg_calls for arg in call) + + +@pytest.mark.asyncio +async def test_recorder_partitions_frames_at_the_seam(tmp_path, monkeypatch): + async def fake_screenshot(device_id, destination, timeout=30.0): + await asyncio.sleep(0.02) + _png(destination) + + monkeypatch.setattr(physical_recording, "devicectl_screenshot", fake_screenshot) + recorder = PhysicalIosRecorder(IPHONE_UDID) + session = await recorder.start(tmp_path) + anchor = session.anchor_monotonic + seam = anchor + 5.0 + session.frames = [ + {"path": tmp_path / "frames_0000" / "f1.png", "at": seam - 2}, + {"path": tmp_path / "frames_0000" / "f2.png", "at": seam - 1}, + {"path": tmp_path / "frames_0000" / "f3.png", "at": seam + 1}, + ] + record = recorder._seal_current_segment(session, seam) + session.poll_task.cancel() + assert len(record["frames"]) == 2 + assert len(session.frames) == 1 + + +@pytest.mark.asyncio +async def test_recorder_fails_closed_when_devicectl_never_delivers(tmp_path, monkeypatch): + async def fake_screenshot(device_id, destination, timeout=30.0): + await asyncio.sleep(0.01) + raise RuntimeError("device disconnected") + + monkeypatch.setattr(physical_recording, "devicectl_screenshot", fake_screenshot) + monkeypatch.setattr(physical_recording, "MAX_CONSECUTIVE_FAILURES", 2) + recorder = PhysicalIosRecorder(IPHONE_UDID) + with pytest.raises(RuntimeError): + await recorder.start(tmp_path) + session = recorder.session + assert session is not None and not session.is_active + assert session.errors + + +# --- WebDriverAgent client and wiring -------------------------------------- + + +def test_normalize_wda_url_accepts_hosts_ips_and_urls(): + assert wda.normalize_wda_url("192.168.0.5") == "http://192.168.0.5:8100" + assert wda.normalize_wda_url("fd20::1") == "http://[fd20::1]:8100" + assert wda.normalize_wda_url("http://10.0.0.2:9000/") == "http://10.0.0.2:9000" + assert wda.normalize_wda_url("localhost:8100") == "http://localhost:8100" + + +def test_wda_url_candidates_order_env_host_tunnel_localhost(monkeypatch): + monkeypatch.delenv(wda.WDA_URL_ENV, raising=False) + monkeypatch.delenv(wda.WDA_HOST_ENV, raising=False) + candidates = wda.wda_url_candidates(tunnel_ip="fd20:85a4::1") + assert candidates == ["http://[fd20:85a4::1]:8100", "http://127.0.0.1:8100"] + monkeypatch.setenv(wda.WDA_URL_ENV, "http://10.1.1.1:8100") + monkeypatch.setenv(wda.WDA_HOST_ENV, "phone.lan") + assert wda.wda_url_candidates(tunnel_ip="fd20:85a4::1") == [ + "http://10.1.1.1:8100", + "http://phone.lan:8100", + "http://[fd20:85a4::1]:8100", + "http://127.0.0.1:8100", + ] + + +def test_parse_wda_elements_maps_tree_to_ui_elements(): + tree = { + "type": "XCUIElementTypeApplication", + "rect": {"x": 0, "y": 0, "width": 100, "height": 200}, + "children": [ + { + "type": "XCUIElementTypeButton", + "name": "saveButton", + "label": "Save", + "rect": {"x": 10, "y": 20, "width": 30, "height": 10}, + "isVisible": True, + "children": [], + }, + { + "type": "XCUIElementTypeStaticText", + "label": "Title", + "value": "hello", + "rect": {"x": 0, "y": 0, "width": 50, "height": 10}, + "children": [], + }, + {"type": "XCUIElementTypeOther", "rect": {"x": 0, "y": 0, "width": 0, "height": 0}}, + ], + } + elements = wda.parse_wda_elements(tree, (3.0, 3.0), 300, 600) + assert len(elements) == 3 # root window + two children with geometry + button = next(e for e in elements if e["resource_id"] == "saveButton") + assert button["text"] == "Save" + assert button["class"] == "Button" + assert button["parsed_bounds"] == {"left": 30, "top": 60, "right": 120, "bottom": 90} + assert button["hit_point"] == [75, 75] + label = next(e for e in elements if e["text"] == "Title") + assert label["value"] == "hello" + + +@pytest.mark.asyncio +async def test_wda_client_unwraps_values_and_raises_errors(monkeypatch): + client = WdaClient("http://wda.test:8100") + + class _Response: + def __init__(self, payload): + self._raw = json.dumps(payload).encode() + + def read(self): + return self._raw + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def fake_urlopen(request, timeout): + if request.full_url.endswith("/status"): + return _Response({"value": {"ready": True}}) + if request.full_url.endswith("/session"): + return _Response({"value": {"sessionId": "abc-123"}}) + return _Response({"value": {"error": "no such", "message": "nope"}}) + + monkeypatch.setattr(wda.urllib.request, "urlopen", fake_urlopen) + assert await client.status() == {"ready": True} + assert await client.open_session() == "abc-123" + with pytest.raises(RuntimeError, match="nope"): + await client.window_size() + + +@pytest.mark.asyncio +async def test_ensure_wda_returns_probed_client(driver, monkeypatch): + client = _FakeWda() + monkeypatch.setattr(physical_driver, "wda_url_candidates", lambda **kw: ["http://a:8100"]) + monkeypatch.setattr(physical_driver, "probe_wda", _async_return(client)) + monkeypatch.setattr(driver, "_tunnel_ip", _async_return("fd20::1")) + assert await driver._ensure_wda() is client + + +@pytest.mark.asyncio +async def test_ensure_wda_launches_runner_then_probes(driver, monkeypatch): + client = _FakeWda() + probes = [] + + async def probe(candidates, timeout=5.0): + probes.append(candidates) + return client if len(probes) > 1 else None + + async def apps_payload(*arguments): + return {"apps": [{"bundleIdentifier": "com.artemis.WebDriverAgentRunner.xctrunner"}]} + + monkeypatch.setattr(physical_driver, "wda_url_candidates", lambda **kw: ["http://a:8100"]) + monkeypatch.setattr(physical_driver, "probe_wda", probe) + monkeypatch.setattr(driver, "_tunnel_ip", _async_return(None)) + monkeypatch.setattr(driver, "_devicectl_json", apps_payload) + monkeypatch.setattr(driver, "_launch_bundle", _async_return(777)) + assert await driver._ensure_wda() is client + assert driver._wda_runner_pid == 777 + + +@pytest.mark.asyncio +async def test_ensure_wda_without_runner_or_server_fails(driver, monkeypatch): + monkeypatch.setattr(physical_driver, "wda_url_candidates", lambda **kw: ["http://a:8100"]) + monkeypatch.setattr(physical_driver, "probe_wda", _async_return(None)) + monkeypatch.setattr(driver, "_tunnel_ip", _async_return(None)) + monkeypatch.setattr(driver, "_devicectl_json", _async_return({"apps": []})) + with pytest.raises(RuntimeError, match="WebDriverAgent"): + await driver._ensure_wda() + + +@pytest.mark.asyncio +async def test_capture_maps_wda_tree_and_screenshot(connected_driver): + connected_driver._wda = _FakeWda( + window=(100.0, 200.0), + tree={ + "type": "XCUIElementTypeApplication", + "rect": {"x": 0, "y": 0, "width": 100, "height": 200}, + "children": [ + { + "type": "XCUIElementTypeButton", + "label": "OK", + "rect": {"x": 10, "y": 10, "width": 20, "height": 20}, + } + ], + }, + ) + data = await connected_driver.get_screen_data() + assert data.platform == "ios" + assert (data.width, data.height) == (300, 600) + assert connected_driver._scale == (3.0, 3.0) + button = next(e for e in data.ui_elements if e["text"] == "OK") + assert button["hit_point"] == [60, 60] + + +@pytest.mark.asyncio +async def test_tap_converts_pixels_to_wda_points(connected_driver): + await connected_driver.tap(150, 300) + assert connected_driver._wda.tapped == [(50.0, 100.0, 100)] + + +@pytest.mark.asyncio +async def test_swipe_converts_endpoints(connected_driver): + await connected_driver.swipe(30, 300, 150, 60, duration_ms=500) + assert connected_driver._wda.swiped == [(10.0, 100.0, 50.0, 20.0, 500)] + + +@pytest.mark.asyncio +async def test_input_text_and_keys_route_to_wda(connected_driver): + await connected_driver.input_text("hi", clear_existing=False) + assert connected_driver._wda.typed == ["hi"] + await connected_driver.press_key("volume_up") + assert connected_driver._wda.buttons == ["volumeUp"] + await connected_driver.press_key("home") + assert connected_driver._wda.buttons == ["volumeUp", "home"] + with pytest.raises(NotImplementedError, match="typing appends"): + await connected_driver.input_text("x", clear_existing=True) + + +@pytest.mark.asyncio +async def test_current_package_uses_wda_active_app(connected_driver): + assert await connected_driver.get_current_package() == "com.example.foreground" + + +@pytest.mark.asyncio +async def test_disconnect_closes_wda_and_terminates_runner(connected_driver, monkeypatch): + connected_driver._wda_runner_pid = 777 + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + await connected_driver.disconnect() + assert connected_driver._wda is None + assert connected_driver._session_key is None + assert connected_driver._wda_runner_pid is None + assert any("terminate" in args and str(777) in args for args in calls) + + +@pytest.mark.asyncio +async def test_connect_establishes_wda_session(driver, monkeypatch): + client = _FakeWda() + monkeypatch.setattr( + physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE)) + ) + monkeypatch.setattr(driver, "_ensure_wda", _async_return(client)) + monkeypatch.setattr(driver, "_require_ios_host", _async_return(None)) + await driver.connect() + assert driver._wda is client + assert driver._session_key == "wda-session" + + +def _async_return(value): + async def _inner(*args, **kwargs): + return value + + return _inner + + +# --- Factory routing -------------------------------------------------------- + + +def test_factory_routes_physical_udids_to_the_physical_driver(monkeypatch): + import artemis.drivers.factory as factory + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="ios", + device_id=IPHONE_UDID, + device_width=1179, + device_height=2556, + ), + agent_config=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.setattr( + discovery, "list_ios_simulators_sync", lambda force_refresh=False: _parsed(SIMULATOR) + ) + monkeypatch.setattr( + discovery, "find_physical_ios_device_sync", lambda identifier: _parsed(PHYSICAL_IPHONE)[0] + ) + driver = factory.create_driver(ctx) + assert isinstance(driver, PhysicalIosDriver) + assert driver.device_id == IPHONE_UDID + + +def test_factory_keeps_simulator_udids_on_the_simulator_driver(monkeypatch): + import artemis.drivers.factory as factory + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="ios", + device_id=SIM_UDID, + device_width=1179, + device_height=2556, + ), + agent_config=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.setattr( + discovery, "list_ios_simulators_sync", lambda force_refresh=False: _parsed(SIMULATOR) + ) + driver = factory.create_driver(ctx) + assert isinstance(driver, XcodeSimulatorDriver) + assert not isinstance(driver, PhysicalIosDriver) + + +def test_factory_defaults_unknown_serials_to_simulator_validation(monkeypatch): + """Unrecognized serials keep the simulator driver so its error path applies.""" + import artemis.drivers.factory as factory + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="ios", + device_id="unknown-serial", + device_width=1179, + device_height=2556, + ), + agent_config=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.setattr(discovery, "list_ios_simulators_sync", lambda force_refresh=False: []) + monkeypatch.setattr(discovery, "find_physical_ios_device_sync", lambda identifier: None) + driver = factory.create_driver(ctx) + assert type(driver) is XcodeSimulatorDriver + + +def test_factory_cloud_mode_still_rejects_ios(monkeypatch): + import artemis.drivers.factory as factory + + ctx = SimpleNamespace( + device=SimpleNamespace(mobile_platform="ios", device_id=IPHONE_UDID), + agent_config=None, + ) + monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1") + with pytest.raises(ValueError, match="local only"): + factory.create_driver(ctx) + + +def test_android_selection_is_unchanged(monkeypatch): + import artemis.drivers.factory as factory + from artemis.drivers.android.adb_driver import AndroidAdbDriver + + ctx = SimpleNamespace( + device=SimpleNamespace( + mobile_platform="android", + device_id="emulator-5554", + device_width=1080, + device_height=2400, + ), + agent_config=None, + adb_client=None, + ui_adb_client=None, + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.delenv("ARTEMIS_MOCK_DRIVER", raising=False) + driver = factory.create_driver(ctx) + assert isinstance(driver, AndroidAdbDriver) + + +# --- Xcode 27 devicectl schema and ambiguity -------------------------------- + + +def _modern_devicectl_device( + udid, + name="Device", + os_version="27.0", + platform="iOS", + reality="physical", + state="connected", + pairing="paired", + visibility="default", +): + """One entry in Xcode 27's modern ``properties``-only devicectl shape.""" + return { + "identifier": udid, + "properties": { + "hardware": { + "udid": udid, + "platform": platform, + "reality": reality, + "productType": "iPhone17,2", + }, + "state": {"name": name, "visibilityClass": visibility}, + "software": { + "osVersionNumber": { + "components": [27, 0, 0], + "stringValue": os_version, + } + }, + "connection": {"pairingState": pairing, "state": state}, + }, + } + + +def test_devicectl_devices_read_the_modern_properties_shape(): + modern = _modern_devicectl_device(IPHONE_UDID, name="Test Phone") + devices = discovery.parse_devicectl_devices(_devicectl_payload(modern)) + assert devices == [ + { + "udid": IPHONE_UDID, + "name": "Test Phone", + "os_version": "27.0", + "platform": "iOS", + "reality": "physical", + "product_type": "iPhone17,2", + "connection_state": "connected", + "pairing_state": "paired", + "visibility": "default", + } + ] + + +def test_devicectl_devices_tolerate_null_modern_sections(): + modern = _modern_devicectl_device(IPHONE_UDID) + modern["properties"]["software"] = None + modern["properties"]["state"] = None + devices = discovery.parse_devicectl_devices(_devicectl_payload(modern)) + assert devices[0]["udid"] == IPHONE_UDID + assert devices[0]["os_version"] is None + assert devices[0]["name"] is None + + +def test_devicectl_devices_prefer_legacy_values_when_both_present(): + device = _devicectl_device(IPHONE_UDID, name="Legacy Name", os_version="26.0") + device["properties"] = _modern_devicectl_device( + IPHONE_UDID, name="Modern Name", os_version="27.0" + )["properties"] + parsed = _parsed(device)[0] + assert parsed["name"] == "Legacy Name" + assert parsed["os_version"] == "26.0" + + +def test_find_physical_rejects_ambiguous_names(monkeypatch): + monkeypatch.setattr( + discovery, + "list_core_devices_sync", + lambda force_refresh=False: _parsed( + _devicectl_device(IPHONE_UDID, name="Office iPhone"), + _devicectl_device(IPAD_UDID, name="Office iPhone", platform="iPadOS"), + ), + ) + with pytest.raises(ValueError, match="UDID"): + discovery.find_physical_ios_device_sync("Office iPhone") + # The exact UDID still resolves uniquely. + found = discovery.find_physical_ios_device_sync(IPAD_UDID) + assert found is not None and found["udid"] == IPAD_UDID + + +@pytest.mark.asyncio +async def test_physical_resolve_rejects_duplicate_names(driver, monkeypatch): + driver._device_id = "Office iPhone" + monkeypatch.setattr( + physical_driver, + "list_core_devices", + _async_devices( + _parsed( + _devicectl_device(IPHONE_UDID, name="Office iPhone"), + _devicectl_device(IPAD_UDID, name="Office iPhone", platform="iPadOS"), + ) + ), + ) + with pytest.raises(ValueError, match="UDID"): + await driver._resolve_device() + + +# --- WebDriverAgent session safety ----------------------------------------- # + + +class _WdaResponder: + """Scripted urllib stub that records requests and can block on demand.""" + + def __init__(self): + self.requests: list[tuple[str, str]] = [] + self.responses: dict[str, object] = {} + self.blockers: dict[str, threading.Event] = {} + self.entered: dict[str, threading.Event] = {} + + class _Response: + def __init__(self, raw): + self._raw = raw + + def read(self): + return self._raw + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def set(self, suffix, payload): + self.responses[suffix] = payload + + def block_on(self, suffix): + self.blockers[suffix] = threading.Event() + self.entered[suffix] = threading.Event() + + def release(self, suffix): + self.blockers[suffix].set() + + def urlopen(self, request, timeout): + method = request.get_method() + url = request.full_url + for suffix, event in self.entered.items(): + if url.endswith(suffix): + event.set() + self.blockers[suffix].wait(timeout=10) + self.requests.append((method, url)) + for suffix, payload in self.responses.items(): + if url.endswith(suffix): + raw = payload if isinstance(payload, bytes) else json.dumps(payload).encode() + return self._Response(raw) + return self._Response(json.dumps({"value": {}}).encode()) + + +@pytest.fixture +def responder(monkeypatch): + stub = _WdaResponder() + monkeypatch.setattr(wda.urllib.request, "urlopen", stub.urlopen) + return stub + + +def test_normalize_wda_url_keeps_the_port_in_the_authority(): + assert wda.normalize_wda_url("http://localhost/proxy/wda") == "http://localhost:8100/proxy/wda" + assert ( + wda.normalize_wda_url("http://localhost:9000/proxy/wda") + == "http://localhost:9000/proxy/wda" + ) + assert wda.normalize_wda_url("http://[fd20::1]/wda") == "http://[fd20::1]:8100/wda" + + +@pytest.mark.asyncio +async def test_wda_status_surfaces_the_outer_session_id(responder): + responder.set( + "/status", + {"value": {"ready": True, "build": {}}, "sessionId": "foreign-777"}, + ) + status = await WdaClient("http://wda.test:8100").status() + assert status == {"ready": True, "build": {}, "sessionId": "foreign-777"} + + +@pytest.mark.asyncio +async def test_wda_status_rejects_non_json_responses(responder): + """HTML/bytes from a foreign service on the port must not pass as WDA.""" + responder.set("/status", b"proxy error") + client = WdaClient("http://wda.test:8100") + assert await client.status() is None + + +@pytest.mark.asyncio +async def test_open_session_refuses_a_foreign_active_session(responder): + responder.set("/status", {"value": {"ready": True}, "sessionId": "someone-elses"}) + client = WdaClient("http://wda.test:8100") + with pytest.raises(RuntimeError, match="Refusing to replace"): + await client.open_session() + methods = {m for m, _url in responder.requests} + assert methods == {"GET"} # status only — no POST, no DELETE + assert client.session_id is None + + +@pytest.mark.asyncio +async def test_open_session_adopts_active_session_when_allowed(responder): + """A driver-launched runner's auto-session is adopted, not replaced.""" + responder.set("/status", {"value": {"ready": True}, "sessionId": "auto-77"}) + client = WdaClient("http://wda.test:8100") + assert await client.open_session(adopt_existing=True) == "auto-77" + assert client.session_id == "auto-77" + methods = {m for m, _url in responder.requests} + assert methods == {"GET"} # adopted via /status — no POST /session + + +@pytest.mark.asyncio +async def test_open_session_reuses_its_own_session(responder): + responder.set("/status", {"value": {"ready": True}, "sessionId": None}) + responder.set("/session", {"value": {"sessionId": "owned-1"}}) + client = WdaClient("http://wda.test:8100") + first = await client.open_session() + again = await client.open_session() + assert first == again == "owned-1" + posts = [u for m, u in responder.requests if m == "POST"] + assert len(posts) == 1 + + +@pytest.mark.asyncio +async def test_open_session_creates_after_a_null_session_status(responder): + responder.set("/status", {"value": {"ready": True}, "sessionId": None}) + responder.set("/session", {"value": {"sessionId": "fresh-1"}}) + client = WdaClient("http://wda.test:8100") + assert await client.open_session() == "fresh-1" + assert client.session_id == "fresh-1" + + +@pytest.mark.asyncio +async def test_request_cancellation_drains_the_blocking_call(responder): + """A cancelled input must not leave the urllib call racing the release.""" + client = WdaClient("http://wda.test:8100") + responder.block_on("/session/s1/actions") + responder.set("/session/s1/actions", {"value": {}}) + + task = asyncio.create_task(client._request("POST", "/session/s1/actions", {})) + try: + # Event.wait blocks a worker thread, never the loop — the request + # coroutine must already be inside urlopen before we cancel it. + entered = responder.entered["/session/s1/actions"] + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) # let the cancellation land on the task + assert not task.done() # still draining the blocked HTTP call + finally: + responder.release("/session/s1/actions") + with pytest.raises(asyncio.CancelledError): + await task + # The drained request finished before cancellation propagated. + assert ("POST", "http://wda.test:8100/session/s1/actions") in responder.requests + + +@pytest.mark.asyncio +async def test_open_session_cancellation_deletes_the_orphaned_session(responder): + client = WdaClient("http://wda.test:8100") + responder.set("/status", {"value": {"ready": True}, "sessionId": None}) + responder.block_on("/session") + responder.set("/session", {"value": {"sessionId": "orphan-9"}}) + + task = asyncio.create_task(client.open_session()) + try: + entered = responder.entered["/session"] + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + finally: + responder.release("/session") + with pytest.raises(asyncio.CancelledError): + await task + deletes = [u for m, u in responder.requests if m == "DELETE"] + assert deletes == ["http://wda.test:8100/session/orphan-9"] + assert client.session_id is None + + +@pytest.mark.asyncio +async def test_device_info_rejects_non_dict_responses(responder): + responder.set("/wda/device/info", b"not-json") + client = WdaClient("http://wda.test:8100") + with pytest.raises(RuntimeError, match="non-JSON"): + await client.device_info() + + +@pytest.mark.asyncio +async def test_device_info_returns_the_dict(responder): + responder.set( + "/wda/device/info", + {"value": {"name": "Test Phone", "isSimulator": False, "uuid": "iv"}}, + ) + info = await WdaClient("http://wda.test:8100").device_info() + assert info["name"] == "Test Phone" + assert info["isSimulator"] is False + + +# --- Connect identity verification ----------------------------------------- # + + +def _connect_stubs(driver, monkeypatch, client): + monkeypatch.setattr( + physical_driver, "list_core_devices", _async_devices(_parsed(PHYSICAL_IPHONE)) + ) + monkeypatch.setattr(driver, "_ensure_wda", _async_return(client)) + monkeypatch.setattr(driver, "_require_ios_host", _async_return(None)) + + +@pytest.mark.asyncio +async def test_connect_verifies_the_wda_endpoint_identity(driver, monkeypatch): + client = _FakeWda(device_name="Jane's iPhone") + _connect_stubs(driver, monkeypatch, client) + await driver.connect() + assert driver._session_key == "wda-session" + assert client.opened_sessions == 1 + + +@pytest.mark.asyncio +async def test_connect_rejects_a_wda_on_the_wrong_device(driver, monkeypatch): + client = _FakeWda(device_name="Someone Else's iPhone") + _connect_stubs(driver, monkeypatch, client) + with pytest.raises(RuntimeError, match="ARTEMIS_IOS_WDA_URL"): + await driver.connect() + assert client.opened_sessions == 0 + assert driver._session_key is None + + +@pytest.mark.asyncio +async def test_connect_rejects_a_simulator_wda(driver, monkeypatch): + client = _FakeWda(device_name="Jane's iPhone", is_simulator=True) + _connect_stubs(driver, monkeypatch, client) + with pytest.raises(RuntimeError, match="ARTEMIS_IOS_WDA_URL"): + await driver.connect() + assert client.opened_sessions == 0 + + +@pytest.mark.asyncio +async def test_connect_accepts_generic_wda_family_name(driver, monkeypatch): + # WDA reports the product family ("iPhone"), not the personalized + # devicectl name ("Jane's iPhone") — that is the same device. + client = _FakeWda(device_name="iPhone") + _connect_stubs(driver, monkeypatch, client) + await driver.connect() + assert client.opened_sessions == 1 + await driver.disconnect() + + +@pytest.mark.asyncio +async def test_connect_adopts_session_when_we_launched_the_runner(driver, monkeypatch): + client = _FakeWda() + _connect_stubs(driver, monkeypatch, client) + driver._wda_runner_pid = 4242 # _ensure_wda launched this runner + await driver.connect() + assert client.adopt_requested is True + await driver.disconnect() + + +@pytest.mark.asyncio +async def test_connect_refuses_adoption_for_discovered_endpoints(driver, monkeypatch): + client = _FakeWda() + _connect_stubs(driver, monkeypatch, client) + await driver.connect() + assert client.adopt_requested is False + await driver.disconnect() + + +# --- Disconnect cleanup ----------------------------------------------------- # + + +class _HungProcess: + """A subprocess stand-in that ignores terminate() and must be reaped.""" + + def __init__(self): + self.returncode = None + self.terminated = False + self.killed = False + self.communicated = False + + def terminate(self): + self.terminated = True + + def kill(self): + self.killed = True + self.returncode = -9 + + async def wait(self): + await asyncio.Event().wait() # never exits on its own + + async def communicate(self): + self.communicated = True + return b"", b"" + + +@pytest.mark.asyncio +async def test_disconnect_reaps_a_test_process_that_ignores_terminate( + connected_driver, monkeypatch +): + process = _HungProcess() + connected_driver._wda_test_process = process + monkeypatch.setattr(physical_driver.asyncio, "wait_for", _hangs_then_times_out) + await connected_driver.disconnect() + assert process.terminated + assert process.killed and process.communicated + assert connected_driver._wda_test_process is None + + +async def _hangs_then_times_out(awaitable, timeout): + awaitable.close() + raise TimeoutError + + +@pytest.mark.asyncio +async def test_disconnect_reaps_children_even_when_close_session_fails( + connected_driver, monkeypatch +): + async def _boom(): + raise RuntimeError("wda already gone") + + connected_driver._wda.close_session = _boom + process = _HungProcess() + connected_driver._wda_test_process = process + connected_driver._wda_runner_pid = 777 + calls = [] + + async def fake_xcrun(*arguments, timeout=30.0): + calls.append(arguments) + return b"" + + monkeypatch.setattr(physical_driver, "run_xcrun", fake_xcrun) + monkeypatch.setattr(physical_driver.asyncio, "wait_for", _hangs_then_times_out) + await connected_driver.disconnect() + assert process.killed and process.communicated + assert any("terminate" in args and str(777) in args for args in calls) + assert connected_driver._wda is None + + +# --- Physical recorder cancellation and ffconcat ---------------------------- # + + +def test_ffconcat_file_line_escapes_apostrophes(): + path = Path("/tmp/Jane's Frames/frame 1.png") + line = PhysicalIosRecorder._ffconcat_file_line(path) + assert line == "file '/tmp/Jane'\\''s Frames/frame 1.png'" + + +@pytest.mark.asyncio +async def test_run_ffmpeg_cancellation_reaps_and_drops_part_file(tmp_path, monkeypatch): + entered = asyncio.Event() + release = asyncio.Event() + reaped: list[object] = [] + + class _FakeProc: + returncode = None + + async def communicate(self): + entered.set() + await release.wait() + return b"", b"" + + async def fake_exec(*args, **kwargs): + return _FakeProc() + + async def fake_reap(process): + reaped.append(process) + + monkeypatch.setattr(physical_recording.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(physical_recording, "reap_process", fake_reap) + part = tmp_path / "segment_0000.part.mp4" + part.write_bytes(b"partial") + task = asyncio.create_task(physical_recording._run_ffmpeg(["-i", "in", str(part)])) + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert len(reaped) == 1 + assert not part.exists() + + +@pytest.mark.asyncio +async def test_run_ffmpeg_timeout_reaps_and_reports_failure(tmp_path, monkeypatch): + reaped: list[object] = [] + + class _FakeProc: + returncode = None + + async def communicate(self): + await asyncio.Event().wait() + + async def fake_exec(*args, **kwargs): + return _FakeProc() + + async def fake_reap(process): + process.returncode = -9 + reaped.append(process) + + monkeypatch.setattr(physical_recording.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(physical_recording, "reap_process", fake_reap) + monkeypatch.setattr(physical_recording, "FFMPEG_TIMEOUT_SECONDS", 0.05) + code, stderr = await physical_recording._run_ffmpeg(["-i", "in", str(tmp_path / "o.mp4")]) + assert code == -1 + assert b"timed out" in stderr + assert len(reaped) == 1 + + +@pytest.mark.asyncio +async def test_input_fails_clearly_when_wda_is_detached(): + """A stale session key without a WDA client fails before input/observe.""" + driver = PhysicalIosDriver(device_id=IPHONE_UDID) + driver._session_key = "stale-session" + driver._wda = None + for call in ( + driver.tap(10, 10), + driver.press_key("home"), + driver.input_text("hi", clear_existing=False), + ): + with pytest.raises(RuntimeError, match="Connect the physical iOS driver"): + await call + + +class _FalseHomeWda(_FakeWda): + """A WDA backend that always reports Home button failure.""" + + async def press_button(self, name): + self.buttons.append(name) + return False + + +@pytest.mark.asyncio +async def test_home_key_falls_back_to_homescreen_when_button_fails(connected_driver): + connected_driver._wda = _FalseHomeWda() + assert await connected_driver.press_key("home") is True + assert connected_driver._wda.buttons == ["home"] + assert connected_driver._wda.homescreen_calls == 1 + + +@pytest.mark.asyncio +async def test_app_switch_raises_when_home_button_is_unavailable(connected_driver): + connected_driver._wda = _FalseHomeWda() + with pytest.raises(NotImplementedError, match="App switching is unavailable"): + await connected_driver.press_key("app_switch") + # No homescreen fallback may masquerade as a successful app switch. + assert connected_driver._wda.homescreen_calls == 0 + + +@pytest.mark.asyncio +async def test_app_switch_double_presses_home_on_capable_backend(connected_driver): + assert await connected_driver.press_key("app_switch") is True + assert connected_driver._wda.buttons == ["home", "home"] + + +@pytest.mark.parametrize("bad_rect", ["missing", None, "bogus", 42]) +def test_parse_wda_elements_skips_bad_parent_rect_but_keeps_children(bad_rect): + child = { + "type": "XCUIElementTypeButton", + "label": "OK", + "rect": {"x": 1, "y": 1, "width": 10, "height": 10}, + } + tree = {"type": "XCUIElementTypeApplication", "children": [child]} + if bad_rect != "missing": + tree["rect"] = bad_rect + elements = wda.parse_wda_elements(tree, (1.0, 1.0), 300, 600) + assert [e["text"] for e in elements] == ["OK"] diff --git a/tests/unit/drivers/test_ios_recording.py b/tests/unit/drivers/test_ios_recording.py new file mode 100644 index 00000000..0865601c --- /dev/null +++ b/tests/unit/drivers/test_ios_recording.py @@ -0,0 +1,800 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic tests for the native iOS simctl recorder.""" + +import asyncio +from collections import deque +import json +from pathlib import Path +import signal +import subprocess +import threading +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import cv2 +import pytest + +from artemis.drivers.ios import recording as rec +from artemis.drivers.ios.recording import ( + IosRecordingSession, + IosScreenRecorder, + finalize_mov_to_mp4, +) +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from third_party.mobile_use.utils.video import ( + get_active_session, + remove_active_session, + set_active_session, +) + +UDID = "E1D9F1D1-04E5-4E95-801F-830B854FD3E2" + + +class FakeStderr: + """Queued stderr lines; hangs like a live pipe until ``feed_eof``.""" + + def __init__(self, lines=None, delay: float = 0.0, eof: bool = False): + self._lines = deque(lines if lines is not None else [b"Recording started\n"]) + self._delay = delay + self._eof = asyncio.Event() + if eof: + self._eof.set() + + def feed(self, line: bytes) -> None: + self._lines.append(line) + + def feed_eof(self) -> None: + self._eof.set() + + async def readline(self): + while True: + if self._lines: + if self._delay: + await asyncio.sleep(self._delay) + return self._lines.popleft() + if self._eof.is_set(): + return b"" + await asyncio.sleep(0.005) + + +class FakeRecorderProcess: + """Minimal asyncio.subprocess.Process stand-in for simctl recordVideo.""" + + def __init__(self, stderr=None, *, ignore_signals=()): + self.stderr = stderr if stderr is not None else FakeStderr() + self.returncode = None + self.signals: list[int] = [] + self.terminated = False + self.killed = False + self._exit = asyncio.Event() + self._ignore = set(ignore_signals) + + def send_signal(self, sig): + self.signals.append(sig) + if sig not in self._ignore and self.returncode is None: + self.returncode = 0 + self._exit.set() + + def terminate(self): + self.terminated = True + if self.returncode is None: + self.returncode = -int(signal.SIGTERM) + self._exit.set() + + def kill(self): + self.killed = True + if self.returncode is None: + self.returncode = -int(signal.SIGKILL) + self._exit.set() + + async def wait(self): + await self._exit.wait() + return self.returncode + + +def spawn_factory(procs): + """Return (spawn, created) where spawn serves FakeRecorderProcess items.""" + created: list[tuple] = [] + + async def spawn(*argv, **kwargs): + created.append(argv) + index = min(len(created) - 1, len(procs) - 1) + return procs[index] + + return spawn, created + + +@pytest.fixture +def recorder_env(monkeypatch): + """Patch the recorder's child-process and probe seams.""" + env = SimpleNamespace(finalized=[], spawns=None) + monkeypatch.setattr(rec, "probe_display_dimensions", AsyncMock(return_value=(1206, 2622))) + + async def fake_finalize(source_path, output_path, width, height, span): + env.finalized.append((Path(source_path), Path(output_path), width, height, span)) + output_path.write_bytes(b"fake mp4") + return True + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", fake_finalize) + monkeypatch.setattr( + "artemis.utils.video.probe_video_segment", + AsyncMock(return_value={"duration": 1.0, "width": 100, "height": 200}), + ) + monkeypatch.setattr(rec, "WATCHDOG_INTERVAL_SECONDS", 0.01) + return env + + +def patch_spawn(monkeypatch, procs): + spawn, created = spawn_factory(procs) + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + return created + + +@pytest.mark.asyncio +async def test_recordvideo_argv_pins_udid_without_force_or_android_tools( + recorder_env, monkeypatch, tmp_path +): + procs = [FakeRecorderProcess()] + created = patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + + argv = created[0] + assert argv[:4] == ("xcrun", "simctl", "io", UDID) + assert "recordVideo" in argv and "--codec=h264" in argv + assert str(tmp_path / "recording.mov") in argv + assert "--force" not in argv + assert not any("adb" in part or "scrcpy" in part for part in argv) + assert session.is_active and session.capture_width == 1206 + await recorder.stop() + + +@pytest.mark.asyncio +async def test_marker_anchors_timeline_not_process_spawn(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(FakeStderr(delay=0.05))] + created = patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + + before = time.time() + session = await recorder.start(output_dir=tmp_path) + + assert session.start_time >= before + 0.03 # anchored at marker, not spawn + assert session.anchor_monotonic is not None + assert session.segment_started_monotonic == session.anchor_monotonic + await recorder.stop() + + +@pytest.mark.asyncio +async def test_missing_marker_times_out_and_reaps(recorder_env, monkeypatch, tmp_path): + monkeypatch.setattr(rec, "STARTUP_TIMEOUT_SECONDS", 0.05) + proc = FakeRecorderProcess(FakeStderr(lines=[])) # hangs, never marks + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + + with pytest.raises(TimeoutError): + await recorder.start(output_dir=tmp_path) + + assert signal.SIGINT in proc.signals + assert proc.returncode == 0 + assert recorder.session is not None and not recorder.session.is_active + + +@pytest.mark.asyncio +async def test_stderr_eof_before_marker_fails(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess(FakeStderr(lines=[], eof=True)) + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + + with pytest.raises(RuntimeError, match="Recording started"): + await recorder.start(output_dir=tmp_path) + assert proc.returncode == 0 # reaped via SIGINT + + +@pytest.mark.asyncio +async def test_startup_cancellation_reaps_child(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess(FakeStderr(lines=[])) # marker never arrives + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + + task = asyncio.create_task(recorder.start(output_dir=tmp_path)) + await asyncio.sleep(0.02) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert proc.returncode is not None + assert signal.SIGINT in proc.signals + + +@pytest.mark.asyncio +async def test_cancelled_probe_after_marker_reaps_child(recorder_env, monkeypatch, tmp_path): + """Cancelling during the dimension probe must not leak the recorder child.""" + proc = FakeRecorderProcess() # marker arrives instantly + patch_spawn(monkeypatch, [proc]) + probe_entered = threading.Event() + probe_gate = threading.Event() + + async def gated_probe(device_id): + probe_entered.set() + while not probe_gate.is_set(): + await asyncio.sleep(0.005) + return (1206, 2622) + + monkeypatch.setattr(rec, "probe_display_dimensions", gated_probe) + recorder = IosScreenRecorder(UDID) + + task = asyncio.create_task(recorder.start(output_dir=tmp_path)) + try: + assert await asyncio.to_thread(probe_entered.wait, 5) + task.cancel() + await asyncio.sleep(0) + assert not task.done() # still gated inside the probe await + finally: + probe_gate.set() + with pytest.raises(asyncio.CancelledError): + await task + + session = recorder.session + assert session is not None and not session.is_active + assert proc.returncode is not None + assert signal.SIGINT in proc.signals + assert session.stderr_task is None or session.stderr_task.done() + + +@pytest.mark.asyncio +async def test_probe_failure_after_marker_reaps_child(recorder_env, monkeypatch, tmp_path): + """A dimension-probe error after the marker must reap the owned child.""" + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + + async def boom(device_id): + raise RuntimeError("display probe exploded") + + monkeypatch.setattr(rec, "probe_display_dimensions", boom) + recorder = IosScreenRecorder(UDID) + + with pytest.raises(RuntimeError, match="display probe exploded"): + await recorder.start(output_dir=tmp_path) + + session = recorder.session + assert session is not None and not session.is_active + assert proc.returncode is not None + assert signal.SIGINT in proc.signals + assert session.stderr_task is None or session.stderr_task.done() + + +@pytest.mark.asyncio +async def test_sigint_terminate_kill_escalation(recorder_env, monkeypatch, tmp_path): + monkeypatch.setattr(rec, "SIGINT_FLUSH_TIMEOUT_SECONDS", 0.05) + monkeypatch.setattr(rec, "TERMINATE_TIMEOUT_SECONDS", 0.05) + term_proc = FakeRecorderProcess(ignore_signals={signal.SIGINT}) + kill_proc = FakeRecorderProcess(ignore_signals={signal.SIGINT}) + kill_proc.terminate = lambda: setattr(kill_proc, "terminated", True) + + patch_spawn(monkeypatch, [term_proc]) + recorder = IosScreenRecorder(UDID) + await recorder.start(output_dir=tmp_path) + await recorder.stop() + assert term_proc.signals == [signal.SIGINT] + assert term_proc.terminated + assert not term_proc.killed + + patch_spawn(monkeypatch, [kill_proc]) + recorder2 = IosScreenRecorder(UDID) + await recorder2.start(output_dir=tmp_path / "b") + await recorder2.stop() + assert kill_proc.terminated and kill_proc.killed + + +@pytest.mark.asyncio +async def test_stderr_buffer_is_bounded(recorder_env, monkeypatch, tmp_path): + stderr = FakeStderr() + proc = FakeRecorderProcess(stderr) + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + for index in range(rec.STDERR_BUFFER_LINES * 2): + stderr.feed(f"line {index}\n".encode()) + await asyncio.sleep(0.05) + + assert len(session.stderr_lines) <= rec.STDERR_BUFFER_LINES + assert session.stderr_lines[-1].startswith("line") + await recorder.stop() + + +@pytest.mark.asyncio +async def test_double_start_rejected_and_stop_without_recording( + recorder_env, monkeypatch, tmp_path +): + patch_spawn(monkeypatch, [FakeRecorderProcess()]) + recorder = IosScreenRecorder(UDID) + + assert await recorder.stop() is None # no session yet + + await recorder.start(output_dir=tmp_path) + with pytest.raises(RuntimeError, match="already active"): + await recorder.start(output_dir=tmp_path) + await recorder.stop() + assert await recorder.stop() is not None # stopped session metadata retained + + +@pytest.mark.asyncio +async def test_output_collision_never_overwrites(recorder_env, monkeypatch, tmp_path): + original = tmp_path / "recording.mov" + original.write_bytes(b"pre-existing capture") + created = patch_spawn(monkeypatch, [FakeRecorderProcess()]) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + + assert "recording_001.mov" in str(created[0][-1]) + assert session.local_video_path.name == "recording_001.mov" + assert original.read_bytes() == b"pre-existing capture" + await recorder.stop() + + +@pytest.mark.asyncio +async def test_rotation_rolls_segment_with_raw_dimensions(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(), FakeRecorderProcess()] + patch_spawn(monkeypatch, procs) + dims_state = {"value": (1206, 2622)} + monkeypatch.setattr( + rec, + "probe_display_dimensions", + AsyncMock(side_effect=lambda *_: dims_state["value"]), + ) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + dims_state["value"] = (2622, 1206) # rotate the display + for _ in range(200): + if session.segments and session.capture_width == 2622: + break + await asyncio.sleep(0.01) + + assert session.segments, "watchdog did not roll on rotation" + first = session.segments[0] + assert (first["width"], first["height"]) == (1206, 2622) + assert session.capture_width == 2622 and session.capture_height == 1206 + assert session.generation >= 1 + assert signal.SIGINT in procs[0].signals + await recorder.stop() + + +@pytest.mark.asyncio +async def test_duration_limit_rolls_segment(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(), FakeRecorderProcess()] + patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path, max_duration_seconds=1) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and not session.segments: + await asyncio.sleep(0.02) + + assert session.segments, "watchdog did not roll on duration limit" + await recorder.stop() + + +@pytest.mark.asyncio +async def test_crash_recovery_is_bounded(recorder_env, monkeypatch, tmp_path): + healthy = FakeRecorderProcess() + dead = [FakeRecorderProcess(FakeStderr(lines=[], eof=True)) for _ in range(5)] + created = patch_spawn(monkeypatch, [healthy] + dead) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + healthy.returncode = 1 # simulate recorder crash + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and session.is_active: + await asyncio.sleep(0.02) + + assert not session.is_active + assert len(created) <= rec.MAX_CONSECUTIVE_FAILURES + 1 + assert any("recovery limit" in error for error in session.errors) + + +@pytest.mark.asyncio +async def test_seal_finalizes_segment_and_continues(recorder_env, monkeypatch, tmp_path): + procs = [FakeRecorderProcess(), FakeRecorderProcess()] + patch_spawn(monkeypatch, procs) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + await asyncio.sleep(0.05) + await recorder.seal(through_time=0.02) + + # The sealed boundary covers through_time; content runs to the SIGINT point. + assert 0.02 <= session.segments[0]["end"] <= 0.5 + assert session.sealed_until >= 0.02 + assert session.generation == 1 + assert session.is_active and session.process is procs[1] + await recorder.seal(through_time=0.01) # already sealed through this value + assert len(session.segments) == 1 + await recorder.stop() + + +@pytest.mark.asyncio +async def test_stop_finalizes_manifest_and_returns_mp4(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + result = await recorder.stop() + + assert result == tmp_path / "recording.mp4" + assert signal.SIGINT in proc.signals + manifest = json.loads((tmp_path / "recording.json").read_text()) + assert manifest["version"] == 2 + assert manifest["segments"] and manifest["segments"][0]["file"] == "recording.mp4" + assert not session.is_active + + +@pytest.mark.asyncio +async def test_stop_cancelled_caller_still_reaps_and_finalizes(recorder_env, monkeypatch, tmp_path): + gate = asyncio.Event() + + async def slow_finalize(source_path, output_path, width, height, span): + await gate.wait() + output_path.write_bytes(b"fake mp4") + return True + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", slow_finalize) + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + recorder = IosScreenRecorder(UDID) + await recorder.start(output_dir=tmp_path) + + task = asyncio.create_task(recorder.stop()) + await asyncio.sleep(0.05) + task.cancel() + gate.set() + with pytest.raises(asyncio.CancelledError): + await task + + assert proc.returncode == 0 + assert (tmp_path / "recording.mp4").exists() + + +@pytest.mark.asyncio +async def test_conversion_failure_reports_partial_and_keeps_raw( + recorder_env, monkeypatch, tmp_path +): + async def failing(*args, **kwargs): + return False + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", failing) + patch_spawn(monkeypatch, [FakeRecorderProcess()]) + recorder = IosScreenRecorder(UDID) + session = await recorder.start(output_dir=tmp_path) + + assert await recorder.stop() is None + assert any("finalization" in error for error in session.errors) + + +@pytest.mark.asyncio +async def test_disconnect_finalizes_recorder_and_clears_registry( + recorder_env, monkeypatch, tmp_path +): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + driver = XcodeSimulatorDriver(device_id=UDID) + driver._session_key = "session-key" + bridge = SimpleNamespace( + connected=True, + call=AsyncMock(), + close=AsyncMock(), + start=AsyncMock(), + ) + driver._bridge = bridge + + await driver.start_video_recording(output_dir=tmp_path) + session = driver.recording_session + set_active_session(UDID, session) + + await driver.disconnect() + + assert proc.returncode == 0 + assert get_active_session(UDID) is None + bridge.call.assert_awaited_once() + assert bridge.call.await_args.args[0] == "DeviceInteractionEndSession" + bridge.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_disconnect_closes_bridge_even_when_finalize_fails( + recorder_env, monkeypatch, tmp_path +): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + monkeypatch.setattr(rec, "finalize_mov_to_mp4", AsyncMock(return_value=False)) + driver = XcodeSimulatorDriver(device_id=UDID) + driver._session_key = "session-key" + bridge = SimpleNamespace(connected=True, call=AsyncMock(), close=AsyncMock(), start=AsyncMock()) + driver._bridge = bridge + + await driver.start_video_recording(output_dir=tmp_path) + set_active_session(UDID, driver.recording_session) + await driver.disconnect() + + assert proc.returncode == 0 + assert get_active_session(UDID) is None + bridge.close.assert_awaited_once() + assert driver.recording_session.errors + + +@pytest.mark.asyncio +async def test_disconnect_leaves_other_sessions_registry(recorder_env, monkeypatch, tmp_path): + proc = FakeRecorderProcess() + patch_spawn(monkeypatch, [proc]) + other = IosRecordingSession(video_id=uuid4(), device_id=UDID, start_time=0.0) + set_active_session(UDID, other) + driver = XcodeSimulatorDriver(device_id=UDID) + driver._session_key = "session-key" + driver._bridge = SimpleNamespace( + connected=True, call=AsyncMock(), close=AsyncMock(), start=AsyncMock() + ) + + await driver.start_video_recording(output_dir=tmp_path) + await driver.disconnect() + + assert get_active_session(UDID) is other + remove_active_session(UDID) + + +# --- Real FFmpeg regression coverage (bundled binary, no Xcode/simulator) --- + + +def _make_vfr_mov(path: Path, width: int = 320, height: int = 240) -> Path: + """Two VFR frames: red at t=0, blue at t=0.5 (from 30fps concat).""" + ffmpeg = rec.get_ffmpeg_path() + subprocess.run( + [ + ffmpeg, + "-y", + "-f", + "lavfi", + "-i", + f"color=c=red:s={width}x{height}:r=30:d=0.5", + "-f", + "lavfi", + "-i", + f"color=c=blue:s={width}x{height}:r=30:d=0.5", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1,select='eq(n,0)+eq(n,15)'[v]", + "-map", + "[v]", + "-fps_mode", + "vfr", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + str(path), + ], + check=True, + capture_output=True, + ) + return path + + +def _frame_at(video: Path, seconds: float): + capture = cv2.VideoCapture(str(video)) + try: + capture.set(cv2.CAP_PROP_POS_MSEC, seconds * 1000) + ok, frame = capture.read() + assert ok and frame is not None + return frame + finally: + capture.release() + + +def _mean_bgr(frame): + return frame.mean(axis=(0, 1)) + + +@pytest.mark.asyncio +async def test_finalize_preserves_vfr_event_times_without_stretching(tmp_path): + source = _make_vfr_mov(tmp_path / "segment.mov") + output = tmp_path / "recording.mp4" + + assert await finalize_mov_to_mp4(source, output, 320, 240, 2.0) + + capture = cv2.VideoCapture(str(output)) + fps = capture.get(cv2.CAP_PROP_FPS) + frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) + capture.release() + assert fps == pytest.approx(30, abs=0.5) + assert frame_count / fps == pytest.approx(2.0, abs=0.05) + + red = _mean_bgr(_frame_at(output, 0.1)) + blue_early = _mean_bgr(_frame_at(output, 0.6)) + blue_tail = _mean_bgr(_frame_at(output, 1.6)) + assert red[2] > 150 and red[0] < 60 # red frame, not stretched into tail + assert blue_early[0] > 150 and blue_early[2] < 60 + assert blue_tail[0] > 150 and blue_tail[2] < 60 # cloned tail stays blue + + +@pytest.mark.asyncio +async def test_finalize_normalizes_mixed_resolution_sources(tmp_path): + portrait = _make_vfr_mov(tmp_path / "portrait.mov", width=240, height=320) + landscape = _make_vfr_mov(tmp_path / "landscape.mov", width=320, height=240) + + out_p = tmp_path / "portrait.mp4" + out_l = tmp_path / "landscape.mp4" + assert await finalize_mov_to_mp4(portrait, out_p, 240, 320, 1.0) + assert await finalize_mov_to_mp4(landscape, out_l, 240, 320, 1.0) + + for produced in (out_p, out_l): + capture = cv2.VideoCapture(str(produced)) + width = capture.get(cv2.CAP_PROP_FRAME_WIDTH) + height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + ok, _frame = capture.read() + capture.release() + assert ok and (width, height) == (240, 320) + + +@pytest.mark.asyncio +async def test_probe_falls_back_to_cv2_only_when_ffprobe_missing(monkeypatch, tmp_path): + from artemis.utils import video as video_utils + + mp4 = tmp_path / "segment.mp4" + assert await finalize_mov_to_mp4(_make_vfr_mov(tmp_path / "src.mov"), mp4, 320, 240, 1.0) + monkeypatch.setattr(video_utils, "get_ffprobe_path", lambda: str(tmp_path / "missing-ffprobe")) + + metadata = await video_utils.probe_video_segment(mp4) + assert metadata["width"] == 320 and metadata["height"] == 240 + assert metadata["duration"] == pytest.approx(1.0, abs=0.05) + + corrupted = tmp_path / "corrupt.mp4" + corrupted.write_bytes(b"not a movie") + assert await video_utils.probe_video_segment(corrupted) == {} + + +# Mirrors real ``simctl io enumerate`` output on an iPhone simulator: +# the LCD is the only display with an IOSurface port; external scene displays +# (CarPlay wireless, resizable) report Default dims only and must be ignored. +ENUMERATE_SAMPLE = """\ +Port: + UUID: 0E2F25F7-2FF1-44B7-93A2-36AE37042D2D + Class: Unknown + Port Identifier: com.apple.display.captureservice + Power state: On + +Port: + UUID: 99261FD3-48E9-4CB4-B58D-5D2AF9CB5538 + Class: Display + Port Identifier: com.apple.framebuffer.display + Power state: On + Display class: 1 + Default width: 720 + Default height: 480 + Default pixel format: 'BGRA' + +Port: + UUID: 9A996636-389D-48C9-86D4-D2A252086D44 + Class: Display + Port Identifier: com.apple.framebuffer.display + Power state: On + Display class: 0 + Default width: 1206 + Default height: 2622 + Default pixel format: 'BGRA' + IOSurface port: + width = 1206 + height = 2622 + bytes per row = 4864 + size = 12763136 + +Port: + UUID: E7A14973-04D1-456A-858D-BD0EEF2FF5D5 + Class: Display + Port Identifier: com.apple.framebuffer.display + Power state: On + Display class: 1 + Default width: 7680 + Default height: 4320 + Default pixel format: 'BGRA' +""" + + +def test_parse_display_dimensions_uses_iosurface_not_default_dims(): + assert rec._parse_display_dimensions(ENUMERATE_SAMPLE) == (1206, 2622) + rotated = ENUMERATE_SAMPLE.replace( + "width = 1206", "width = 2622" + ).replace("height = 2622", "height = 1206") + assert rec._parse_display_dimensions(rotated) == (2622, 1206) + assert rec._parse_display_dimensions("Port:\n Class: Unknown\n") is None + assert rec._parse_display_dimensions("") is None + + +@pytest.mark.asyncio +async def test_young_crash_loop_caps_respawns_and_stops(recorder_env, monkeypatch, tmp_path): + """Spawns that succeed then die young must bound the crash loop. + + Every fake proc delivers the 'Recording started' marker (spawn succeeds) + and then exits immediately — the watchdog's young-crash counter must stop + the respawn loop at MAX_CONSECUTIVE_FAILURES instead of spawning forever. + """ + created: list[FakeRecorderProcess] = [] + + async def spawn(*argv, **kwargs): + proc = FakeRecorderProcess() + created.append(proc) + + def die_young(): + if proc.returncode is None: + proc.returncode = 1 + proc._exit.set() + + asyncio.get_running_loop().call_later(0.01, die_young) + return proc + + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + recorder = IosScreenRecorder(UDID) + + session = await recorder.start(output_dir=tmp_path) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and session.is_active: + await asyncio.sleep(0.02) + + assert not session.is_active + assert len(created) <= rec.MAX_CONSECUTIVE_FAILURES + 1 + assert any("recovery limit" in error for error in session.errors) + # No orphan process survives the cap. + assert all(proc.returncode is not None for proc in created) + + +@pytest.mark.asyncio +async def test_finalize_cancellation_reaps_child_and_drops_part(tmp_path, monkeypatch): + """Cancelling finalize must reap the ffmpeg child and remove its .part.""" + source = tmp_path / "segment_0000.mov" + source.write_bytes(b"mov") + entered = asyncio.Event() + release = asyncio.Event() + reaped: list[object] = [] + + class _Proc: + returncode = None + + async def communicate(self): + entered.set() + await release.wait() + return b"", b"" + + async def fake_exec(*args, **kwargs): + return _Proc() + + async def fake_reap(process): + reaped.append(process) + + monkeypatch.setattr(rec.asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(rec, "reap_process", fake_reap) + output = tmp_path / "segment_0000.mp4" + task = asyncio.create_task(finalize_mov_to_mp4(source, output, 100, 200, 1.0)) + await asyncio.wait_for(entered.wait(), timeout=5) + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert len(reaped) == 1 + assert not output.exists() + assert not (tmp_path / "segment_0000.part.mp4").exists() diff --git a/tests/unit/mcp/test_diagnose_tool.py b/tests/unit/mcp/test_diagnose_tool.py index cd4627f5..75d76798 100644 --- a/tests/unit/mcp/test_diagnose_tool.py +++ b/tests/unit/mcp/test_diagnose_tool.py @@ -1285,3 +1285,148 @@ def test_newer_helper_than_bundle_is_not_a_finding(temp_trace_env): result = _run(_healthy_probes(), helper_status=status) assert result["verdict"] == "ready" assert not any("Accessibility helper" in s for s in result["next_steps"]) + + +# --------------------------------------------------------------------------- # +# iOS device selection +# --------------------------------------------------------------------------- # + +_IOS_SIM_BOOTED = {"udid": "SIM-AAAA", "name": "Sim Booted", "state": "Booted"} +_IOS_SIM_SHUT = {"udid": "SIM-BBBB", "name": "Sim Off", "state": "Shutdown"} +_IOS_PHYS = {"udid": "PHYS-0001", "name": "Office Phone", "state": "device"} + + +def _ios_probe(simulators=None, physical=None, status=ProbeStatus.PASS): + return _probe( + "ios_simulators", + status, + category=ProbeCategory.DEVICE, + metadata={ + "xcode_27_or_newer": True, + "simulators": ([_IOS_SIM_BOOTED, _IOS_SIM_SHUT] if simulators is None else simulators), + "connected_physical_devices": ([_IOS_PHYS] if physical is None else physical), + }, + ) + + +def test_ios_requested_device_matches_udid_case_insensitively(): + selected = diagnose._ios_requested_device(_ios_probe(), "sim-aaaa") + assert selected == { + "serial": "SIM-AAAA", + "name": "Sim Booted", + "state": "device", + "is_emulator": True, + } + + +def test_ios_requested_device_matches_exact_sim_and_physical_names(): + sim = diagnose._ios_requested_device(_ios_probe(), "Sim Off") + assert sim["serial"] == "SIM-BBBB" and sim["is_emulator"] is True + phys = diagnose._ios_requested_device(_ios_probe(), "Office Phone") + assert phys["serial"] == "PHYS-0001" and phys["is_emulator"] is False + # Name matching is exact — case differences do not bind. + assert diagnose._ios_requested_device(_ios_probe(), "sim off") is None + + +def test_ios_requested_device_explicit_booted_needs_exactly_one(): + assert diagnose._ios_requested_device(_ios_probe(), "booted")["serial"] == "SIM-AAAA" + two_booted = [_IOS_SIM_BOOTED, {**_IOS_SIM_SHUT, "state": "Booted"}] + assert diagnose._ios_requested_device(_ios_probe(simulators=two_booted), "booted") is None + none_booted = [dict(_IOS_SIM_SHUT)] + assert diagnose._ios_requested_device(_ios_probe(simulators=none_booted), "booted") is None + + +def test_ios_requested_device_ambiguous_names_return_none(): + twins = [ + {**_IOS_SIM_BOOTED, "name": "Twin"}, + {**_IOS_SIM_SHUT, "name": "Twin"}, + ] + assert diagnose._ios_requested_device(_ios_probe(simulators=twins), "Twin") is None + + +def test_ios_requested_device_never_autopicks_hardware(): + # No request: exactly one booted sim wins; a sole shutdown sim is next. + assert diagnose._ios_requested_device(_ios_probe(), None)["serial"] == "SIM-AAAA" + only_off = diagnose._ios_requested_device(_ios_probe(simulators=[_IOS_SIM_SHUT]), None) + assert only_off["serial"] == "SIM-BBBB" + # Physical-only inventory is never auto-selected. + assert diagnose._ios_requested_device(_ios_probe(simulators=[]), None) is None + + +def test_ios_probe_serial_routes_canonical_udid(monkeypatch): + """_run_device_probe must smoke the canonical UDID, not the raw request.""" + smoke = AsyncMock(return_value=_smoke_ok("SIM-AAAA")) + monkeypatch.setattr(diagnose, "_device_smoke_test", smoke) + result = asyncio.run(diagnose._run_device_probe(None, "sim-aaaa", _ios_probe(), platform="ios")) + smoke.assert_awaited_once_with("SIM-AAAA", platform="ios") + assert result["ok"] is True + + +def test_ios_probe_without_canonical_target_fails_with_udid_guidance(monkeypatch): + smoke = AsyncMock() + monkeypatch.setattr(diagnose, "_device_smoke_test", smoke) + result = asyncio.run(diagnose._run_device_probe(None, "ghost", _ios_probe(), platform="ios")) + assert result["ok"] is False + assert "UDID" in result["error"] + smoke.assert_not_awaited() + # No request and ambiguous inventory -> same guidance shape. + twins = [_IOS_SIM_BOOTED, {**_IOS_SIM_SHUT, "state": "Booted"}] + result = asyncio.run( + diagnose._run_device_probe(None, None, _ios_probe(simulators=twins), platform="ios") + ) + assert result["ok"] is False + assert "device_serial" in result["error"] + smoke.assert_not_awaited() + + +def test_requested_device_ready_ios_uses_the_same_selection(): + results = [_ios_probe()] + assert diagnose._requested_device_ready(results, "SIM-AAAA", platform="ios") + assert diagnose._requested_device_ready(results, "Office Phone", platform="ios") + assert not diagnose._requested_device_ready(results, "ghost", platform="ios") + assert diagnose._requested_device_ready(results, None, platform="ios") + + +def test_ios_diagnose_reports_the_selected_ios_device(temp_trace_env): + """The iOS `device` payload describes the selected iOS target, not ADB's.""" + probes = _healthy_probes() + [_ios_probe()] + result = _run(probes, platform="ios") + device = result["device"] + assert device == { + "serial": "SIM-AAAA", + "model": "Sim Booted", + "platform": "ios", + "is_emulator": True, + } + + +def test_ios_diagnose_smoke_uses_selected_udid(temp_trace_env): + probes = _healthy_probes() + [_ios_probe()] + smoke = AsyncMock(return_value=_smoke_ok("SIM-AAAA")) + result = _run(probes, platform="ios", probe_device=True, smoke=smoke) + smoke.assert_awaited_once_with("SIM-AAAA", platform="ios") + assert result["device_probe"]["ok"] is True + + +def test_android_diagnose_device_shape_unchanged(temp_trace_env): + """Regression: Android responses keep the compact active-device payload.""" + device = DeviceInfo( + serial="pixel-1", + model="Pixel 8", + android_version="15", + is_locked=False, + installed_packages=["a"], + screen_resolution="1080x2400", + ) + result = _run(_healthy_probes(), report=_report(_healthy_probes(), active_device=device)) + device_out = result["device"] + # The accessibility_helper attachment is part of the compact payload. + assert isinstance(device_out.pop("accessibility_helper"), dict) + assert device_out == { + "serial": "pixel-1", + "state": "device", + "model": "Pixel 8", + "android_version": "15", + "is_locked": False, + "is_emulator": False, + } diff --git a/tests/unit/mcp/test_mcp_stdio_lifecycle.py b/tests/unit/mcp/test_mcp_stdio_lifecycle.py index 05dea880..920cdd29 100644 --- a/tests/unit/mcp/test_mcp_stdio_lifecycle.py +++ b/tests/unit/mcp/test_mcp_stdio_lifecycle.py @@ -117,11 +117,13 @@ def test_mcp_stdio_handshake_immediate_input(): p.stdin.write(json.dumps(init_req).encode("utf-8") + b"\n") p.stdin.flush() - # Read initialize response with a strict timeout - init_resp_line = _readline_with_timeout(p.stdout, 6.0) + # Read initialize response with a strict timeout. The deadline only needs + # to catch a wedged subprocess; cold-start imports on a loaded machine can + # take well over 6s, so allow generous-but-finite headroom. + init_resp_line = _readline_with_timeout(p.stdout, 30.0) assert init_resp_line is not None, ( - "MCP server failed to respond to initialize request within 6 seconds (deadlock detected)!" + "MCP server failed to respond to initialize request within 30 seconds (deadlock detected)!" ) init_data = json.loads(init_resp_line) assert init_data.get("id") == 1 @@ -134,7 +136,7 @@ def test_mcp_stdio_handshake_immediate_input(): p.stdin.write(json.dumps(tools_req).encode("utf-8") + b"\n") p.stdin.flush() - tools_resp_line = _readline_with_timeout(p.stdout, 4.0) + tools_resp_line = _readline_with_timeout(p.stdout, 15.0) assert tools_resp_line is not None, "MCP server failed to respond to tools/list request!" tools_data = json.loads(tools_resp_line) diff --git a/tests/unit/mcp/test_mcp_tools.py b/tests/unit/mcp/test_mcp_tools.py index 0064ed9a..f1eed944 100644 --- a/tests/unit/mcp/test_mcp_tools.py +++ b/tests/unit/mcp/test_mcp_tools.py @@ -402,6 +402,64 @@ async def test_mobile_get_device_state_passes_device_serial(): get_ctrl_mock.assert_called_once_with(device_serial="device-serial-abc") +@pytest.mark.asyncio +async def test_mobile_get_device_state_rejects_invalid_platform_before_factory(): + """An invalid platform token must never reach the controller factory.""" + with ( + patch("mcp_server.tools.device_state._get_controller") as get_ctrl_mock, + patch( + "mcp_server.tools.device_state.observe_ios_controller", new_callable=AsyncMock + ) as observe_mock, + ): + result = await mobile_get_device_state(view_type="hierarchy", platform="windows-phone") + + assert result.startswith("Error:") + assert "platform" in result.lower() + get_ctrl_mock.assert_not_called() + observe_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_mobile_get_device_state_ios_routes_through_observation_helper(): + """A case/whitespace-tolerant iOS token builds an iOS controller and + observes it through the lease-owning helper, not a raw connect.""" + controller = MagicMock() + controller.ctx.device.mobile_platform = "ios" + controller.ctx.device.device_width = 1206 + controller.ctx.device.device_height = 2622 + screen = SimpleNamespace( + base64="aVBORw==", + elements=[ + { + "class": "XCUIElementTypeButton", + "text": "Continue", + "bounds": "[100,500][400,600]", + } + ], + width=1206, + height=2622, + ) + + with ( + patch( + "mcp_server.tools.device_state._get_controller", return_value=controller + ) as get_ctrl_mock, + patch( + "mcp_server.tools.device_state.observe_ios_controller", + new_callable=AsyncMock, + return_value=screen, + ) as observe_mock, + patch("mcp_server.tools.device_state.is_ocr_configured", return_value=False), + ): + result = await mobile_get_device_state(view_type="hierarchy", platform=" iOS ") + + assert "Continue" in result + assert not result.startswith("Error:") + get_ctrl_mock.assert_called_once_with(device_serial=None, target_platform="ios") + observe_mock.assert_awaited_once_with(controller) + controller.get_screen_data.assert_not_called() + + @pytest.mark.asyncio async def test_mobile_inspect_trace_includes_device_serial(temp_trace_env): import sqlite3 diff --git a/tests/unit/runtime/test_device_lock.py b/tests/unit/runtime/test_device_lock.py index 008f4256..33d491b8 100644 --- a/tests/unit/runtime/test_device_lock.py +++ b/tests/unit/runtime/test_device_lock.py @@ -326,6 +326,26 @@ def test_active_owner_is_discoverable_and_can_be_annotated(monkeypatch): owner_lock.release() +def test_annotation_preserves_lock_scope(monkeypatch): + """Annotating a scoped (e.g. iOS) owner must not strip its lock_scope.""" + monkeypatch.setenv("ARTEMIS_TASK_INGRESS", "daemon") + owner_lock = DeviceExecutionLock("00008100-000000000000001E", "iOS task", lock_scope="ios") + owner_lock.acquire() + try: + assert DeviceExecutionLock.annotate_active_owner( + session_id="ios-session", + device_id="00008100-000000000000001E", + ) + annotated = DeviceExecutionLock.get_active_owner( + "00008100-000000000000001E", lock_scope="ios" + ) + assert annotated is not None + assert annotated.lock_scope == "ios" + assert annotated.session_id == "ios-session" + finally: + owner_lock.release() + + def test_multi_device_locks_can_run_concurrently(): lock_a = DeviceExecutionLock("emulator-5554", "task on device A") lock_b = DeviceExecutionLock("pixel-9-test", "task on device B") diff --git a/tests/unit/runtime/test_ios_observation_lifecycle.py b/tests/unit/runtime/test_ios_observation_lifecycle.py new file mode 100644 index 00000000..4edffc8b --- /dev/null +++ b/tests/unit/runtime/test_ios_observation_lifecycle.py @@ -0,0 +1,266 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lifecycle tests for the shared one-shot iOS observation helper. + +``observe_ios_controller`` must acquire the real iOS-scoped lease before any +native setup, and must always disconnect + release — including on failure and +cancellation. All native calls are stubbed; only the lock registry is real +(isolated per test via ``get_temp_dir``). +""" + +import asyncio +import base64 +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE +from artemis.runtime.device_lock import DeviceBusyError, DeviceExecutionLock +from artemis.runtime.ios_observation import observe_ios_controller + +UDID = "AAAA-1111-0000" + + +@pytest.fixture(autouse=True) +def isolated_lock_dir(tmp_path, monkeypatch): + monkeypatch.setattr("artemis.runtime.device_lock.get_temp_dir", lambda *a, **k: tmp_path) + + +def _screen_data(): + return SimpleNamespace( + base64=base64.b64encode(b"\x89PNG" + bytes(600)).decode("ascii"), + elements='', + width=1170, + height=2532, + ) + + +def _controller(events, *, connect_error=None, capture_error=None): + """A lazy iOS controller whose driver records its call order.""" + + async def resolve_device(): + events.append("resolve") + return UDID + + async def connect(): + events.append("connect") + if connect_error is not None: + raise connect_error + + async def disconnect(): + events.append("disconnect") + + async def capture(): + events.append("capture") + if capture_error is not None: + raise capture_error + return _screen_data() + + driver = SimpleNamespace(resolve_device=resolve_device, connect=connect, disconnect=disconnect) + device = SimpleNamespace(device_id="booted", device_width=0, device_height=0) + return SimpleNamespace( + _driver=driver, ctx=SimpleNamespace(device=device), get_screen_data=capture + ) + + +def _ios_owner(device_id: str = UDID): + owners = DeviceExecutionLock.get_active_owners() + return next( + (o for o in owners.values() if o.device_id == device_id and o.lock_scope == IOS_LOCK_SCOPE), + None, + ) + + +@pytest.mark.asyncio +async def test_observation_orders_resolve_lease_connect_capture_cleanup(): + events: list[str] = [] + controller = _controller(events) + + data = await observe_ios_controller(controller) + + assert events == ["resolve", "connect", "capture", "disconnect"] + assert data.width == 1170 + # Real metrics and the resolved UDID land on the context. + assert controller.ctx.device.device_id == UDID + assert controller.ctx.device.device_width == 1170 + assert controller.ctx.device.device_height == 2532 + # The lease is released after disconnect. + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_observation_holds_ios_lease_during_capture(): + events: list[str] = [] + controller = _controller(events) + held: list[bool] = [] + + original_capture = controller.get_screen_data + + async def capture(): + held.append(_ios_owner() is not None) + return await original_capture() + + controller.get_screen_data = capture + await observe_ios_controller(controller) + assert held == [True] + + +@pytest.mark.asyncio +async def test_held_ios_lease_blocks_observation_before_connect(): + events: list[str] = [] + controller = _controller(events) + held = DeviceExecutionLock( + UDID, + "running iOS task", + ingress="mcp", + concurrency_mode="per_device", + lock_scope=IOS_LOCK_SCOPE, + ) + held.acquire(blocking=False) + try: + with pytest.raises(DeviceBusyError): + await observe_ios_controller(controller) + finally: + held.release() + assert "connect" not in events + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_android_lock_on_same_text_does_not_block_ios(): + """Identical serial text in the Android scope is a different device.""" + events: list[str] = [] + controller = _controller(events) + android_lock = DeviceExecutionLock(UDID, "android task") + android_lock.acquire(blocking=False) + try: + await observe_ios_controller(controller) + finally: + android_lock.release() + assert events == ["resolve", "connect", "capture", "disconnect"] + + +@pytest.mark.asyncio +async def test_capture_failure_disconnects_and_releases(): + events: list[str] = [] + controller = _controller(events, capture_error=RuntimeError("capture blew up")) + + with pytest.raises(RuntimeError, match="capture blew up"): + await observe_ios_controller(controller) + assert events == ["resolve", "connect", "capture", "disconnect"] + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_connect_failure_still_disconnects_and_releases(): + """A half-open native session is closed when connect was attempted.""" + events: list[str] = [] + controller = _controller(events, connect_error=RuntimeError("bridge down")) + + with pytest.raises(RuntimeError, match="bridge down"): + await observe_ios_controller(controller) + assert events == ["resolve", "connect", "disconnect"] + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_disconnect_error_does_not_mask_capture_failure(): + events: list[str] = [] + controller = _controller(events, capture_error=RuntimeError("native lost")) + + async def disconnect(): + events.append("disconnect") + raise RuntimeError("disconnect failed") + + controller._driver.disconnect = disconnect + with pytest.raises(RuntimeError, match="native lost"): + await observe_ios_controller(controller) + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_cancellation_still_disconnects_and_releases(): + events: list[str] = [] + controller = _controller(events) + started = asyncio.Event() + + async def capture(): + events.append("capture") + started.set() + await asyncio.Event().wait() # hang until cancelled + + controller.get_screen_data = capture + task = asyncio.create_task(observe_ios_controller(controller)) + await asyncio.wait_for(started.wait(), timeout=5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert events == ["resolve", "connect", "capture", "disconnect"] + assert _ios_owner() is None + + +@pytest.mark.asyncio +async def test_busy_acquire_failure_never_connects(): + """A failed non-blocking acquire must not touch the native session.""" + events: list[str] = [] + controller = _controller(events) + held = DeviceExecutionLock( + UDID, "owner", concurrency_mode="per_device", lock_scope=IOS_LOCK_SCOPE + ) + held.acquire(blocking=False) + try: + with pytest.raises(DeviceBusyError): + await observe_ios_controller(controller) + finally: + held.release() + assert events == ["resolve"] + assert controller.ctx.device.device_id == UDID + + +@pytest.mark.asyncio +async def test_cancelled_acquire_is_drained_and_never_connects(monkeypatch): + """Cancelling while lease.acquire runs must wait for the gated thread. + + The acquisition eventually lands the real isolated lock; the observer + must not propagate until the thread drains, never connect, and leave no + owned lease behind. + """ + events: list[str] = [] + controller = _controller(events) + gate = threading.Event() + entered = threading.Event() + real_acquire = DeviceExecutionLock.acquire + + def gated_acquire(self, blocking=True): + entered.set() + gate.wait(timeout=10) + return real_acquire(self, blocking=blocking) + + monkeypatch.setattr(DeviceExecutionLock, "acquire", gated_acquire) + task = asyncio.create_task(observe_ios_controller(controller)) + try: + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + await asyncio.sleep(0) # let the cancellation land + assert not task.done() # still draining the gated acquire thread + finally: + gate.set() + with pytest.raises(asyncio.CancelledError): + await task + # Cleanup released the landed lease; connect was never attempted. + assert events == ["resolve"] + assert _ios_owner() is None diff --git a/tests/unit/sdk/test_agent_config_builder_tuning.py b/tests/unit/sdk/test_agent_config_builder_tuning.py index ad200411..7aa6e08a 100644 --- a/tests/unit/sdk/test_agent_config_builder_tuning.py +++ b/tests/unit/sdk/test_agent_config_builder_tuning.py @@ -13,19 +13,19 @@ def test_with_verification_level_off_disables_the_checker(): - cfg = AgentConfigBuilder().with_verification_level("off").build() + cfg = AgentConfigBuilder().with_verification_level("off").build(validate_profiles=False) assert cfg.disable_checker is True def test_with_verification_level_final_keeps_factory_layering(): - cfg = AgentConfigBuilder().with_verification_level("final").build() + cfg = AgentConfigBuilder().with_verification_level("final").build(validate_profiles=False) assert cfg.disable_checker is False assert cfg.disable_midway_checks is True assert cfg.disable_final_check is False def test_with_verification_level_checkpoints_enables_midway_checks(): - cfg = AgentConfigBuilder().with_verification_level("checkpoints").build() + cfg = AgentConfigBuilder().with_verification_level("checkpoints").build(validate_profiles=False) assert cfg.disable_checker is False assert cfg.disable_midway_checks is False assert cfg.disable_final_check is False @@ -34,7 +34,7 @@ def test_with_verification_level_checkpoints_enables_midway_checks(): def test_with_verification_level_strict_applies_the_full_preset(): - cfg = AgentConfigBuilder().with_verification_level("strict").build() + cfg = AgentConfigBuilder().with_verification_level("strict").build(validate_profiles=False) preset = VERIFICATION_LEVEL_PRESETS["strict"] assert cfg.disable_checker is False assert cfg.disable_midway_checks is False @@ -46,7 +46,7 @@ def test_with_verification_level_strict_applies_the_full_preset(): def test_with_verification_level_is_case_and_whitespace_insensitive(): - cfg = AgentConfigBuilder().with_verification_level(" STRICT ").build() + cfg = AgentConfigBuilder().with_verification_level(" STRICT ").build(validate_profiles=False) assert cfg.assert_failure_policy == "halt" @@ -55,12 +55,12 @@ def test_with_verification_level_rejects_unknown_preset(): with pytest.raises(ValueError, match="Unknown verification level"): builder.with_verification_level("paranoid") # The builder is untouched by a rejected call. - assert builder.build().disable_checker is False + assert builder.build(validate_profiles=False).disable_checker is False def test_with_verification_level_matches_the_shared_preset_table(): for level in VERIFICATION_LEVEL_PRESETS: - cfg = AgentConfigBuilder().with_verification_level(level).build() + cfg = AgentConfigBuilder().with_verification_level(level).build(validate_profiles=False) overrides = checker_overrides_for_level(level) assert cfg.disable_checker is (not overrides["enabled"]) if "midway_checks" in overrides: @@ -68,7 +68,12 @@ def test_with_verification_level_matches_the_shared_preset_table(): def test_with_verification_level_is_chainable_and_explicit_checker_wins(): - cfg = AgentConfigBuilder().with_verification_level("strict").with_checker(enabled=False).build() + cfg = ( + AgentConfigBuilder() + .with_verification_level("strict") + .with_checker(enabled=False) + .build(validate_profiles=False) + ) assert cfg.disable_checker is True # The preset's other fields survive the later master switch. assert cfg.assert_failure_policy == "halt" @@ -78,7 +83,7 @@ def test_with_pro_config_carries_both_knobs(): cfg = ( AgentConfigBuilder() .with_pro_config(verification_level="checkpoints", explorer_mode="ultra") - .build() + .build(validate_profiles=False) ) assert cfg.disable_checker is False assert cfg.disable_midway_checks is False @@ -88,13 +93,17 @@ def test_with_pro_config_carries_both_knobs(): def test_with_pro_config_checker_switch_wins_over_verification_level(): - cfg = AgentConfigBuilder().with_pro_config(verification_level="strict", checker=False).build() + cfg = ( + AgentConfigBuilder() + .with_pro_config(verification_level="strict", checker=False) + .build(validate_profiles=False) + ) assert cfg.disable_checker is True def test_with_explorer_pro_mode_is_the_explorer_mode_knob(monkeypatch): monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False) - cfg = AgentConfigBuilder().with_explorer(pro_mode="pro").build() + cfg = AgentConfigBuilder().with_explorer(pro_mode="pro").build(validate_profiles=False) assert cfg.explorer.pro_mode == "pro" assert cfg.get_explorer_version(agent_name=None, profile="pro") == "pro" # The shipped per-agent override is empty, so the Pro agents follow the knob. @@ -104,7 +113,7 @@ def test_with_explorer_pro_mode_is_the_explorer_mode_knob(monkeypatch): def test_with_flash_config_explorer_mode_is_the_flash_profile_knob(monkeypatch): monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False) - cfg = AgentConfigBuilder().with_flash_config(explorer_mode="pro").build() + cfg = AgentConfigBuilder().with_flash_config(explorer_mode="pro").build(validate_profiles=False) assert cfg.flash.explorer_mode == "pro" assert cfg.explorer.flash_mode == "pro" assert cfg.get_explorer_version(agent_name="flash") == "pro" @@ -117,7 +126,68 @@ def test_with_explorer_versions_is_an_advanced_per_agent_override(monkeypatch): cfg = ( AgentConfigBuilder() .with_explorer(pro_mode="flash", versions={"validator": "ultra"}) - .build() + .build(validate_profiles=False) ) assert cfg.get_explorer_version(agent_name="validator") == "ultra" assert cfg.get_explorer_version(agent_name="operator") == "flash" + + +def test_ios_workspace_path_defaults_to_none_and_survives_build(): + cfg = AgentConfigBuilder().build(validate_profiles=False) + assert cfg.ios_workspace_path is None + + +def test_with_ios_workspace_accepts_str_and_path_and_clears(tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + builder = AgentConfigBuilder() + cfg = builder.with_ios_workspace(str(project)).build(validate_profiles=False) + assert cfg.ios_workspace_path == project + cfg = builder.with_ios_workspace(None).build(validate_profiles=False) + assert cfg.ios_workspace_path is None + + +def test_for_ios_simulator_workspace_kwarg_and_omission(tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + cfg = ( + AgentConfigBuilder() + .for_ios_simulator(workspace_path=project) + .build(validate_profiles=False) + ) + assert cfg.ios_workspace_path == project + # Omitting the kwarg does not clear a workspace set earlier on the builder. + cfg = ( + AgentConfigBuilder() + .with_ios_workspace(project) + .for_ios_simulator() + .build(validate_profiles=False) + ) + assert cfg.ios_workspace_path == project + + +def test_for_ios_device_and_alias_preserve_device_and_workspace(tmp_path): + """The canonical name and the simulator-era alias carry identical config.""" + from artemis.context import DevicePlatform + + project = tmp_path / "App.xcworkspace" + project.mkdir() + udid = "AAAA-1111-0000" + canonical = ( + AgentConfigBuilder() + .for_ios_device(udid, workspace_path=project) + .build(validate_profiles=False) + ) + alias = ( + AgentConfigBuilder() + .for_ios_simulator(udid, workspace_path=project) + .build(validate_profiles=False) + ) + for cfg in (canonical, alias): + assert cfg.device_platform == DevicePlatform.IOS + assert cfg.device_id == udid + assert cfg.ios_workspace_path == project + # The default iOS target stays the lazy 'booted' selector. + default = AgentConfigBuilder().for_ios_device().build(validate_profiles=False) + assert default.device_platform == DevicePlatform.IOS + assert default.device_id == "booted" diff --git a/tests/unit/services/test_llm_endpoint.py b/tests/unit/services/test_llm_endpoint.py new file mode 100644 index 00000000..82821999 --- /dev/null +++ b/tests/unit/services/test_llm_endpoint.py @@ -0,0 +1,94 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for endpoint field pass-through in ``_resolve_endpoint``. + +Custom/self-hosted providers declare ``api_base``/``api_key``/``max_tokens``/ +``timeout`` in JSONC config. Previously the LLM schema dropped them as extra +fields and ``_resolve_endpoint`` never read them, so a custom provider always +silently fell back to the environment OpenAI base URL. +""" + +from types import SimpleNamespace + +from artemis.services.llm import _resolve_endpoint +from third_party.mobile_use.config.llm import LLM, LLMWithFallback + + +def _ctx(llm_cfg): + return SimpleNamespace( + llm_config=SimpleNamespace( + get_agent=lambda _name: llm_cfg, + get_utils=lambda _name: llm_cfg, + ) + ) + + +def test_llm_model_accepts_endpoint_fields(): + llm = LLM( + provider="custom", + model="gemma4-e4b", + api_base="http://127.0.0.1:8080/v1", + api_key="k", + max_tokens=4096, + timeout=120.0, + ) + assert llm.api_base == "http://127.0.0.1:8080/v1" + assert llm.api_key == "k" + assert llm.max_tokens == 4096 + assert llm.timeout == 120.0 + + +def test_resolve_endpoint_passes_custom_fields(): + llm = LLM( + provider="custom", + model="gemma4-e4b", + api_base="http://127.0.0.1:8080/v1", + api_key="k", + max_tokens=4096, + timeout=120.0, + ) + ep = _resolve_endpoint(_ctx(llm), "planner") + assert ep.api_base == "http://127.0.0.1:8080/v1" + assert ep.api_key == "k" + assert ep.max_tokens == 4096 + assert ep.timeout_seconds == 120.0 + + +def test_resolve_endpoint_defaults_when_fields_absent(): + llm = LLM(provider="google", model="gemini-2.5-flash") + ep = _resolve_endpoint(_ctx(llm), "planner") + assert ep.api_base is None + assert ep.api_key is None + assert ep.max_tokens is None + assert ep.timeout_seconds == 60.0 + + +def test_resolve_endpoint_fallback_uses_fallback_endpoint_fields(): + llm = LLMWithFallback( + provider="google", + model="gemini-2.5-flash", + timeout=45.0, + fallback=LLM( + provider="custom", + model="gemma4-e4b", + api_base="http://127.0.0.1:8080/v1", + timeout=15.0, + ), + ) + primary = _resolve_endpoint(_ctx(llm), "planner") + assert primary.timeout_seconds == 45.0 + fallback = _resolve_endpoint(_ctx(llm), "planner", use_fallback=True) + assert fallback.api_base == "http://127.0.0.1:8080/v1" + assert fallback.timeout_seconds == 15.0 diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 11e9077a..4392ed7a 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -48,6 +48,8 @@ def test_cli_run_help(): assert "--traces-path" in result.output assert "--verification-level" in result.output assert "--explorer-pro-mode" in result.output + assert "--platform" in result.output + assert "--ios-workspace" in result.output def test_cli_batch_help(): diff --git a/tests/unit/test_ios_integration.py b/tests/unit/test_ios_integration.py new file mode 100644 index 00000000..79cea42f --- /dev/null +++ b/tests/unit/test_ios_integration.py @@ -0,0 +1,867 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic tests for the CLI/SDK/native-action boundaries of iOS support.""" + +import asyncio +import io +import sys +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from PIL import Image +from typer.testing import CliRunner + +from artemis.context import ArtemisContext, DeviceContext, DevicePlatform +from artemis.controllers.unified_controller import UnifiedMobileController +from artemis.drivers.base import ScreenData +from artemis.drivers.factory import create_driver +from artemis.graph.state import State +from artemis.interfaces.cli.commands import run as run_module +from artemis.interfaces.cli.main import app +from artemis.mcp.action_session import get_action_session +from artemis.mcp.action_types import ActionCode +from artemis.mcp.actuators.ios import IosActuator +from artemis.sdk.agent import Agent +from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder +from artemis.tools.mobile.launch_app import find_package, launch_app +from artemis.drivers.ios.bridge import XcodeApprovalRequiredError +from artemis.runtime import DeviceBusyError +from third_party.mobile_use.sdk.agent import AgentBase +from third_party.mobile_use.sdk.types.exceptions import AgentError, AgentNotInitializedError + + +@pytest.fixture +def native_driver(monkeypatch): + driver = MagicMock() + driver.device_id = "00000000-0000-0000-0000-000000000001" + driver.screen_size = (1170, 2532) + for method in ( + "connect", + "resolve_device", + "disconnect", + "input_text", + "press_key", + "open_url", + "tap", + "swipe", + "long_press", + "launch_app", + "stop_app", + "install_app", + "execute_shell", + ): + setattr(driver, method, AsyncMock(return_value=True)) + driver.list_apps = AsyncMock(return_value={"com.apple.Preferences": "Settings"}) + driver.get_screen_data = AsyncMock( + return_value=ScreenData( + screenshot_bytes=b"png", + screenshot_base64="cG5n", + platform="ios", + width=1170, + height=2532, + ui_hierarchy_xml="", + ) + ) + constructor = MagicMock(return_value=driver) + monkeypatch.setitem( + sys.modules, + "artemis.drivers.ios.xcode_driver", + SimpleNamespace(XcodeSimulatorDriver=constructor), + ) + monkeypatch.delenv("ARTEMIS_CLOUD_MODE", raising=False) + monkeypatch.delenv("ARTEMIS_MOCK_DRIVER", raising=False) + driver.constructor = constructor + return driver + + +def ios_context(driver=None): + context = ArtemisContext( + device=DeviceContext( + mobile_platform=DevicePlatform.IOS, + device_id="booted", + device_width=1170, + device_height=2532, + ) + ) + context._active_driver = driver + return context + + +def ios_config(): + return AgentConfigBuilder().for_ios_simulator().build(validate_profiles=False) + + +def test_factory_selects_ios_without_creating_adb(native_driver, monkeypatch): + adb = MagicMock(side_effect=AssertionError("must not create ADB client")) + monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) + context = ios_context() + assert create_driver(context) is native_driver + native_driver.constructor.assert_called_once_with( + device_id="booted", width=1170, height=2532, workspace_path=None + ) + assert context.adb_client is None + + +def test_factory_forwards_configured_ios_workspace(native_driver, tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + context = ios_context() + context.agent_config = ( + AgentConfigBuilder() + .for_ios_simulator() + .with_ios_workspace(project) + .build(validate_profiles=False) + ) + assert create_driver(context) is native_driver + native_driver.constructor.assert_called_once_with( + device_id="booted", width=1170, height=2532, workspace_path=project + ) + + +def test_factory_tolerates_missing_agent_config(native_driver): + context = ios_context() + assert create_driver(context) is native_driver + native_driver.constructor.assert_called_once_with( + device_id="booted", width=1170, height=2532, workspace_path=None + ) + + +def test_macos_host_keeps_android_default(monkeypatch): + context = ArtemisContext(device=DeviceContext(host_platform="DARWIN", device_id="android-1")) + android = MagicMock() + adb = MagicMock() + monkeypatch.setattr("artemis.drivers.factory.AndroidAdbDriver", android) + monkeypatch.setattr("artemis.drivers.factory.AdbClient", adb) + assert create_driver(context) is android.return_value + adb.assert_called_once() + assert context.device.mobile_platform == DevicePlatform.ANDROID + + +def test_factory_rejects_cloud_ios_before_importing_gateway(native_driver, monkeypatch): + monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1") + with pytest.raises(ValueError, match="local only"): + create_driver(ios_context()) + native_driver.constructor.assert_not_called() + + +@pytest.mark.asyncio +async def test_sdk_ios_initialization_bypasses_android_and_cleans(native_driver, monkeypatch): + monkeypatch.setattr( + AgentBase, "_init_internal", AsyncMock(side_effect=AssertionError("ADB init")) + ) + agent = Agent(config=ios_config()) + configured_video = agent._config.video_recording_tools_enabled + agent._prewarm_llm_connections = AsyncMock() + assert await agent.init() is True + await asyncio.sleep(0) + native_driver.resolve_device.assert_awaited_once() + native_driver.connect.assert_not_awaited() + assert agent._device_context.device_id == native_driver.device_id + assert (agent._device_context.device_width, agent._device_context.device_height) == (1170, 2532) + assert agent._adb_client is None and agent._ui_adb_client is None + assert agent._config.video_recording_tools_enabled == configured_video + assert agent._config.disable_device_probes is True + await agent.clean() + native_driver.disconnect.assert_awaited_once() + assert agent._initialized is False + + +@pytest.mark.asyncio +async def test_sdk_init_forwards_ios_workspace(native_driver, tmp_path): + project = tmp_path / "Example.xcodeproj" + project.mkdir() + config = ( + AgentConfigBuilder() + .for_ios_simulator(workspace_path=project) + .build(validate_profiles=False) + ) + agent = Agent(config=config) + agent._prewarm_llm_connections = AsyncMock() + assert await agent.init() is True + native_driver.constructor.assert_called_once_with(device_id="booted", workspace_path=project) + await agent.clean() + + +@pytest.mark.asyncio +async def test_sdk_failed_ios_init_closes_partial_transport(native_driver): + native_driver.resolve_device.side_effect = RuntimeError("bridge failed") + agent = Agent(config=ios_config()) + with pytest.raises(RuntimeError, match="bridge failed"): + await agent.init() + native_driver.disconnect.assert_awaited_once() + assert agent._initialized is False + + +@pytest.mark.asyncio +async def test_sdk_device_overrides_preserve_ios(native_driver): + agent = Agent(config=ios_config(), device_serial="simulator-2") + assert agent._config.device_platform == DevicePlatform.IOS + agent._init_internal = AsyncMock(return_value=True) + await agent.init(device_serial="simulator-3") + assert agent._config.device_id == "simulator-3" + assert agent._config.device_platform == DevicePlatform.IOS + + +@pytest.mark.asyncio +async def test_sdk_sequential_run_reconnects_driver_without_android_unlock(native_driver): + agent = Agent(config=ios_config()) + agent._ios_driver = native_driver + agent._device_context = ios_context().device + await agent._ensure_device_unlocked() + await ios_context(native_driver).disconnect_driver() + await agent._ensure_device_unlocked() + assert native_driver.connect.await_count == 2 + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_context_closes_session_before_driver_and_only_once(native_driver): + order = [] + native_driver.disconnect.side_effect = lambda: order.append("driver") + context = ios_context(native_driver) + context.action_session = SimpleNamespace( + aclose=AsyncMock(side_effect=lambda: order.append("session")) + ) + with pytest.raises(RuntimeError, match="task failed"): + async with context: + raise RuntimeError("task failed") + await context.disconnect_driver() + assert order == ["session", "driver"] + assert context._active_driver is None + + +@pytest.mark.asyncio +async def test_action_session_selects_native_actuator_and_normalizes_coordinates(native_driver): + context = ios_context(native_driver) + async with context: + session = await get_action_session(context) + clicked = await session.call("click", {"target": [500, 500]}) + typed = await session.call("input_text", {"text": "Hello 한글", "clear_exist": False}) + assert clicked.ok and typed.ok + native_driver.tap.assert_awaited_once_with(585, 1266, duration_ms=100, times=1, delay_ms=100) + native_driver.input_text.assert_awaited_once_with("Hello 한글", clear_existing=False) + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_key_failure_is_not_reported_as_success(native_driver): + native_driver.press_key.return_value = False + result = await IosActuator(ios_context(native_driver)).press_key("back") + assert result.ok is False and result.code == ActionCode.UNSUPPORTED + + +@pytest.mark.asyncio +async def test_ios_clear_refusal_happens_before_focus_mutation(native_driver): + result = await IosActuator(ios_context(native_driver)).input_text( + "hello", target=(500, 500), clear_exist=True + ) + assert result.code == ActionCode.UNSUPPORTED + native_driver.tap.assert_not_awaited() + native_driver.input_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_coordinates_follow_latest_orientation(native_driver): + native_driver.screen_size = (2532, 1170) + assert (await IosActuator(ios_context(native_driver)).click(500, 500)).ok + native_driver.tap.assert_awaited_once_with(1266, 585, duration_ms=100, times=1, delay_ms=100) + + +@pytest.mark.asyncio +async def test_ios_urls_and_clear_use_native_driver(native_driver): + controller = UnifiedMobileController(ios_context(native_driver)) + assert await controller.open_url("https://example.com") is True + native_driver.open_url.assert_awaited_once_with("https://example.com") + native_driver.input_text.return_value = False + assert await controller.erase_text() is False + native_driver.input_text.assert_awaited_once_with("", clear_existing=True) + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_apps_resolve_native_names_and_launch_without_android_poll(native_driver): + context = ios_context(native_driver) + assert await find_package(context, "settings") == "com.apple.Preferences" + assert await find_package(context, "com.apple.Preferences") == "com.apple.Preferences" + result = await launch_app.execute(ctx=context, app_name="Settings") + assert result == "Launched app 'Settings' (com.apple.Preferences)." + native_driver.launch_app.assert_awaited_once_with("com.apple.Preferences") + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_sdk_rejects_unverifiable_ios_app_lock(native_driver): + agent = Agent(config=ios_config()) + task = SimpleNamespace(request=SimpleNamespace(locked_app_package="com.apple.Preferences")) + with pytest.raises(AgentError, match="foreground ownership"): + await agent._prepare_app_lock(task, ios_context(native_driver)) + + +@pytest.mark.asyncio +async def test_execute_task_ios_ignores_android_selection(monkeypatch): + builder = MagicMock() + automation = AsyncMock() + monkeypatch.setattr(run_module, "new_default_config_builder", lambda: builder) + monkeypatch.setattr(run_module, "run_automation", automation) + monkeypatch.setenv("ADB_DEVICE_SERIAL", "android-env-device") + monkeypatch.setattr( + "artemis.runtime.device_pool.select_device", + MagicMock(side_effect=AssertionError("ADB pool")), + ) + await run_module.execute_task("Open Settings", platform=DevicePlatform.IOS) + builder.for_device.assert_called_once_with(DevicePlatform.IOS, "booted") + builder.with_video_recording_tools.assert_not_called() + automation.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_execute_task_ios_forwards_workspace(native_driver, monkeypatch, tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + builder = MagicMock() + automation = AsyncMock() + monkeypatch.setattr(run_module, "new_default_config_builder", lambda: builder) + monkeypatch.setattr(run_module, "run_automation", automation) + await run_module.execute_task( + "Open Settings", platform=DevicePlatform.IOS, ios_workspace_path=project + ) + builder.with_ios_workspace.assert_called_once_with(project) + automation.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_execute_task_rejects_workspace_without_ios(native_driver, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + with pytest.raises(ValueError, match="--platform ios"): + await run_module.execute_task( + "Open Settings", platform=DevicePlatform.ANDROID, ios_workspace_path=project + ) + + +def test_cli_ios_runs_locally_without_android_status_or_daemon(monkeypatch): + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setattr( + run_module, + "display_local_device_status", + MagicMock(side_effect=AssertionError("ADB status")), + ) + monkeypatch.setattr( + "artemis.runtime.ensure_daemon_running", MagicMock(side_effect=AssertionError("daemon")) + ) + monkeypatch.delenv("ARTEMIS_TASK_WORKER", raising=False) + monkeypatch.delenv("ARTEMIS_DEVICE_QUEUE_TICKET", raising=False) + result = CliRunner().invoke( + app, + [ + "run", + "--platform", + "ios", + "--standalone", + "--device-serial", + "booted", + "Open Settings", + ], + ) + assert result.exit_code == 0, result.output + assert execute.call_args.kwargs["platform"] == DevicePlatform.IOS + assert execute.call_args.kwargs["device_serial"] == "booted" + + +def test_cli_forwards_ios_workspace(monkeypatch, tmp_path): + project = tmp_path / "My App.xcodeproj" + project.mkdir() + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + result = CliRunner().invoke( + app, + [ + "run", + "--platform", + "ios", + "--standalone", + "--ios-workspace", + str(project), + "Open Settings", + ], + ) + assert result.exit_code == 0, result.output + assert execute.call_args.kwargs["ios_workspace_path"] == project + + +def test_cli_rejects_ios_workspace_for_android(monkeypatch, tmp_path): + project = tmp_path / "App.xcodeproj" + project.mkdir() + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setattr( + "artemis.runtime.ensure_daemon_running", MagicMock(side_effect=AssertionError("daemon")) + ) + result = CliRunner().invoke(app, ["run", "--ios-workspace", str(project), "Open Settings"]) + assert result.exit_code != 0 + execute.assert_not_called() + + +def test_cli_approval_error_exits_2_with_guidance_panel(monkeypatch): + monkeypatch.setattr( + run_module, + "execute_task", + AsyncMock( + side_effect=XcodeApprovalRequiredError( + "DeviceInteractionStartSession", + "This agent isn't approved to use Xcode's tools yet.", + ) + ), + ) + result = CliRunner().invoke(app, ["run", "--platform", "ios", "--standalone", "Open Settings"]) + assert result.exit_code == 2, result.output + assert "Xcode Approval Required" in result.output + assert "Always Allow" in result.output + assert "isn't approved" in result.output + assert "Missing API Key" not in result.output + assert "GEMINI" not in result.output + + +def test_cli_rejects_ios_cloud_mode_before_run(monkeypatch): + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setenv("ARTEMIS_CLOUD_MODE", "1") + result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) + assert result.exit_code != 0 + execute.assert_not_called() + + +def test_cli_allows_ios_queue_worker(monkeypatch): + """iOS daemon workers are supported; only cloud mode is rejected.""" + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + monkeypatch.setenv("ARTEMIS_TASK_WORKER", "1") + monkeypatch.setenv("ARTEMIS_DEVICE_QUEUE_TICKET", "ticket-1") + result = CliRunner().invoke(app, ["run", "--platform", "ios", "Open Settings"]) + assert result.exit_code == 0, result.output + execute.assert_awaited_once() + assert execute.call_args.kwargs["platform"] == DevicePlatform.IOS + + +def test_ios_probe_and_pro_tool_gates(native_driver, monkeypatch): + from artemis.agents.checker.checker import probes_enabled + from artemis.tools import index + + context = ios_context(native_driver) + assert probes_enabled(context) is False + tools = [ + SimpleNamespace(name=n) + for n in ("run_adb_command", "manage_task", "save_note", "ask_explorer") + ] + monkeypatch.setattr(index, "build_tools_from_wrappers", lambda *args, **kwargs: tools) + assert [t.name for t in index.get_tools_from_wrappers(context, [])] == [ + "save_note", + "ask_explorer", + ] + + +@pytest.mark.asyncio +async def test_ios_hierarchy_tool_returns_accessibility_json(native_driver): + import json + from artemis.tools.mobile.read_hierarchy import get_ui_hierarchy, get_ui_hierarchy_tool + + data = native_driver.get_screen_data.return_value + data.ui_hierarchy_xml = None + data.ui_elements = [{"text": "설정", "bounds": "[0,0][100,100]"}] + context = ios_context(native_driver) + result = await get_ui_hierarchy.execute(driver=native_driver, ctx=context) + assert json.loads(result) == data.ui_elements + assert "iOS accessibility" in get_ui_hierarchy_tool(context).description + + +def test_flash_and_pro_prompts_teach_ios_constraints(native_driver): + from artemis.agents.flash.runner import FlashRunner + from artemis.agents.operator.prompts import ( + PLAN_HISTORY_TEMPLATE_SECTION, + render_transcript_static_system, + resolve_operator_prompt_tools, + ) + + context = ios_context(native_driver) + context.actuator = IosActuator(context) + runner = object.__new__(FlashRunner) + runner.ctx = context + runner.goal = "Open Settings" + flash = runner._render_system_prompt([]) + prompts = {"main_template": PLAN_HISTORY_TEMPLATE_SECTION + "\n# CURRENT OBSERVATION"} + pro = render_transcript_static_system(prompts, context, State.initial("Open Settings")) + for prompt in (flash, pro): + assert "Target platform: iOS." in prompt + assert "clear_exist=false" in prompt + assert ( + "press_key supports enter, home, " + "power, volume_up, volume_down, and app_switch" in prompt + ) + assert "Back/Enter/Delete" not in prompt + assert "run_adb_command" not in resolve_operator_prompt_tools(context) + assert "manage_task" not in resolve_operator_prompt_tools(context) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("profile", ["flash", "pro"]) +async def test_sdk_profiles_route_repeated_tasks_through_native_actions( + native_driver, monkeypatch, tmp_path, profile +): + import third_party.mobile_use.sdk.agent as sdk_base + + agent = Agent(config=ios_config()) + agent._tmp_traces_dir = tmp_path + agent._prewarm_llm_connections = AsyncMock() + agent._prepare_trace_paths = MagicMock(return_value="native-test") + agent._prepare_output_files = MagicMock() + agent._finalize_tracing_safely = AsyncMock() + agent._extract_output = AsyncMock(return_value="completed") + engine = MagicMock() + engine.shutdown = AsyncMock() + monkeypatch.setattr("artemis.sdk.agent.DataEngine", lambda **kwargs: engine) + lease = MagicMock() + lease.get_active_owner.return_value = None + monkeypatch.setattr(sdk_base, "DeviceExecutionLock", lease) + + class NativeFlash: + def __init__(self, context, **kwargs): + self.context = context + + async def run(self, state): + assert isinstance(self.context.actuator, IosActuator) + session = await get_action_session(self.context) + assert (await session.call("click", {"target": [500, 500]})).ok + return {"status": "completed"} + + async def native_graph(context): + assert isinstance(context.actuator, IosActuator) + + async def stream(**kwargs): + session = await get_action_session(context) + assert (await session.call("click", {"target": [500, 500]})).ok + yield "values", State.initial("Open Settings").model_dump() + + return SimpleNamespace(astream=stream) + + helper_lease = MagicMock() + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", helper_lease) + monkeypatch.setattr(sdk_base, "FlashRunner", NativeFlash) + monkeypatch.setattr(sdk_base, "get_graph", native_graph) + app_dir = tmp_path / "Demo.app" + app_dir.mkdir() + native_driver.install_app.return_value = "com.example.demo" + await agent.init() + for _ in range(2): + await agent.run_task(goal="Open Settings", profile=profile, app_path=app_dir) + await asyncio.sleep(0) + assert native_driver.tap.await_count == 2 + # Native app installation runs under the task's own lease and session, so + # no additional helper lease is constructed and no extra session is opened. + helper_lease.assert_not_called() + assert native_driver.install_app.await_count == 2 + native_driver.install_app.assert_awaited_with(app_dir) + assert native_driver.connect.await_count == 2 + assert native_driver.disconnect.await_count == 2 + native_driver.execute_shell.assert_not_awaited() + assert all(task.status == "completed" for task in agent._tasks) + + +def _png_bytes(width=1170, height=2532, color=(18, 52, 86)) -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (width, height), color).save(buffer, format="PNG") + return buffer.getvalue() + + +def _ordered_lease(order: list[str]): + class FakeLease: + def __init__(self, *args, **kwargs): + pass + + def acquire(self, *, cancel_event=None, **kwargs): + order.append("acquire") + + def release(self): + order.append("release") + + return FakeLease + + +async def _init_ios_agent(native_driver): + agent = Agent(config=ios_config()) + agent._prewarm_llm_connections = AsyncMock() + assert await agent.init() is True + await asyncio.sleep(0) + return agent + + +@pytest.mark.asyncio +async def test_ios_public_screenshot_owns_a_leased_session(native_driver, monkeypatch): + order: list[str] = [] + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", _ordered_lease(order)) + native_driver.connect.side_effect = lambda *a, **k: order.append("connect") + native_driver.disconnect.side_effect = lambda *a, **k: order.append("disconnect") + png = _png_bytes() + + async def capture(*args, **kwargs): + order.append("capture") + return ScreenData( + screenshot_bytes=png, + screenshot_base64="cG5n", + platform="ios", + width=1170, + height=2532, + ui_hierarchy_xml="", + ) + + native_driver.get_screen_data.side_effect = capture + agent = await _init_ios_agent(native_driver) + + image = await agent.get_screenshot() + assert isinstance(image, Image.Image) + assert image.size == (1170, 2532) + assert image.getpixel((0, 0)) == (18, 52, 86) + assert order == ["acquire", "connect", "capture", "disconnect", "release"] + + # A second public call opens a fresh lease and native session. + order.clear() + again = await agent.get_screenshot() + assert again.size == (1170, 2532) + assert order == ["acquire", "connect", "capture", "disconnect", "release"] + assert native_driver.connect.await_count == 2 + native_driver.execute_shell.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ios_public_install_app_owns_a_leased_session(native_driver, monkeypatch, tmp_path): + order: list[str] = [] + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", _ordered_lease(order)) + native_driver.connect.side_effect = lambda *a, **k: order.append("connect") + native_driver.disconnect.side_effect = lambda *a, **k: order.append("disconnect") + + async def install(path): + order.append("install") + return "com.example.demo" + + native_driver.install_app.side_effect = install + app_dir = tmp_path / "Demo.app" + app_dir.mkdir() + (app_dir / "Info.plist").write_bytes(b"placeholder") + agent = await _init_ios_agent(native_driver) + + bundle = await agent.install_app(app_dir) + assert bundle == "com.example.demo" + native_driver.install_app.assert_awaited_once_with(app_dir) + assert order == ["acquire", "connect", "install", "disconnect", "release"] + + +@pytest.mark.asyncio +async def test_ios_public_helpers_require_initialization(native_driver, monkeypatch, tmp_path): + lease = MagicMock() + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", lease) + agent = Agent(config=ios_config()) + with pytest.raises(AgentNotInitializedError): + await agent.get_screenshot() + with pytest.raises(AgentNotInitializedError): + await agent.install_app(tmp_path / "Demo.app") + lease.assert_not_called() + native_driver.constructor.assert_not_called() + native_driver.connect.assert_not_awaited() + + # Android selection still delegates to the base implementation unchanged. + android = Agent(config=AgentConfigBuilder().build(validate_profiles=False)) + with pytest.raises(AgentNotInitializedError): + await android.get_screenshot() + with pytest.raises(FileNotFoundError): + await android.install_app(tmp_path / "missing.apk") + + +@pytest.mark.asyncio +async def test_ios_helper_failures_release_session_and_lease(native_driver, monkeypatch, tmp_path): + order: list[str] = [] + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", _ordered_lease(order)) + native_driver.disconnect.side_effect = lambda *a, **k: order.append("disconnect") + agent = await _init_ios_agent(native_driver) + + native_driver.connect.side_effect = RuntimeError("connect failed") + with pytest.raises(RuntimeError, match="connect failed"): + await agent.get_screenshot() + assert order == ["acquire", "disconnect", "release"] + + order.clear() + native_driver.connect.side_effect = lambda *a, **k: order.append("connect") + native_driver.get_screen_data.side_effect = RuntimeError("capture failed") + with pytest.raises(RuntimeError, match="capture failed"): + await agent.get_screenshot() + assert order == ["acquire", "connect", "disconnect", "release"] + + order.clear() + native_driver.install_app.side_effect = RuntimeError("install failed") + app_dir = tmp_path / "Demo.app" + app_dir.mkdir() + with pytest.raises(RuntimeError, match="install failed"): + await agent.install_app(app_dir) + assert order == ["acquire", "connect", "disconnect", "release"] + + +@pytest.mark.asyncio +async def test_ios_operation_cancellation_drains_queued_acquire(native_driver, monkeypatch): + started = threading.Event() + finished = threading.Event() + released = MagicMock() + + class FakeLease: + def __init__(self, *args, **kwargs): + pass + + def acquire(self, *, cancel_event=None, **kwargs): + started.set() + try: + cancel_event.wait(timeout=30) + finally: + finished.set() + raise DeviceBusyError("queue wait cancelled") + + def release(self): + released() + + monkeypatch.setattr("artemis.sdk.agent.DeviceExecutionLock", FakeLease) + agent = await _init_ios_agent(native_driver) + + helper = asyncio.ensure_future(agent.get_screenshot()) + assert await asyncio.to_thread(started.wait, 10) is True + helper.cancel() + with pytest.raises(asyncio.CancelledError): + await helper + assert finished.is_set() + released.assert_called_once() + native_driver.connect.assert_not_awaited() + native_driver.get_screen_data.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_context_disconnect_driver_swallows_expected_cleanup_errors(native_driver): + native_driver.disconnect.side_effect = OSError("transport already closed") + context = ios_context(native_driver) + await context.disconnect_driver() + assert context._active_driver is None + + native_driver.disconnect.side_effect = KeyError("unexpected") + context = ios_context(native_driver) + with pytest.raises(KeyError): + await context.disconnect_driver() + + +@pytest.mark.parametrize( + "flag", ["--with-video-recording-tools", "--without-video-recording-tools"] +) +def test_cli_ios_accepts_video_flag_in_standalone(monkeypatch, flag): + execute = AsyncMock() + monkeypatch.setattr(run_module, "execute_task", execute) + result = CliRunner().invoke( + app, ["run", "--platform", "ios", "--standalone", flag, "Open Settings"] + ) + assert result.exit_code == 0, result.output + execute.assert_awaited_once() + + +def _unconfigured_agent_cfg(monkeypatch, builder_module): + """Neutralize any configured video_analyzer.enabled so detection runs.""" + real_load = builder_module.load_agent_config + + def load(): + cfg = real_load() + cfg.video_analyzer.enabled = None + return cfg + + monkeypatch.setattr(builder_module, "load_agent_config", load) + + +def test_builder_auto_detects_video_tools_per_platform(monkeypatch): + from artemis.sdk.builders import agent_config_builder as builder_module + + _unconfigured_agent_cfg(monkeypatch, builder_module) + observed: list[str] = [] + monkeypatch.setattr( + builder_module, + "detect_video_tools_enabled", + lambda platform="android": observed.append(platform) or platform == "ios", + ) + + ios_config = AgentConfigBuilder().for_ios_simulator().build(validate_profiles=False) + assert ios_config.video_recording_tools_enabled is True + assert observed[-1] == "ios" + + android_config = AgentConfigBuilder().build(validate_profiles=False) + assert android_config.video_recording_tools_enabled is False + assert observed[-1] == "android" + + +def test_builder_explicit_video_flag_wins_regardless_of_order(monkeypatch): + from artemis.sdk.builders import agent_config_builder as builder_module + + monkeypatch.setattr( + builder_module, + "detect_video_tools_enabled", + lambda platform="android": True, + ) + + before = ( + AgentConfigBuilder() + .with_video_recording_tools(enabled=False) + .for_ios_simulator() + .build(validate_profiles=False) + ) + after = ( + AgentConfigBuilder() + .for_ios_simulator() + .with_video_recording_tools(enabled=False) + .build(validate_profiles=False) + ) + enabled = ( + AgentConfigBuilder() + .for_ios_simulator() + .with_video_recording_tools(enabled=True) + .build(validate_profiles=False) + ) + assert before.video_recording_tools_enabled is False + assert after.video_recording_tools_enabled is False + assert enabled.video_recording_tools_enabled is True + + +def test_agent_config_default_factory_detects_platform_from_data(monkeypatch): + from artemis.sdk.types import agent as agent_types + + monkeypatch.setattr( + agent_types, + "detect_video_tools_enabled", + lambda platform="android": platform == "ios", + ) + from artemis.sdk.types.agent import AgentConfig + + data = AgentConfigBuilder().build(validate_profiles=False).model_dump() + data.pop("video_recording_tools_enabled") + data["device_platform"] = DevicePlatform.IOS + ios = AgentConfig.model_validate(data) + data["device_platform"] = DevicePlatform.ANDROID + android = AgentConfig.model_validate(data) + explicit = AgentConfig.model_validate( + {**data, "device_platform": DevicePlatform.IOS, "video_recording_tools_enabled": False} + ) + assert ios.video_recording_tools_enabled is True + assert android.video_recording_tools_enabled is False + assert explicit.video_recording_tools_enabled is False diff --git a/tests/unit/test_ios_observation.py b/tests/unit/test_ios_observation.py new file mode 100644 index 00000000..4abd95c9 --- /dev/null +++ b/tests/unit/test_ios_observation.py @@ -0,0 +1,103 @@ +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native hit points must survive the shared perception-to-action pipeline.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from PIL import Image +import pytest + +from artemis.agents.explorer.screen_index import ScreenIndex +from artemis.context import ArtemisContext, DeviceContext, DevicePlatform +from artemis.drivers.ios.hierarchy import parse_hierarchy +from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver +from artemis.mcp.action_executor import McpActionExecutor +from artemis.mcp.actuators.ios import IosActuator +from artemis.mcp.observation import observe +from artemis.utils.visualization import ( + format_minimal_list_with_elements, + format_minimal_list_with_points, +) + + +HIERARCHY = """Application bundle identifier: com.example.Test +Application, pid: 123, label: ' ' + Window, {{0.0, 0.0}, {400.0, 800.0}}, hitPoint: {200.0, 400.0} + UIButton \"Continue\" {{100.0, 200.0}, {60.0, 30.0}}, hitPoint: {150.0, 220.0}, identifier: 'continueButton', activationBundleId: com.example.Test +""" + + +@pytest.mark.asyncio +async def test_ios_observation_preserves_hit_point_for_indexed_native_tap(tmp_path, monkeypatch): + screenshot = tmp_path / "native.png" + Image.new("RGB", (800, 1600)).save(screenshot) + hierarchy = tmp_path / "native.txt" + hierarchy.write_text(HIERARCHY) + driver = XcodeSimulatorDriver("simulator-id") + driver._session_key = "session" + driver._bridge = SimpleNamespace( + call=AsyncMock( + return_value={"screenshotPath": str(screenshot), "hierarchyPath": str(hierarchy)} + ) + ) + context = ArtemisContext( + device=DeviceContext(mobile_platform=DevicePlatform.IOS, device_id="simulator-id") + ) + context._active_driver = driver + actuator = IosActuator(context) + monkeypatch.setattr("artemis.mcp.observation.get_temp_dir", lambda name: tmp_path) + + observation, image = await observe(context, actuator.controller, settle_ms=0) + assert observation.ok and observation.hierarchy_ok and image == screenshot.read_bytes() + assert len(observation.elements) == 1 + element = observation.elements[0] + assert element["center"] == element["hit_point"] == [300, 440] + assert element["resource_id"] == "continueButton" + assert element["activation_bundle_id"] == "com.example.Test" + assert "Continue" in observation.elements_text + + state = SimpleNamespace(indexed_elements=observation.elements) + executor = McpActionExecutor(context, actuator=actuator) + name, arguments, _, _ = executor._translate("click", {"target": 1}, state) + assert name == "click" and arguments["target"] == [375, 275] + assert (await actuator.click(*arguments["target"])).ok + command = driver._bridge.call.await_args.args[1]["interactionCommand"] + assert command == "t 150.0000 220.0000 0.100" + assert driver._bridge.call.await_args.args[1]["activationBundleId"] == "com.example.Test" + + +def test_ios_explorer_and_point_formatter_keep_native_hit_point(): + elements, _ = parse_hierarchy(HIERARCHY, 800, 1600) + _, points, _ = format_minimal_list_with_points(elements, 800, 1600) + assert points == [[300, 440]] + index = ScreenIndex.from_hierarchy(elements, 800, 1600) + match = index.search_text("Continue")[0].element + assert match.center == (300, 440) + assert match.resource_id == "continueButton" + + +@pytest.mark.parametrize("point", [None, [True, 20], [float("nan"), 20], [800, 200], [-1, 10], [1]]) +def test_shared_formatter_keeps_android_centroid_for_missing_or_invalid_native_point(point): + node = { + "text": "Continue", + "bounds": "[200,400][320,460]", + "resource-id": "android:id/button1", + } + if point is not None: + node["hit_point"] = point + _, elements, _ = format_minimal_list_with_elements([node], 800, 1600) + _, points, _ = format_minimal_list_with_points([node], 800, 1600) + assert elements[0]["center"] == points[0] == [260, 430] + assert elements[0]["resource_id"] == "android:id/button1" + assert "hit_point" not in elements[0] + assert ScreenIndex.from_hierarchy([node], 800, 1600).elements[0].center == (260, 430) diff --git a/tests/unit/test_ios_parity.py b/tests/unit/test_ios_parity.py new file mode 100644 index 00000000..b0e14d62 --- /dev/null +++ b/tests/unit/test_ios_parity.py @@ -0,0 +1,1099 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""iOS Simulator platform-parity tests: discovery, pool, locks, queue, routing. + +Every test is hermetic — ``simctl``/``xcrun``/adb are never invoked; device +enumeration is stubbed at the discovery boundary. +""" + +import os +from unittest.mock import AsyncMock, MagicMock, Mock + +import pytest + +from artemis.drivers.ios.discovery import ( + clear_ios_simulator_cache, + list_ios_simulators, + list_ios_simulators_sync, + parse_simctl_devices, +) +from artemis.runtime.adb_endpoint import AdbEndpoint, AdbTarget, IOS_LOCK_SCOPE, IosTarget +from artemis.runtime.device_lock import DeviceExecutionLock, DeviceLockOwner +from artemis.runtime.ios_device_pool import IosDevicePool + + +@pytest.fixture(autouse=True) +def isolated_lock_directory(tmp_path, monkeypatch): + monkeypatch.setattr( + "artemis.runtime.device_lock.get_temp_dir", + lambda _name: tmp_path, + ) + + +@pytest.fixture(autouse=True) +def fresh_simulator_enumeration(): + clear_ios_simulator_cache() + yield + clear_ios_simulator_cache() + + +SIMCTL_PAYLOAD = { + "devices": { + "com.apple.CoreSimulator.SimRuntime.iOS-26-0": [ + { + "udid": "AAAA-1111", + "name": "iPhone 17 Pro", + "state": "Booted", + "isAvailable": True, + }, + { + "udid": "BBBB-2222", + "name": "iPhone 17", + "state": "Shutdown", + "isAvailable": True, + }, + { + "udid": "CCCC-3333", + "name": "iPad Unavailable", + "state": "Shutdown", + "isAvailable": False, + }, + ], + "com.apple.CoreSimulator.SimRuntime.watchOS-26-0": [ + { + "udid": "DDDD-4444", + "name": "Watch", + "state": "Shutdown", + "isAvailable": True, + } + ], + "com.apple.CoreSimulator.SimRuntime.iOS-26-1": [ + { + "udid": "", + "name": "No UDID", + "state": "Shutdown", + "isAvailable": True, + } + ], + } +} + +SIM_LIST = parse_simctl_devices(SIMCTL_PAYLOAD) + + +# --------------------------------------------------------------------------- # +# discovery +# --------------------------------------------------------------------------- # + + +def test_parse_simctl_devices_filters_ios_available_only(): + devices = parse_simctl_devices(SIMCTL_PAYLOAD) + udids = {d["udid"] for d in devices} + assert udids == {"AAAA-1111", "BBBB-2222"} + by_udid = {d["udid"]: d for d in devices} + assert by_udid["AAAA-1111"]["state"] == "Booted" + assert by_udid["AAAA-1111"]["name"] == "iPhone 17 Pro" + assert "iOS-26-0" in by_udid["BBBB-2222"]["runtime"] + + +def test_parse_simctl_devices_empty_payload(): + assert parse_simctl_devices({}) == [] + assert parse_simctl_devices({"devices": {}}) == [] + + +@pytest.mark.asyncio +async def test_list_ios_simulators_returns_none_on_failure(monkeypatch): + monkeypatch.setattr( + "artemis.drivers.ios.discovery.run_xcrun", + AsyncMock(side_effect=RuntimeError("no xcode")), + ) + assert await list_ios_simulators() is None + + +@pytest.mark.asyncio +async def test_list_ios_simulators_parses_json(monkeypatch): + import json + + monkeypatch.setattr( + "artemis.drivers.ios.discovery.run_xcrun", + AsyncMock(return_value=json.dumps(SIMCTL_PAYLOAD).encode()), + ) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + devices = await list_ios_simulators() + assert {d["udid"] for d in devices} == {"AAAA-1111", "BBBB-2222"} + + +def test_list_ios_simulators_sync_skips_without_simctl(monkeypatch): + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: False) + assert list_ios_simulators_sync() is None + + +def test_list_ios_simulators_sync_parses_json(monkeypatch): + import json + import subprocess + + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps(SIMCTL_PAYLOAD).encode() + ) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + monkeypatch.setattr("artemis.drivers.ios.discovery.subprocess.run", lambda *a, **k: completed) + devices = list_ios_simulators_sync() + assert {d["udid"] for d in devices} == {"AAAA-1111", "BBBB-2222"} + + +@pytest.mark.asyncio +async def test_list_ios_simulators_reuses_cached_enumeration(monkeypatch): + import json + + run_xcrun = AsyncMock(return_value=json.dumps(SIMCTL_PAYLOAD).encode()) + monkeypatch.setattr("artemis.drivers.ios.discovery.run_xcrun", run_xcrun) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + assert await list_ios_simulators() is not None + assert await list_ios_simulators() is not None + # Second call inside the TTL serves the cache — no second simctl spawn. + assert run_xcrun.await_count == 1 + + +@pytest.mark.asyncio +async def test_list_ios_simulators_does_not_cache_failures(monkeypatch): + import json + + run_xcrun = AsyncMock( + side_effect=[RuntimeError("cold simctl"), json.dumps(SIMCTL_PAYLOAD).encode()] + ) + monkeypatch.setattr("artemis.drivers.ios.discovery.run_xcrun", run_xcrun) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + assert await list_ios_simulators() is None + assert await list_ios_simulators() is not None + assert run_xcrun.await_count == 2 + + +@pytest.mark.asyncio +async def test_list_ios_simulators_force_refresh_bypasses_cache(monkeypatch): + import json + + run_xcrun = AsyncMock(return_value=json.dumps(SIMCTL_PAYLOAD).encode()) + monkeypatch.setattr("artemis.drivers.ios.discovery.run_xcrun", run_xcrun) + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + await list_ios_simulators() + await list_ios_simulators(force_refresh=True) + assert run_xcrun.await_count == 2 + + +def test_list_ios_simulators_sync_shares_cache(monkeypatch): + import json + import subprocess + + completed = subprocess.CompletedProcess( + args=[], returncode=0, stdout=json.dumps(SIMCTL_PAYLOAD).encode() + ) + calls = [] + monkeypatch.setattr("artemis.drivers.ios.discovery.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.drivers.ios.discovery.subprocess.run", + lambda *a, **k: calls.append(1) or completed, + ) + assert list_ios_simulators_sync() is not None + assert list_ios_simulators_sync() is not None + assert len(calls) == 1 + + +# --------------------------------------------------------------------------- # +# IosTarget / lock scope +# --------------------------------------------------------------------------- # + + +def test_ios_target_lock_scope_and_key(): + target = IosTarget(serial="AAAA-1111") + assert target.platform == "ios" + assert target.lock_scope == IOS_LOCK_SCOPE + assert target.lock_key == f"{IOS_LOCK_SCOPE}/AAAA-1111" + assert target.to_dict() == {"platform": "ios", "serial": "AAAA-1111"} + + +def test_ios_target_apply_to_environment_sets_scope_and_scrubs_adb(monkeypatch): + env = {"ADB_DEVICE_SERIAL": "emulator-5554", "ADB_HOST": "10.0.0.1"} + IosTarget(serial="AAAA-1111").apply_to_environment(env) + assert env[DeviceExecutionLock.LOCK_SCOPE_ENV] == IOS_LOCK_SCOPE + # iOS workers never touch ADB: a stale serial must not leak in. + assert "ADB_DEVICE_SERIAL" not in env + + +def test_ios_and_android_locks_do_not_collide(): + """A UDID and an ADB serial with identical text lock independently.""" + shared_id = "emulator-5554" + ios_lock = DeviceExecutionLock(shared_id, "ios task", lock_scope=IOS_LOCK_SCOPE) + android_lock = DeviceExecutionLock(shared_id, "android task") + assert ios_lock.clean_device_id != android_lock.clean_device_id + + ios_lock.acquire() + try: + owners = list(DeviceExecutionLock.get_active_owners().values()) + ios_owner = next(o for o in owners if o.device_id == shared_id) + assert ios_owner.lock_scope == IOS_LOCK_SCOPE + # Android lock for the same text is unaffected and acquirable. + android_lock.acquire(blocking=False) + try: + owners = list(DeviceExecutionLock.get_active_owners().values()) + by_scope = {o.lock_scope: o for o in owners if o.device_id == shared_id} + assert IOS_LOCK_SCOPE in by_scope + assert len(by_scope) == 2 + finally: + android_lock.release() + finally: + ios_lock.release() + + +def test_get_active_owner_scoped_lookup_isolates_platform(monkeypatch): + monkeypatch.delenv(DeviceExecutionLock.LOCK_SCOPE_ENV, raising=False) + shared_id = "emulator-5554" + ios_lock = DeviceExecutionLock(shared_id, "ios task", lock_scope=IOS_LOCK_SCOPE) + ios_lock.acquire() + try: + owner = DeviceExecutionLock.get_active_owner(shared_id, lock_scope=IOS_LOCK_SCOPE) + assert owner is not None and owner.description == "ios task" + # An unscoped (Android) lookup must not see the iOS owner. + assert DeviceExecutionLock.get_active_owner(shared_id, lock_scope=None) is None + finally: + ios_lock.release() + + +# --------------------------------------------------------------------------- # +# IosDevicePool +# --------------------------------------------------------------------------- # + + +def _pool_with_devices(monkeypatch, devices=None): + import importlib + + pool = IosDevicePool() + devices = SIM_LIST if devices is None else devices + # artemis.runtime.ios_device_pool resolves to the singleton instance in the + # package namespace; patch the module object itself. + module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(devices))) + monkeypatch.setattr(module, "list_ios_simulators_sync", lambda: list(devices)) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=[])) + monkeypatch.setattr(module, "list_core_devices_sync", lambda: []) + return pool + + +@pytest.mark.asyncio +async def test_ios_pool_statuses_carry_platform_and_runtime(monkeypatch): + pool = _pool_with_devices(monkeypatch) + devices = await pool.list_devices_async() + by_serial = {d.serial: d for d in devices} + booted = by_serial["AAAA-1111"] + assert booted.platform == "ios" + assert booted.state == "device" # Booted maps to the generic ready state + assert booted.is_emulator is True + assert "iOS 26 0" in booted.product + shutdown = by_serial["BBBB-2222"] + assert shutdown.state == "Shutdown" + + +@pytest.mark.asyncio +async def test_ios_pool_busy_status_uses_ios_lock_scope(monkeypatch): + pool = _pool_with_devices(monkeypatch) + lock = DeviceExecutionLock("AAAA-1111", "ios task", lock_scope=IOS_LOCK_SCOPE) + lock.acquire() + try: + devices = await pool.list_devices_async() + by_serial = {d.serial: d for d in devices} + assert by_serial["AAAA-1111"].is_busy is True + assert by_serial["AAAA-1111"].active_task_desc == "ios task" + assert by_serial["BBBB-2222"].is_busy is False + finally: + lock.release() + + +@pytest.mark.asyncio +async def test_ios_pool_android_lock_does_not_mark_ios_busy(monkeypatch): + """An Android lock on the same text must not mark the simulator busy.""" + pool = _pool_with_devices(monkeypatch) + lock = DeviceExecutionLock("AAAA-1111", "android task") + lock.acquire() + try: + devices = await pool.list_devices_async() + by_serial = {d.serial: d for d in devices} + assert by_serial["AAAA-1111"].is_busy is False + finally: + lock.release() + + +@pytest.mark.asyncio +async def test_ios_pool_validate_explicit_serial(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert await pool.validate_explicit_serial_async("AAAA-1111") is None + assert await pool.validate_explicit_serial_async("bbbb-2222") is None + rejection = await pool.validate_explicit_serial_async("NOPE-9999") + assert rejection is not None and "not available" in rejection + # Shutdown sims are valid; unavailable devices were filtered at parse time. + assert await pool.validate_explicit_serial_async("CCCC-3333") is not None + + +@pytest.mark.asyncio +async def test_ios_pool_validate_fails_open_on_enumeration_error(monkeypatch): + pool = IosDevicePool() + import importlib + + module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=None)) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=None)) + assert await pool.validate_explicit_serial_async("ANY") is None + + +def test_ios_pool_validate_explicit_serial_sync(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert pool.validate_explicit_serial("AAAA-1111") is None + rejection = pool.validate_explicit_serial("NOPE-9999") + assert rejection is not None and "not available" in rejection + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_prefers_booted_and_idle(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert await pool.select_device_async() == "AAAA-1111" + # Explicit preference always wins. + assert await pool.select_device_async(preferred_serial="BBBB-2222") == "BBBB-2222" + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_never_picks_physical(monkeypatch): + """Auto-selection must never target paired hardware without an explicit serial.""" + import importlib + + physical = { + "udid": "00008130-0000ABCD1234FFFF", + "name": "Test iPhone", + "reality": "physical", + "platform": "iOS", + "pairing_state": "paired", + "connection_state": "connected", + "os_version": "26.0", + } + module = importlib.import_module("artemis.runtime.ios_device_pool") + pool = IosDevicePool() + + # Physical alone: nothing to auto-pick. + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=[])) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=[physical])) + assert await pool.select_device_async() is None + + # Physical alongside one booted sim: only the sim is eligible. + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(SIM_LIST))) + assert await pool.select_device_async() == "AAAA-1111" + + # Explicit serial still passes through untouched. + assert await pool.select_device_async(preferred_serial=physical["udid"]) == physical["udid"] + + +@pytest.mark.asyncio +async def test_ios_pool_validate_fails_open_on_partial_enumeration(monkeypatch): + """A dead devicectl enumeration cannot disprove a physical serial.""" + import importlib + + module = importlib.import_module("artemis.runtime.ios_device_pool") + pool = IosDevicePool() + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=list(SIM_LIST))) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=None)) + assert await pool.validate_explicit_serial_async("00008130-0000ABCD1234FFFF") is None + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_boots_single_shutdown_sim(monkeypatch): + pool = _pool_with_devices(monkeypatch, [d for d in SIM_LIST if d["udid"] == "BBBB-2222"]) + assert await pool.select_device_async() == "BBBB-2222" + + +@pytest.mark.asyncio +async def test_ios_pool_select_device_skips_busy_booted(monkeypatch): + two_booted = [ + {**SIM_LIST[0]}, + {**SIM_LIST[1], "state": "Booted"}, + ] + pool = _pool_with_devices(monkeypatch, two_booted) + lock = DeviceExecutionLock("AAAA-1111", "ios task", lock_scope=IOS_LOCK_SCOPE) + lock.acquire() + try: + assert await pool.select_device_async() == "BBBB-2222" + finally: + lock.release() + + +# --------------------------------------------------------------------------- # +# Task queue routing +# --------------------------------------------------------------------------- # + + +def test_task_target_routes_ios_to_ios_target(): + from apps.admin_console.services.task_queue_service import TaskQueueService + + target = TaskQueueService._task_target({"platform": "ios", "device_serial": "AAAA-1111"}) + assert isinstance(target, IosTarget) + assert target.serial == "AAAA-1111" + + +def test_task_target_defaults_to_android_adb_target(): + from apps.admin_console.services.task_queue_service import TaskQueueService + + target = TaskQueueService._task_target({"device_serial": "emulator-5554"}) + assert isinstance(target, AdbTarget) + assert target.serial == "emulator-5554" + + +def test_build_worker_invocation_ios(monkeypatch): + from apps.admin_console.services.task_queue_service import TaskQueueService + + monkeypatch.delenv("ADB_DEVICE_SERIAL", raising=False) + cmd, env = TaskQueueService._build_worker_invocation( + { + "platform": "ios", + "device_serial": "AAAA-1111", + "ios_workspace": "/proj/App.xcodeproj", + }, + run_key="rk", + sess_id="sess-1", + goal="open settings", + profile="flash", + target=IosTarget(serial="AAAA-1111"), + ) + assert "--platform" in cmd + assert cmd[cmd.index("--platform") + 1] == "ios" + assert cmd[cmd.index("--device-serial") + 1] == "AAAA-1111" + assert cmd[cmd.index("--ios-workspace") + 1] == "/proj/App.xcodeproj" + assert env[DeviceExecutionLock.LOCK_SCOPE_ENV] == IOS_LOCK_SCOPE + assert "ADB_DEVICE_SERIAL" not in env + + +def test_build_worker_invocation_android_unchanged(): + from apps.admin_console.services.task_queue_service import TaskQueueService + + endpoint = AdbEndpoint.local() + target = AdbTarget(endpoint=endpoint, serial="emulator-5554") + cmd, env = TaskQueueService._build_worker_invocation( + {"device_serial": "emulator-5554"}, + run_key="rk", + sess_id="sess-1", + goal="open settings", + profile="flash", + target=target, + ) + assert "--platform" not in cmd + assert cmd[cmd.index("--device-serial") + 1] == "emulator-5554" + assert env["ADB_DEVICE_SERIAL"] == "emulator-5554" + assert env[DeviceExecutionLock.LOCK_SCOPE_ENV] == endpoint.identity + + +@pytest.mark.asyncio +async def test_enqueue_tasks_ios_carries_platform_and_lock_scope(monkeypatch): + from apps.admin_console.services.task_queue_service import TaskQueueService + from apps.admin_console.core.state import state + + monkeypatch.setattr(TaskQueueService, "ensure_worker_running", staticmethod(lambda: None)) + monkeypatch.setattr( + TaskQueueService, + "_find_duplicate_submission", + classmethod(lambda cls, *a, **k: None), + ) + monkeypatch.setattr( + IosDevicePool, + "validate_explicit_serial_async", + AsyncMock(return_value=None), + ) + state.queue_items.clear() + try: + resp = await TaskQueueService.enqueue_tasks( + ["open settings"], + "flash", + None, + None, + None, + None, + "AAAA-1111", + "frontend", + session_id="sess-1", + conversation_id=None, + platform="ios", + ios_workspace="/proj/App.xcodeproj", + ) + item = state.queue_items[-1] + assert item["platform"] == "ios" + assert item["ios_workspace"] == "/proj/App.xcodeproj" + assert item["device_serial"] == "AAAA-1111" + # The queue reservation was scoped ios — check via the target builder. + target = TaskQueueService._task_target(item) + assert target.lock_scope == IOS_LOCK_SCOPE + assert not resp.get("rejected") + finally: + state.queue_items.clear() + + +# --------------------------------------------------------------------------- # +# Stream service target resolution +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_stream_target_prefers_active_ios_owner(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + owner = DeviceLockOwner( + pid=os.getpid(), + process_created_at=0.0, + token="t", + device_id="AAAA-1111", + description="ios task", + acquired_at="now", + lock_scope=IOS_LOCK_SCOPE, + ) + monkeypatch.setattr( + DeviceExecutionLock, + "get_active_owners", + classmethod(lambda cls: {"ios__AAAA-1111": owner}), + ) + service = DeviceStreamService() + target = await service.get_stream_target() + assert target == {"platform": "ios", "serial": "AAAA-1111"} + + +@pytest.mark.asyncio +async def test_stream_target_falls_back_to_android(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {})) + service = DeviceStreamService() + monkeypatch.setattr(service, "_android_serial", AsyncMock(return_value="emulator-5554")) + monkeypatch.setattr(IosDevicePool, "select_device_async", AsyncMock(return_value=None)) + target = await service.get_stream_target() + assert target == {"platform": "android", "serial": "emulator-5554"} + + +@pytest.mark.asyncio +async def test_stream_target_falls_back_to_booted_sim(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {})) + service = DeviceStreamService() + monkeypatch.setattr(service, "_android_serial", AsyncMock(return_value=None)) + monkeypatch.setattr(IosDevicePool, "select_device_async", AsyncMock(return_value="AAAA-1111")) + target = await service.get_stream_target() + assert target == {"platform": "ios", "serial": "AAAA-1111"} + + +@pytest.mark.asyncio +async def test_stream_target_none_without_devices(monkeypatch): + from apps.admin_console.services.device_stream_service import DeviceStreamService + + monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {})) + service = DeviceStreamService() + monkeypatch.setattr(service, "_android_serial", AsyncMock(return_value=None)) + monkeypatch.setattr(IosDevicePool, "select_device_async", AsyncMock(return_value=None)) + assert await service.get_stream_target() is None + + +# --------------------------------------------------------------------------- # +# MCP task runner platform routing +# --------------------------------------------------------------------------- # + + +def test_mcp_validate_device_serial_ios_uses_ios_pool(monkeypatch): + from artemis.runtime import device_pool as adb_pool + from artemis.runtime.ios_device_pool import ios_device_pool + from mcp_server.tools import task_runner + + ios_validate = MagicMock(return_value=None) + adb_validate = MagicMock(return_value="should not be called") + monkeypatch.setattr(ios_device_pool, "validate_explicit_serial", ios_validate) + monkeypatch.setattr(adb_pool, "validate_explicit_serial", adb_validate) + assert task_runner._validate_device_serial("AAAA-1111", "ios") is None + ios_validate.assert_called_once_with("AAAA-1111") + adb_validate.assert_not_called() + + +def test_mcp_validate_device_serial_ios_rejection(monkeypatch): + from artemis.runtime.ios_device_pool import ios_device_pool + from mcp_server.tools import task_runner + + monkeypatch.setattr( + ios_device_pool, + "validate_explicit_serial", + lambda serial: f"iOS simulator '{serial}' is not available.", + ) + result = task_runner._validate_device_serial("NOPE", "ios") + assert result is not None and result["status"] == "failed" + assert "simctl" in result["message"] + + +# --------------------------------------------------------------------------- # +# Daemon client payloads +# --------------------------------------------------------------------------- # + + +def test_submit_task_to_daemon_forwards_platform(monkeypatch): + from artemis.runtime import daemon_client + + captured = {} + + class FakeResponse: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return b'{"status": "enqueued"}' + + def fake_urlopen(req, timeout=0): + import json + + captured.update(json.loads(req.data.decode())) + return FakeResponse() + + monkeypatch.setattr("artemis.runtime.daemon_client.urllib.request.urlopen", fake_urlopen) + resp = daemon_client.submit_task_to_daemon( + "open settings", + device_serial="AAAA-1111", + platform="ios", + ios_workspace="/proj/App.xcodeproj", + ) + assert resp == {"status": "enqueued"} + assert captured["platform"] == "ios" + assert captured["ios_workspace"] == "/proj/App.xcodeproj" + assert captured["device_serial"] == "AAAA-1111" + + +def test_submit_batch_to_daemon_forwards_platform(monkeypatch): + from artemis.runtime import daemon_client + + captured = {} + + class FakeResponse: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return b'{"status": "enqueued"}' + + def fake_urlopen(req, timeout=0): + import json + + captured.update(json.loads(req.data.decode())) + return FakeResponse() + + monkeypatch.setattr("artemis.runtime.daemon_client.urllib.request.urlopen", fake_urlopen) + daemon_client.submit_batch_to_daemon( + ["a", "b"], + device_serial="AAAA-1111", + platform="ios", + ios_workspace="/proj/App.xcodeproj", + ) + assert captured["platform"] == "ios" + assert captured["ios_workspace"] == "/proj/App.xcodeproj" + + +# --------------------------------------------------------------------------- # +# Replay context platform preservation +# --------------------------------------------------------------------------- # + + +def test_replay_device_context_preserves_ios_platform(): + from apps.admin_console.replay_manager import ReplayManager + from artemis.context import DevicePlatform + + ctx = ReplayManager._replay_device_context( + { + "mobile_platform": "ios", + "device_id": "AAAA-1111", + "device_width": 1206, + "device_height": 2622, + }, + override_device_id="BBBB-2222", + ) + assert ctx.mobile_platform == DevicePlatform.IOS + assert ctx.device_id == "BBBB-2222" + assert ctx.device_width == 1206 + + +def test_replay_device_context_android_ignores_override(): + from apps.admin_console.replay_manager import ReplayManager + from artemis.context import DevicePlatform + + ctx = ReplayManager._replay_device_context( + {"mobile_platform": "android", "device_id": "emulator-5554"}, + override_device_id="AAAA-1111", + ) + assert ctx.mobile_platform == DevicePlatform.ANDROID + # Android replay keeps the recorded device; the picker stays decorative. + assert ctx.device_id == "emulator-5554" + + +def test_replay_device_context_missing_platform_defaults_android(): + from apps.admin_console.replay_manager import ReplayManager + from artemis.context import DevicePlatform + + ctx = ReplayManager._replay_device_context({"device_id": "emulator-5554"}, None) + assert ctx.mobile_platform == DevicePlatform.ANDROID + assert ctx.device_id == "emulator-5554" + + +# --------------------------------------------------------------------------- # +# iOS readiness probe +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_ios_probe_skipped_without_simctl(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: False) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.SKIPPED + assert result.is_blocker is False + + +@pytest.mark.asyncio +async def test_ios_probe_warns_on_old_xcode(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.run_xcrun", + AsyncMock(return_value=b"Xcode 16.4\nBuild version 16F6"), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", + AsyncMock(return_value=SIM_LIST), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_core_devices", + AsyncMock(return_value=[]), + ) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.WARN + assert result.metadata["xcode_27_or_newer"] is False + + +@pytest.mark.asyncio +async def test_ios_probe_passes_with_xcode27_and_sims(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.run_xcrun", + AsyncMock(return_value=b"Xcode 27.0\nBuild version 27A266a"), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", + AsyncMock(return_value=SIM_LIST), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_core_devices", + AsyncMock(return_value=[]), + ) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.PASS + assert result.metadata["simulator_count"] == 2 + assert result.metadata["booted_udids"] == ["AAAA-1111"] + assert result.is_blocker is False + + +@pytest.mark.asyncio +async def test_ios_probe_warns_without_simulators(monkeypatch): + from artemis.core.diagnostics.probes.ios_probe import IosSimulatorProbe + from artemis.core.diagnostics.schema import ProbeStatus + + monkeypatch.setattr("artemis.core.diagnostics.probes.ios_probe.simctl_available", lambda: True) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.run_xcrun", + AsyncMock(return_value=b"Xcode 27.0"), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_ios_simulators", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "artemis.core.diagnostics.probes.ios_probe.list_core_devices", + AsyncMock(return_value=[]), + ) + result = await IosSimulatorProbe().probe() + assert result.status is ProbeStatus.WARN + assert "No Devices" in result.summary + + +def test_ios_probe_registered_in_engine(): + from artemis.core.diagnostics.engine import readiness_engine + + assert "ios_simulators" in readiness_engine._probes + assert readiness_engine._probes["ios_simulators"].is_blocker is False + + +# --------------------------------------------------------------------------- # +# web: platform-aware device selection +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_select_device_ios_validates_udid_and_skips_adb_target(monkeypatch): + from unittest.mock import AsyncMock + + from apps.admin_console.routers import system as system_router + from apps.admin_console.routers.system import SelectDeviceRequest + + validate = AsyncMock(return_value=None) + monkeypatch.setattr(IosDevicePool, "validate_explicit_serial_async", validate) + set_target = Mock() + monkeypatch.setattr(system_router.readiness_engine, "set_probe_target_serial", set_target) + + result = await system_router.select_active_device( + SelectDeviceRequest(serial="AAAA-1111", platform="ios") + ) + + assert result["status"] == "success" + assert result["selected_serial"] == "AAAA-1111" + assert result["platform"] == "ios" + validate.assert_awaited_once_with("AAAA-1111") + set_target.assert_not_called() + + +@pytest.mark.asyncio +async def test_select_device_ios_rejects_unknown_udid(monkeypatch): + from unittest.mock import AsyncMock + + from fastapi import HTTPException + + from apps.admin_console.routers.system import SelectDeviceRequest, select_active_device + + monkeypatch.setattr( + IosDevicePool, + "validate_explicit_serial_async", + AsyncMock(return_value="iOS Simulator UDID 'NOPE' was not found."), + ) + + with pytest.raises(HTTPException) as exc_info: + await select_active_device(SelectDeviceRequest(serial="NOPE", platform="ios")) + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_select_device_rejects_unknown_platform(): + from fastapi import HTTPException + + from apps.admin_console.routers.system import SelectDeviceRequest, select_active_device + + with pytest.raises(HTTPException) as exc_info: + await select_active_device(SelectDeviceRequest(serial="dev-1", platform="tvos")) + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_select_device_android_unchanged(monkeypatch): + from unittest.mock import AsyncMock + + from apps.admin_console.routers import system as system_router + from apps.admin_console.routers.system import SelectDeviceRequest + + set_target = Mock() + report = object() + monkeypatch.setattr(system_router.readiness_engine, "set_probe_target_serial", set_target) + monkeypatch.setattr(system_router.readiness_engine, "run_all", AsyncMock(return_value=report)) + + result = await system_router.select_active_device(SelectDeviceRequest(serial="emulator-5554")) + + assert result["status"] == "success" + assert result["selected_serial"] == "emulator-5554" + set_target.assert_called_once_with("emulator-5554") + + +def test_ios_press_key_vocabulary_passes_the_operator_gate(): + """Every key the iOS guidance advertises must survive the Operator key gate.""" + from artemis.agents.operator.operator import SUPPORTED_PRESS_KEYS + + ios_supported = {"enter", "home", "power", "volume_up", "volume_down", "app_switch"} + assert ios_supported <= {key.lower() for key in SUPPORTED_PRESS_KEYS} + + +# --------------------------------------------------------------------------- # +# lazy MCP controller selection +# --------------------------------------------------------------------------- # + + +def test_lazy_mcp_ios_controller_uses_ios_context_and_isolated_cache(monkeypatch): + """The lazy MCP path builds an iOS DeviceContext without a native session.""" + from types import SimpleNamespace + + from artemis.context import DevicePlatform + from artemis.mcp import adb_server + + captured = {} + + def fake_controller(ctx): + captured["ctx"] = ctx + return SimpleNamespace(ctx=ctx) + + monkeypatch.setattr(adb_server, "_CONTROLLERS", {}) + monkeypatch.setattr(adb_server, "_GLOBAL_CONTROLLER", None) + monkeypatch.setattr(adb_server, "UnifiedMobileController", fake_controller) + monkeypatch.delenv("ARTEMIS_DEVICE_ID", raising=False) + monkeypatch.delenv("ADB_DEVICE_SERIAL", raising=False) + + controller = adb_server._get_controller(target_platform="ios") + assert captured["ctx"].device.mobile_platform == DevicePlatform.IOS + assert captured["ctx"].device.device_id == "booted" + assert adb_server._CONTROLLERS == {"ios:booted": controller} + + again = adb_server._get_controller(target_platform="ios") + assert again is controller + + # iOS lookups never populate the Android global slot or un-namespaced keys. + assert adb_server._GLOBAL_CONTROLLER is None + assert all(key.startswith("ios:") for key in adb_server._CONTROLLERS) + + +def test_lazy_mcp_ios_controller_never_reads_adb_device_serial(monkeypatch): + """ADB_DEVICE_SERIAL is Android-only: it must not seed an iOS target.""" + from types import SimpleNamespace + + from artemis.context import DevicePlatform + from artemis.mcp import adb_server + + captured = {} + + def fake_controller(ctx): + captured["ctx"] = ctx + return SimpleNamespace(ctx=ctx) + + monkeypatch.setattr(adb_server, "_CONTROLLERS", {}) + monkeypatch.setattr(adb_server, "_GLOBAL_CONTROLLER", None) + monkeypatch.setattr(adb_server, "UnifiedMobileController", fake_controller) + monkeypatch.delenv("ARTEMIS_DEVICE_ID", raising=False) + monkeypatch.setenv("ADB_DEVICE_SERIAL", "android-only") + + controller = adb_server._get_controller(target_platform="ios") + assert captured["ctx"].device.mobile_platform == DevicePlatform.IOS + assert captured["ctx"].device.device_id == "booted" + assert set(adb_server._CONTROLLERS) == {"ios:booted"} + assert adb_server._GLOBAL_CONTROLLER is None + + +# --------------------------------------------------------------------------- # +# iOS pool: "booted" selector and name ambiguity +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_pins_the_unique_booted_sim(monkeypatch): + pool = _pool_with_devices(monkeypatch) + assert await pool.select_device_async(preferred_serial="booted") == "AAAA-1111" + assert await pool.validate_explicit_serial_async("booted") is None + assert pool.validate_explicit_serial("booted") is None + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_tolerates_case_and_whitespace(monkeypatch): + """' Booted ' / 'BOOTED' normalize to the reserved selector, not a name.""" + pool = _pool_with_devices(monkeypatch) + assert await pool.select_device_async(preferred_serial=" Booted ") == "AAAA-1111" + assert await pool.validate_explicit_serial_async(" BOOTED ") is None + assert pool.validate_explicit_serial("Booted") is None + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_fails_open_on_ambiguity(monkeypatch): + """Zero or multiple booted sims keep the literal so the driver's guidance fires.""" + two_booted = [dict(SIM_LIST[0]), {**SIM_LIST[1], "state": "Booted"}] + pool = _pool_with_devices(monkeypatch, two_booted) + assert await pool.select_device_async(preferred_serial="booted") == "booted" + rejection = await pool.validate_explicit_serial_async("booted") + assert rejection is not None and "exactly one booted" in rejection + assert "UDID" in rejection + + no_booted = [{**SIM_LIST[0], "state": "Shutdown"}, dict(SIM_LIST[1])] + pool = _pool_with_devices(monkeypatch, no_booted) + assert await pool.select_device_async(preferred_serial="booted") == "booted" + rejection = await pool.validate_explicit_serial_async("booted") + assert rejection is not None and "0 found" in rejection + + +@pytest.mark.asyncio +async def test_ios_pool_booted_selector_fails_open_on_enumeration_error(monkeypatch): + pool = IosDevicePool() + import importlib + + module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr(module, "list_ios_simulators", AsyncMock(return_value=None)) + monkeypatch.setattr(module, "list_core_devices", AsyncMock(return_value=None)) + assert await pool.select_device_async(preferred_serial="booted") == "booted" + assert await pool.validate_explicit_serial_async("booted") is None + + +def test_ios_pool_rejects_duplicate_device_names(monkeypatch): + """Two usable devices sharing a name must be rejected, not first-matched.""" + twins = [ + {**SIM_LIST[0], "name": "Office iPhone"}, + {**SIM_LIST[1], "name": "Office iPhone", "state": "Booted"}, + ] + pool = _pool_with_devices(monkeypatch, twins) + rejection = pool.validate_explicit_serial("Office iPhone") + assert rejection is not None and "UDID" in rejection + # UDID pins resolve fine. + assert pool.validate_explicit_serial("AAAA-1111") is None + + +# --------------------------------------------------------------------------- # +# Admin capabilities endpoint (thin-SDK iOS wire contract) +# --------------------------------------------------------------------------- # + + +@pytest.mark.asyncio +async def test_admin_capabilities_endpoint_is_static_and_advertises_ios(monkeypatch): + """GET /api/v1/capabilities advertises 'platform.ios' without probing.""" + import importlib + + tasks_router = importlib.import_module("apps.admin_console.routers.tasks") + pool_module = importlib.import_module("artemis.runtime.ios_device_pool") + monkeypatch.setattr( + pool_module, + "list_core_devices", + AsyncMock(side_effect=AssertionError("device discovery must not run")), + ) + monkeypatch.setattr( + pool_module, + "list_ios_simulators", + AsyncMock(side_effect=AssertionError("device discovery must not run")), + ) + + response = await tasks_router.get_capabilities() + + assert response["api_version"] == "1" + assert "platform.ios" in response["features"] + assert { + "tasks.submit", + "tasks.get", + "tasks.stop", + "devices.list", + "system.readiness", + }.issubset(response["features"]) diff --git a/tests/unit/test_jsonc_comments.py b/tests/unit/test_jsonc_comments.py new file mode 100644 index 00000000..7b02a709 --- /dev/null +++ b/tests/unit/test_jsonc_comments.py @@ -0,0 +1,86 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for ``strip_json_comments``. + +The previous regex implementation treated ``//`` inside string literals as a +comment start, corrupting URL values like ``"api_base": "http://127.0.0.1:8080"``. +These tests pin comment removal to positions outside string literals. +""" + +import io +import json + +from third_party.mobile_use.utils.file import load_jsonc, strip_json_comments + + +def test_line_comment_removed_outside_string(): + text = '{\n // a comment\n "a": 1\n}' + assert json.loads(strip_json_comments(text)) == {"a": 1} + + +def test_url_scheme_not_treated_as_comment(): + text = '{"api_base": "http://127.0.0.1:8080/v1"}' + assert json.loads(strip_json_comments(text)) == {"api_base": "http://127.0.0.1:8080/v1"} + + +def test_https_and_trailing_comment_markers_in_string(): + text = '{"a": "https://x//y", "b": "keep // this"}' + assert json.loads(strip_json_comments(text)) == { + "a": "https://x//y", + "b": "keep // this", + } + + +def test_block_comment_removed_outside_string(): + text = '{\n /* multi\n line */\n "a": 1\n}' + assert json.loads(strip_json_comments(text)) == {"a": 1} + + +def test_block_comment_markers_inside_string_preserved(): + text = '{"a": "not a /* comment */", "b": 2}' + assert json.loads(strip_json_comments(text)) == { + "a": "not a /* comment */", + "b": 2, + } + + +def test_escaped_quote_does_not_end_string(): + # \" inside the string must not toggle string state, so the real "//" + # comment after the closing quote is still removed. + text = '{"a": "he said \\"http://x\\""} // real comment\n' + assert json.loads(strip_json_comments(text)) == {"a": 'he said "http://x"'} + + +def test_escaped_backslash_before_quote(): + # After \\ the quote is not escaped, so the string ends there and the + # tail is a comment. + text = '{"a": "c:\\\\"} // tail\n' + assert json.loads(strip_json_comments(text)) == {"a": "c:\\"} + + +def test_comment_only_line_leaves_newline_for_stable_line_numbers(): + text = '{\n// comment\n"a": 1\n}' + stripped = strip_json_comments(text) + assert stripped.count("\n") == 3 + + +def test_unterminated_comment_consumes_to_eof(): + text = '{"a": 1} // never ends' + assert json.loads(strip_json_comments(text)) == {"a": 1} + + +def test_load_jsonc_roundtrip(tmp_path): + cfg = '{"llm": {"api_base": "http://127.0.0.1:8080/v1"}} // env' + assert load_jsonc(io.StringIO(cfg)) == {"llm": {"api_base": "http://127.0.0.1:8080/v1"}} diff --git a/tests/unit/test_unified_controller_video.py b/tests/unit/test_unified_controller_video.py index e87330dd..ad1eeb38 100644 --- a/tests/unit/test_unified_controller_video.py +++ b/tests/unit/test_unified_controller_video.py @@ -15,9 +15,11 @@ """Unit tests for UnifiedMobileController video recording and playback features.""" import asyncio +import collections import subprocess import time from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 @@ -794,3 +796,256 @@ def test_segment_session_offsets_without_data_engine_anchor(tmp_path): assert offsets[tmp_path / "recording_001.mp4"] == 7.5 # An emergency fallback remux has no record; it starts at the recording anchor. assert offsets[fallback] == 0.0 + + +# --- iOS native recording through the shared controller contract --- + + +class _FakeIosStderr: + def __init__(self, lines=None, eof: bool = False): + self._lines = collections.deque(lines if lines is not None else [b"Recording started\n"]) + self._eof = asyncio.Event() + if eof: + self._eof.set() + + async def readline(self): + if self._lines: + return self._lines.popleft() + await self._eof.wait() + return b"" + + +class _FakeIosRecorderProcess: + def __init__(self, stderr=None): + self.stderr = stderr or _FakeIosStderr() + self.returncode = None + self.signals = [] + self._exit = asyncio.Event() + + def send_signal(self, sig): + self.signals.append(sig) + self.returncode = 0 + self._exit.set() + + def terminate(self): + self.returncode = -15 + self._exit.set() + + def kill(self): + self.returncode = -9 + self._exit.set() + + async def wait(self): + await self._exit.wait() + return self.returncode + + +@pytest.fixture +def ios_ctx(): + from artemis.drivers.ios.xcode_driver import XcodeSimulatorDriver + + ctx = MagicMock(spec=ArtemisContext) + ctx.device = MagicMock() + ctx.device.device_id = "sim-udid-1" + ctx.device.mobile_platform = "ios" + ctx.data_engine = MagicMock() + ctx.data_engine.current_session_id = uuid4() + ctx.data_engine.session_start_time = time.time() - 1.0 + ctx.data_engine.storage = MagicMock() + driver = XcodeSimulatorDriver(device_id="sim-udid-1") + driver._session_key = "test-session" + driver._bridge = SimpleNamespace( + connected=True, call=AsyncMock(), close=AsyncMock(), start=AsyncMock() + ) + ctx._active_driver = driver + return ctx + + +@pytest.fixture +def ios_recording_seams(monkeypatch): + """Fake the simctl child process, display probe, and MP4 finalization.""" + from artemis.drivers.ios import recording as rec + from artemis.utils import video as video_utils + + procs: list[_FakeIosRecorderProcess] = [] + + async def spawn(*argv, **kwargs): + proc = _FakeIosRecorderProcess() + procs.append(proc) + return proc + + async def fake_finalize(source_path, output_path, width, height, span): + Path(output_path).write_bytes(b"fake mp4") + return True + + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + monkeypatch.setattr(rec, "probe_display_dimensions", AsyncMock(return_value=(1206, 2622))) + monkeypatch.setattr(rec, "finalize_mov_to_mp4", fake_finalize) + monkeypatch.setattr( + video_utils, + "probe_video_segment", + AsyncMock(return_value={"duration": 1.0, "width": 100, "height": 200}), + ) + monkeypatch.setattr(rec, "WATCHDOG_INTERVAL_SECONDS", 0.01) + return procs + + +@pytest.mark.asyncio +async def test_ios_start_registers_driver_session_and_publishes_de_anchor( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + + res = await controller.start_video_recording(output_dir=tmp_path) + + try: + assert res.success is True + session = get_active_session("sim-udid-1") + assert session is not None + # Controller and driver share one recording session/process. + assert session is ios_ctx._active_driver.recording_session + assert session.data_engine_start_time == ios_ctx.data_engine.session_start_time + ios_ctx.data_engine.record_video_start.assert_called_once() + assert res.video_id == session.video_id + + duplicate = await controller.start_video_recording(output_dir=tmp_path) + assert duplicate.success is False + finally: + remove_active_session("sim-udid-1") + await ios_ctx._active_driver.stop_video_recording() + + +@pytest.mark.asyncio +async def test_ios_start_failure_publishes_start_and_failure( + ios_ctx, ios_recording_seams, tmp_path, monkeypatch +): + from artemis.drivers.ios import recording as rec + + async def dead_spawn(*argv, **kwargs): + return _FakeIosRecorderProcess(_FakeIosStderr(lines=[], eof=True)) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", dead_spawn) + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + + res = await controller.start_video_recording(output_dir=tmp_path) + + assert res.success is False + assert "failed to start" in res.message + ios_ctx.data_engine.record_video_start.assert_called_once() + ios_ctx.data_engine.record_video_failure.assert_called_once() + assert get_active_session("sim-udid-1") is None + # The driver's failed session is still inspectable. + assert ios_ctx._active_driver.recording_session is not None + + +@pytest.mark.asyncio +async def test_ios_stop_records_stop_metadata_and_clears_registry( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + + res = await controller.stop_video_recording() + + assert res.success is True + assert res.video_path == tmp_path / "recording.mp4" + assert res.source_revision and res.source_revision.endswith(":ready") + ios_ctx.data_engine.record_video_stop.assert_called_once() + assert get_active_session("sim-udid-1") is None + + +@pytest.mark.asyncio +async def test_ios_stop_failure_reports_unsuccessful_result( + ios_ctx, ios_recording_seams, tmp_path, monkeypatch +): + from artemis.drivers.ios import recording as rec + + monkeypatch.setattr(rec, "finalize_mov_to_mp4", AsyncMock(return_value=False)) + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + + res = await controller.stop_video_recording() + + assert res.success is False + ios_ctx.data_engine.record_video_failure.assert_called() + assert get_active_session("sim-udid-1") is None + # The failed session is retained with actionable errors for reporting. + retained = ios_ctx._active_driver.recording_session + assert retained is not None and retained.errors + + +@pytest.mark.asyncio +async def test_ios_extract_seals_then_renders_only_final_mp4( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + await asyncio.sleep(0.05) + session = get_active_session("sim-udid-1") + seal_spy = AsyncMock(wraps=ios_ctx._active_driver.seal_recording_segment) + ios_ctx._active_driver.seal_recording_segment = seal_spy + + rendered = {} + + async def fake_render(segments, start, end, output_path, **kwargs): + rendered["segments"] = segments + rendered["range"] = (start, end) + Path(output_path).write_bytes(b"clip") + return True + + output_path = tmp_path / "clip.mp4" + shift = session.start_time - session.data_engine_start_time + with patch( + "artemis.controllers.unified_controller.render_timeline_clip", + AsyncMock(side_effect=fake_render), + ) as render: + res = await controller.extract_segment_metadata( + start_time=shift + 0.0, end_time=shift + 0.02, output_path=output_path + ) + assert res.success is True + assert res.video_path == output_path + seal_spy.assert_awaited_once() + # Only finalized MP4 segment paths are rendered — never the open .mov. + assert rendered["segments"] + assert all(s["path"].endswith(".mp4") for s in rendered["segments"]) + assert all(".mov" not in s["path"] for s in rendered["segments"]) + + # Generation-scoped cache: same range reuses the clip without render. + res2 = await controller.extract_segment_metadata( + start_time=shift + 0.0, end_time=shift + 0.02, output_path=output_path + ) + assert res2.success is True + assert render.await_count == 1 + + await controller._driver.stop_video_recording() + remove_active_session("sim-udid-1") + + +@pytest.mark.asyncio +async def test_ios_extract_rejects_range_outside_sealed_capture( + ios_ctx, ios_recording_seams, tmp_path +): + remove_active_session("sim-udid-1") + controller = UnifiedMobileController(ios_ctx) + await controller.start_video_recording(output_dir=tmp_path) + session = get_active_session("sim-udid-1") + shift = session.start_time - session.data_engine_start_time + + with patch( + "artemis.controllers.unified_controller.render_timeline_clip", + AsyncMock(return_value=True), + ) as render: + res = await controller.extract_segment_metadata( + start_time=shift + 50.0, end_time=shift + 55.0 + ) + + assert res.success is False + assert "overlap" in res.message + render.assert_not_awaited() + await controller._driver.stop_video_recording() + remove_active_session("sim-udid-1") diff --git a/tests/unit/tools/test_explorer_tool.py b/tests/unit/tools/test_explorer_tool.py index 5e363b3f..fa7ed6ca 100644 --- a/tests/unit/tools/test_explorer_tool.py +++ b/tests/unit/tools/test_explorer_tool.py @@ -355,7 +355,7 @@ def fake_draw_dots(screenshot_path, points, labels, output_path, **kwargs): assert "target=3, a bare integer" in result[0]["text"] assert "target=[x, y]" not in result[0]["text"] assert result[1]["type"] == "image_url" - assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert result[1]["image_url"]["url"].startswith("data:image/png;base64,") def test_render_operator_blocks_numbers_annotations_sequentially(tmp_path): diff --git a/tests/unit/utils/test_image_mime.py b/tests/unit/utils/test_image_mime.py new file mode 100644 index 00000000..0fa74693 --- /dev/null +++ b/tests/unit/utils/test_image_mime.py @@ -0,0 +1,60 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for screenshot media-type detection and data-URI construction.""" + +import base64 + +from artemis.utils.image_mime import image_data_uri, image_mime_type + +PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 20 +JPEG_BYTES = b"\xff\xd8\xff\xe0" + b"\x00" * 20 + + +def test_png_bytes_detected(): + assert image_mime_type(PNG_BYTES) == "image/png" + + +def test_jpeg_bytes_detected(): + assert image_mime_type(JPEG_BYTES) == "image/jpeg" + + +def test_unknown_bytes_default_to_png(): + # Screenshot paths emit PNG; an unrecognized header keeps the honest + # project default rather than the historical hardcoded jpeg. + assert image_mime_type(b"\x00\x01\x02\x03") == "image/png" + + +def test_base64_string_detected(): + b64 = base64.b64encode(JPEG_BYTES).decode() + assert image_mime_type(b64) == "image/jpeg" + b64 = base64.b64encode(PNG_BYTES).decode() + assert image_mime_type(b64) == "image/png" + + +def test_data_uri_uses_detected_type(): + uri = image_data_uri(PNG_BYTES) + b64 = base64.b64encode(PNG_BYTES).decode() + assert uri == f"data:image/png;base64,{b64}" + + +def test_data_uri_for_jpeg_bytes(): + uri = image_data_uri(JPEG_BYTES) + b64 = base64.b64encode(JPEG_BYTES).decode() + assert uri == f"data:image/jpeg;base64,{b64}" + + +def test_data_uri_from_base64_input(): + b64 = base64.b64encode(PNG_BYTES).decode() + assert image_data_uri(b64) == f"data:image/png;base64,{b64}" diff --git a/tests/unit/utils/test_video_manifest.py b/tests/unit/utils/test_video_manifest.py index f0e0009f..ca75365f 100644 --- a/tests/unit/utils/test_video_manifest.py +++ b/tests/unit/utils/test_video_manifest.py @@ -130,3 +130,97 @@ async def test_manifest_returns_none_without_valid_segments(tmp_path): ): assert await write_recording_manifest(tmp_path, [tmp_path / "nope.mp4"]) is None assert not (tmp_path / "recording.json").exists() + + +@pytest.mark.asyncio +async def test_manifest_forwards_probe_timeout_only_when_supplied(tmp_path): + """iOS callers bound the probe; default callers keep the old signature.""" + import artemis.utils.video as video + + seg = tmp_path / "recording.mp4" + seg.write_bytes(b"mp4") + seen: list = [] + + async def probe(path, timeout_seconds=None): + seen.append(timeout_seconds) + return {"duration": 1.0, "width": 1080, "height": 1920} + + with patch("artemis.utils.video.probe_video_segment", AsyncMock(side_effect=probe)): + await write_recording_manifest(tmp_path, [seg], probe_timeout_seconds=30.0) + assert seen == [30.0] + + +@pytest.mark.asyncio +async def test_probe_timeout_kills_and_drains_the_ffprobe_child(tmp_path, monkeypatch): + """A bounded probe must reap its owned ffprobe child on timeout.""" + import asyncio + + import artemis.utils.video as video + + class _Proc: + def __init__(self): + self.returncode = None + self.killed = False + self.drained = False + + def kill(self): + self.killed = True + self.returncode = -9 + + async def communicate(self): + if self.killed: + self.drained = True + return b"", b"" + await asyncio.Event().wait() # never answers in time + return b"", b"" + + proc = _Proc() + + async def fake_exec(*args, **kwargs): + return proc + + monkeypatch.setattr(video.asyncio, "create_subprocess_exec", fake_exec) + seg = tmp_path / "seg.mp4" + seg.write_bytes(b"mp4") + with pytest.raises(TimeoutError): + await video.probe_video_segment(seg, timeout_seconds=0.05) + assert proc.killed and proc.drained + + +@pytest.mark.asyncio +async def test_probe_cancellation_reaps_the_child_before_propagating(tmp_path, monkeypatch): + import asyncio + + import artemis.utils.video as video + + class _Proc: + def __init__(self): + self.returncode = None + self.killed = False + self.drained = False + + def kill(self): + self.killed = True + self.returncode = -9 + + async def communicate(self): + if self.killed: + self.drained = True + return b"", b"" + await asyncio.Event().wait() + return b"", b"" + + proc = _Proc() + + async def fake_exec(*args, **kwargs): + return proc + + monkeypatch.setattr(video.asyncio, "create_subprocess_exec", fake_exec) + seg = tmp_path / "seg.mp4" + seg.write_bytes(b"mp4") + task = asyncio.create_task(video.probe_video_segment(seg, timeout_seconds=30.0)) + await asyncio.sleep(0.02) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert proc.killed and proc.drained diff --git a/third_party/mobile_use/config/llm.py b/third_party/mobile_use/config/llm.py index be9f509a..f93c5bc8 100644 --- a/third_party/mobile_use/config/llm.py +++ b/third_party/mobile_use/config/llm.py @@ -63,6 +63,10 @@ class LLM(BaseModel): reasoning_effort: Literal["none", "low", "medium", "high"] | None = None include_thoughts: bool | None = None enable_grounding: bool | None = None + api_base: str | None = None + api_key: str | None = None + max_tokens: int | None = None + timeout: float | None = None def validate_provider(self, name: str) -> None: """Ensure the required API key or credentials exist in settings for this provider.""" diff --git a/third_party/mobile_use/context.py b/third_party/mobile_use/context.py index 89533dc7..a9405138 100644 --- a/third_party/mobile_use/context.py +++ b/third_party/mobile_use/context.py @@ -50,6 +50,7 @@ class DevicePlatform(StrEnum): """Mobile device platform enumeration.""" ANDROID = "android" + IOS = "ios" class DeviceContext(BaseModel): diff --git a/third_party/mobile_use/sdk/agent.py b/third_party/mobile_use/sdk/agent.py index c6a81203..57b7e568 100644 --- a/third_party/mobile_use/sdk/agent.py +++ b/third_party/mobile_use/sdk/agent.py @@ -66,6 +66,7 @@ from artemis.graph.graph import get_graph from artemis.graph.state import State from artemis.runtime import DeviceExecutionLock +from artemis.runtime.adb_endpoint import IOS_LOCK_SCOPE from artemis.sdk.run_outcome import attach_test_summary, resolve_trace_suffix from artemis.sdk.types.agent import AgentConfig from artemis.utils.startup_progress import publish_startup_progress @@ -123,7 +124,10 @@ async def init( target_dev = device_serial or device_id if target_dev: self._config = self._config.model_copy( - update={"device_id": target_dev, "device_platform": DevicePlatform.ANDROID} + update={ + "device_id": target_dev, + "device_platform": self._config.device_platform or DevicePlatform.ANDROID, + } ) return await self._init_internal( @@ -440,6 +444,8 @@ async def _run_task( llm_config=agent_profile.llm_config, agent_config=self._config, ) + if context.device.mobile_platform == DevicePlatform.IOS: + context._active_driver = getattr(self, "_ios_driver", None) output_config = None if request.output_description or request.output_format: @@ -467,7 +473,14 @@ async def _execute_task_logic(): or getattr(task, "id", None) or getattr(getattr(task, "request", None), "task_name", None) ) - active_owner = DeviceExecutionLock.get_active_owner(self._device_context.device_id) + # iOS simulators lock under the "ios" scope so a UDID can never + # collide with an Android serial carrying the same text. + lock_scope = ( + IOS_LOCK_SCOPE if context.device.mobile_platform == DevicePlatform.IOS else None + ) + active_owner = DeviceExecutionLock.get_active_owner( + self._device_context.device_id, lock_scope=lock_scope + ) already_held = ( active_owner is not None and active_owner.pid == os.getpid() @@ -486,6 +499,7 @@ async def _execute_task_logic(): max_concurrency=effective_max, session_id=str(sess_id) if sess_id else None, ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", + lock_scope=lock_scope, ) ) try: @@ -516,12 +530,15 @@ async def _execute_task_logic(): # DataEngine uses a shared database, so a queued task must not # publish a new active session while the current task is still # finishing. + if context.device.mobile_platform == DevicePlatform.IOS: + await self._ensure_device_unlocked() self._prepare_tracing(task=task, context=context) self._prepare_output_files(task=task) if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": if self._ui_adb_client is not None: await self._connect_screen_client(context, str(sess_id)) - await self._ensure_device_unlocked() + if context.device.mobile_platform != DevicePlatform.IOS: + await self._ensure_device_unlocked() publish_startup_progress( "environment", "Preparing the device environment", session_id=str(sess_id) ) @@ -749,6 +766,10 @@ async def _safe_stop_recording(): raise finally: + # The native simulator transport is task-owned even when setup + # fails before entering the context manager. + if context.device.mobile_platform == DevicePlatform.IOS: + await context.disconnect_driver() try: # Background ADB processes (logcat, screenrecord, ...) started # by the Operator must not outlive the automation task. diff --git a/third_party/mobile_use/utils/file.py b/third_party/mobile_use/utils/file.py index e7930014..055adc10 100644 --- a/third_party/mobile_use/utils/file.py +++ b/third_party/mobile_use/utils/file.py @@ -17,14 +17,51 @@ # See third_party/mobile_use/METADATA for the upstream source and local modifications. import json -import re from typing import IO def strip_json_comments(text: str) -> str: - text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return text + """Remove ``//`` and ``/* */`` comments while preserving string literals. + + The naive regex version stripped ``//`` sequences inside quoted strings, + which made URL values (e.g. ``"api_base": "http://127.0.0.1:8080/v1"``) + unparseable. This scan tracks string/escape state and only treats the + comment markers that appear outside strings as comments. + """ + out: list[str] = [] + i, n = 0, len(text) + in_string = False + while i < n: + char = text[i] + if in_string: + out.append(char) + if char == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + if char == '"': + in_string = False + i += 1 + continue + if char == '"': + in_string = True + out.append(char) + i += 1 + continue + if char == "/" and i + 1 < n: + if text[i + 1] == "/": + while i < n and text[i] != "\n": + i += 1 + continue + if text[i + 1] == "*": + i += 2 + while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + i += 2 + continue + out.append(char) + i += 1 + return "".join(out) def load_jsonc(file: IO) -> dict: