diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 49d3f28dd..5ea55bd29 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,6 +56,7 @@ updates: - "/src/agent_env/agents/agents/pydantic-ai" - "/src/agent_env/examples/mail-mcp/envs/mail" - "/src/agent_env/computers/artifacts/ubuntu-24.04" + - "/src/agent_env/computers/artifacts/ubuntu-24.04-headless" - "/src/agent_env/computers/disk/linux/boot" - "/src/agent_env/computers/disk/linux/tools" - "/src/agent_env/computers/disk/linux/launcher" diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 15d6e69a1..fd0b10d57 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -56,6 +56,7 @@ "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile", "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile", "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile", + "agentenv-ubuntu-24.04-headless": "src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile", "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile", "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", @@ -118,6 +119,12 @@ " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], + "agentenv-ubuntu-24.04-headless": [ + "docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "start-computer & for _ in $(seq 60); do sleep 1;" + " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" + " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" + " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], "agentenv-disk-tools": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", "command -v disk image-env mkfs.ext4 mkfs.vfat mcopy sfdisk qemu-img"], "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index b6cc8c649..33b09b2fe 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -505,7 +505,8 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope - **The input queue.** Input calls run one at a time in arrival order; one that waits past `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. - **The computer skill.** `computer/v1` declares the seven tools, its params the driver's - features. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base + features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website + browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base class): a SKILL.md written from the computer as it booted (its platform, session user and home, the live screen size, what its input takes, its OS's shortcuts) and ending with the driver class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of @@ -513,7 +514,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope computer skill leads with it, the computer's part following. - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a capture, the driver's own `ready_checks()`, and one per app on the computer, asked of the app's - own server (see Computer apps); tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool + own server (see Computer apps); tools refuse `device_unavailable` until they pass once; a check + that can't pass again is `final`, and while one fails the status carries `failed`, saying why, + which `boot.ready()` raises at once rather than waiting out its timeout), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and, with `recording`, `computer-recording/v1` (below). @@ -742,7 +745,7 @@ its card lists each one's card as a child environment, rooted at `/svc//`, requests to the app's server. `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` -env stores, and `python -m agentenv_protocol.computer._declare ` prints it as +env stores (`browser: true` for a website browser), and `python -m agentenv_protocol.computer._declare ` prints it as JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed. ### Website browsers @@ -769,8 +772,8 @@ are the profile's template. The base class adds: `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off. Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`, `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go - through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`, - except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy. + through. Alone, on a computer with no display (`XDG_SESSION_TYPE=none`, a browser computer's), it + also takes `--window-size` and `--headless=new`, and refuses to start with no proxy. `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, or any other way to open a debugging port. - **Before each launch** it empties its cache, so no launch inherits another's, and writes the deploy's @@ -803,7 +806,10 @@ are the profile's template. The base class adds: launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made and each exit it didn't, with its exit `code` or the `signal` that ended it, since the computer booted. Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra - failure, its ready check false. + failure, its ready check false. `page/viewport`, `page/capture` and `page/input`, framework-only, are + its active tab's page as a page driver sees it: its size, which headless is its window's; a PNG of it, + or `blank` for a tab that paints nothing within 3 s (1 s with no URL yet); and CDP input calls + (`Input.dispatchMouseEvent`, `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in `~/Downloads`, folders it makes. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 22877b7b2..5cdef1cb0 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -236,7 +236,7 @@ class AppDeclaration(BaseModel): """What a ComputerAppEnvironment class declares, as a ``computer_app`` env stores it: the platforms it ``supports``, as a driver names them; ``os``, what it declares for each OS type; ``settings`` and ``build_settings``, the JSON schemas of its ``Settings`` and of its ``BuildSettings`` when it declares any; its ``skill``; and the ``data`` - methods, ``install_steps`` and ``ready_checks`` it serves, by name.""" + methods, ``install_steps`` and ``ready_checks`` it serves, by name; and ``browser``, true for a website browser.""" model_config = ConfigDict(extra="forbid") @@ -249,6 +249,7 @@ class AppDeclaration(BaseModel): data: list[str] install_steps: list[InstallStep] ready_checks: list[AppReadyCheck] + browser: Optional[bool] = None @model_validator(mode="before") @classmethod diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index 9de569ed5..3f638d986 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -8,6 +8,7 @@ import asyncio import copy +import json from pathlib import Path from typing import Optional @@ -16,7 +17,7 @@ from .. import client from ..types import RPC_PATH, WELL_KNOWN_PATH, EnvironmentCard from .app import COMPUTER_APP, FAILURE, SOCKET -from .app_runtime import CHECK_TIMEOUT_S, AppReadiness +from .app_runtime import CHECK_TIMEOUT_S, LAUNCHES, AppReadiness from .operations import ReadyCheck #: Where a composed computer installs its apps, a folder each. @@ -54,7 +55,7 @@ async def check(folder: Path) -> ReadyCheck: async def children(folders: list[Path]) -> list[EnvironmentCard]: """Each app in ``folders`` as a child environment of its computer: its server's card, rooted at ``/svc//``, or, while its server isn't answering, its name and route alone.""" - cards = await asyncio.gather(*(_card(folder) for folder in folders)) + cards = await asyncio.gather(*(served_card(folder) for folder in folders)) return [EnvironmentCard.model_validate(child_card(folder.name, card)) if card is not None else EnvironmentCard(name=folder.name, url=f"/svc/{folder.name}{RPC_PATH}") for folder, card in zip(folders, cards)] @@ -85,7 +86,7 @@ def rooted(path: object) -> object: async def owned_paths(folders: list[Path]) -> list[str]: """The paths under the session user's home that the apps in ``folders`` keep their data in, as their cards say.""" - cards = await asyncio.gather(*(_card(folder) for folder in folders)) + cards = await asyncio.gather(*(served_card(folder) for folder in folders)) return [path for card in cards if card is not None for path in client.extension_params(card, COMPUTER_APP).get("paths") or []] @@ -104,7 +105,7 @@ def endpoint(card: dict, method: str) -> str: return client.find_extension_method(card, COMPUTER_APP, method)["endpoint"] -async def _card(folder: Path) -> Optional[dict]: +async def served_card(folder: Path) -> Optional[dict]: """The card the app in ``folder`` serves, or None while its server isn't answering.""" try: async with _client(folder / SOCKET) as http: @@ -113,6 +114,33 @@ async def _card(folder: Path) -> Optional[dict]: return None +def ended_on_its_own(folder: Path) -> Optional[str]: + """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says it + ended on its own, an infrastructure failure; None while it hasn't.""" + for entry in launch_log(folder): + if entry.get("event") == "exit": + how = (f"by signal {entry['signal']}" if "signal" in entry else + f"with code {entry['code']}" if "code" in entry else "in a way that wasn't recorded") + return (f"it ended on its own at {entry.get('at')}, {how}, and nothing launches it again: an " + "infrastructure failure") + return None + + +def launch_log(folder: Path) -> list[dict]: + """Each entry of the launch log of the app in ``folder`` that's whole, in order.""" + try: + lines = (folder / LAUNCHES).read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + return [] + entries = [] + for line in lines: + try: + entries.append(json.loads(line)) + except ValueError: + continue + return entries + + def _unanswered(folder: Path) -> str: try: return f"its server failed to start: {(folder / FAILURE).read_text()}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py index a8453b7e2..20e46fd21 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -89,7 +89,7 @@ def computer_features(driver: ComputerDriverEnvironment) -> dict: "max_wait_ms": driver.max_wait_ms, "max_drag_points": driver.max_drag_points, "max_duration_ms": driver.max_duration_ms, "max_key_repeat": driver.max_key_repeat, "max_scroll_amount": driver.max_scroll_amount, "delivery": "foreground", "platform": driver.platform.as_dict(), - "supports": list(driver.supports), "default_data_paths": list(DEFAULT_DATA_PATHS), + "supports": list(driver.supports), "default_data_paths": list(DEFAULT_DATA_PATHS), "screen": driver.screen_kind, } diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py index 6d0736eb6..a36266a4c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -37,6 +37,8 @@ def check_features(cls: type) -> None: problems.append("keys are W3C key values in their W3C case") except AgentEnvError as e: problems.append(f"keys: {e.message}") + if cls.screen_kind not in ("desktop", "page"): + problems.append(f"screen_kind is desktop or page, got {cls.screen_kind!r}") if ("set_display" in declared(cls)) != bool(cls.display_modes): problems.append("display_modes and @set_display come together") problems += check_recording(cls.recording) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 6177a4aac..a5b8f90b9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -4,7 +4,7 @@ from __future__ import annotations import asyncio -from typing import Any, Callable, ClassVar +from typing import Any, Callable, ClassVar, Literal import httpx from pydantic import BaseModel, ConfigDict @@ -79,6 +79,9 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen recording: ClassVar[Recording | None] = Recording() #: What an agent should know about this driver's computer beyond what its card says, as markdown: the end of its skill. notes: ClassVar[str] = "" + #: What the computer's screen shows, its card's ``screen``: a ``desktop``, or a website browser's ``page``, its + #: active tab. + screen_kind: ClassVar[Literal["desktop", "page"]] = "desktop" tool_access = True #: What the base class keeps for the driver, built by ``__init__``. _computer: ComputerRuntime | None = None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index ff12c3b36..a569d094f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -45,8 +45,12 @@ async def _computer_skill(self) -> tuple[str, str, dict[str, str]]: description="Whether the computer is usable now, by real checks; never sends input.") async def _computer_ready_status(self) -> dict: checks = await self._computer.ready_checks() - return {"ready": all(c.ok for c in checks), - "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]} + status = {"ready": all(c.ok for c in checks), + "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail, **({"final": True} if c.final else {})} + for c in checks]} + if failed := [c for c in checks if c.final and not c.ok]: + status["failed"] = "; ".join(f"{c.name}: {c.detail}" for c in failed) + return status @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.") async def _computer_display_get(self) -> dict: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index 1e9936ec1..b6ead0141 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -74,6 +74,8 @@ class ReadyCheck: name: str ok: bool detail: str = "" + #: Failed for good: the check can't pass again, so the computer has failed. + final: bool = False @dataclass(frozen=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py index 3f2587e79..241b5dd30 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py @@ -28,14 +28,15 @@ def as_dict(self) -> dict: def detect(environ: dict | None = None, os_release: Path = Path("/etc/os-release")) -> Platform: """The running platform: the OS, ``/etc/os-release``'s ID and VERSION_ID on Linux, the machine, and the - display from ``XDG_SESSION_TYPE``, else ``DISPLAY`` (x11) or ``WAYLAND_DISPLAY`` (wayland).""" + display from ``XDG_SESSION_TYPE``, ``none`` for a computer with no display, else ``DISPLAY`` (x11) or + ``WAYLAND_DISPLAY`` (wayland).""" environ = os.environ if environ is None else environ name = _OS.get(sys.platform, sys.platform) release = _os_release(os_release) if name == "linux" else {} display = None if name == "linux": session = (environ.get("XDG_SESSION_TYPE") or "").strip().lower() - if session in ("x11", "wayland"): + if session in ("x11", "wayland", "none"): display = session elif environ.get("DISPLAY"): display = "x11" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index a5a435caf..c24cdd2a9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -2,8 +2,9 @@ its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's command, the unattended-use policies, a CDP connection over the browser's debugging pipe, the settings every browser takes, a fresh cache and the deploy's prompt answers at every launch, another window for a launch while it runs, its -skill and ``browser/v1`` on its card. Whether it runs alone, on the open web and through which site proxy come from -variables agentenv sets, never from its settings.""" +skill and ``browser/v1`` on its card. It runs alone, headless, on a computer with no display, a browser computer's, +whose screen is its page; which site proxy it goes through comes from a variable agentenv sets, never from its +settings.""" from __future__ import annotations import asyncio @@ -23,11 +24,11 @@ from .agent_env_environment import agent_skill, extension from .computer import AppDeclaration, ComputerAppEnvironment, Launch, install_step, ready_check +from .computer import app as computer_app +from .computer.errors import DeviceUnavailable, InvalidArgument logger = logging.getLogger(__name__) -ALONE_ENV = "AGENTENV_BROWSER_ALONE" -OPEN_WEB_ENV = "AGENTENV_BROWSER_OPEN_WEB" SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" BROWSER = "urn:agentenv:browser/v1" #: The flags agentenv sets, by prefix, which a subclass's command may not carry: among them every way to open a debugging @@ -51,6 +52,9 @@ AGENT = "agent" #: The targets agentenv's connection follows. PAGES = [{"type": "page"}] +#: The CDP input a computer's page driver may send to the active tab, and how long its screenshot may take. +PAGE_INPUT = frozenset({"Input.dispatchMouseEvent", "Input.dispatchKeyEvent", "Input.insertText"}) +CAPTURE_TIMEOUT_S, UNNAVIGATED_CAPTURE_TIMEOUT_S = 3.0, 1.0 #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" @@ -187,8 +191,8 @@ class BrowserControl: ``dialogs`` says to: always leaving a page that asks before it's left, so no answer of agentenv's blocks a navigation.""" - def __init__(self, devtools: DevToolsPipe, dialogs: str) -> None: - self.devtools, self.dialogs = devtools, dialogs + def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tuple[int, int]] = None) -> None: + self.devtools, self.dialogs, self.viewport = devtools, dialogs, viewport #: When this record starts: a server that restarts starts another. self.since = _now() self.tabs: dict[str, dict] = {} @@ -209,7 +213,8 @@ async def attach(self) -> None: downloads unattended: into ``~/Downloads``, a name downloaded again kept beside the first.""" self.ended(STOP) await self.devtools.call("Target.setDiscoverTargets", {"discover": True, "filter": PAGES}) - await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, + await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, + "waitForDebuggerOnStart": self.viewport is not None, "flatten": True, "filter": PAGES}) await self.devtools.call("Browser.setDownloadBehavior", {"behavior": "default", "eventsEnabled": True}) @@ -268,6 +273,36 @@ async def record(self) -> dict: "closed_tabs": list(self.closed_tabs), "downloads": list(self.downloads.values()), "dialogs": list(self.dialog_log)} + async def active_session(self) -> str: + """The session of the active tab, else of the first open one; DeviceUnavailable with no tab open.""" + active = await self.active_tab() or next(iter(self.tabs), None) + session = next((session for session, target in self._sessions.items() if target == active), None) + if session is None: + raise DeviceUnavailable("the browser has no tab open") + return session + + async def capture(self) -> Optional[str]: + """The active tab's page as it shows now, a PNG, base64; None for a tab that paints nothing within + ``CAPTURE_TIMEOUT_S``, which a desktop browser shows blank, or sooner for a tab with no URL yet, such as a new + tab whose navigation the browser blocked, which may never paint.""" + session = await self.active_session() + navigated = self.tabs.get(self._sessions[session], {}).get("url") + try: + shot = await self.devtools.call("Page.captureScreenshot", {"format": "png", "optimizeForSpeed": True}, + session_id=session, + timeout_s=CAPTURE_TIMEOUT_S if navigated else UNNAVIGATED_CAPTURE_TIMEOUT_S) + except asyncio.TimeoutError: + return None + return shot["data"] + + async def dispatch(self, events: list[dict]) -> None: + """Each of ``events``, a CDP input call, sent to the active tab in order; InvalidArgument for any other call.""" + if refused := [event.get("method") for event in events if event.get("method") not in PAGE_INPUT]: + raise InvalidArgument(f"page input takes {', '.join(sorted(PAGE_INPUT))}, not {refused}") + session = await self.active_session() + for event in events: + await self.devtools.call(event["method"], event.get("params") or {}, session_id=session) + async def _refresh(self, target: Optional[str] = None) -> None: """The URL and title of the tab ``target``, or of every open tab, as the browser has them now: it sends no event when a page's title changes.""" @@ -307,9 +342,19 @@ def _on_Target_targetDestroyed(self, params: dict, session: Optional[str]) -> No def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> None: if params["targetInfo"].get("type") == "page": self._sessions[params["sessionId"]] = params["targetInfo"]["targetId"] - self._spawn(self.devtools.call("Page.enable", session_id=params["sessionId"])) - # A page attached as it opens, a popup say, waits for this before it navigates. - self._spawn(self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=params["sessionId"])) + self._spawn(self._let_run(params["sessionId"])) + + async def _let_run(self, session: str) -> None: + """A page it attached to, enabled and, headless, held to its window's size, then told to run: a page attached as + it opens, a popup say, or any new page headless, waits for that before it runs.""" + try: + await self.devtools.call("Page.enable", session_id=session) + if self.viewport is not None: + width, height = self.viewport + await self.devtools.call("Emulation.setDeviceMetricsOverride", { + "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) + finally: + await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: self._sessions.pop(params["sessionId"], None) @@ -433,16 +478,19 @@ def __init__(self) -> None: for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) #: agentenv's control of it over that connection, and the record it keeps. - self.control = BrowserControl(self.devtools, self.settings.dialogs) + self.control = BrowserControl(self.devtools, self.settings.dialogs, + (self.settings.window.width, self.settings.window.height) if _alone() + else None) self.devtools.on_close(self._pipe_closed) @classmethod def declaration(cls) -> AppDeclaration: - """What the class declares; ValueError for a launch whose command carries a flag agentenv owns.""" + """What the class declares, a website browser; ValueError for a launch whose command carries a flag agentenv + owns.""" for declared in cls._installs.values(): if declared.launch is not None: _refuse_owned(cls.__name__, declared.launch) - return super().declaration() + return super().declaration().model_copy(update={"browser": True}) def profile_dir(self) -> Path: return Path.home() / self.profile @@ -469,7 +517,7 @@ def agentenv_flags(self) -> list[str]: "names none") if alone: flags.append(f"--window-size={settings.window.width},{settings.window.height}") - if _headless(): + if alone: flags.append("--headless=new") return flags @@ -491,7 +539,7 @@ def _on_desktop(self) -> bool: def _command_line_tabs(self) -> list[str]: """The start tabs its command line opens: headless Chromium takes one, and the rest open over its pipe.""" - return self.settings.start_urls[:1] if _headless() else self.settings.start_urls + return self.settings.start_urls[:1] if _alone() else self.settings.start_urls async def _launch_fds(self) -> dict[int, int]: """Before each launch, an empty cache and the deploy's prompt answers in its profile; and a new debugging pipe: @@ -568,10 +616,27 @@ async def devtools_answers(self) -> bool: "A website browser agentenv runs: its build, the clock its pages see and, alone, its window. record returns " "what agentenv's own connection to it saw: its open tabs, each with whether it's active, the tabs closed and " "who closed them, its downloads with their size and sha256, its JavaScript dialogs and who answered each, and " - "each launch, stop and exit of it since the computer booted, with its cause.")) + "each launch, stop and exit of it since the computer booted, with its cause. page/viewport, page/capture and " + "page/input are its active tab's page as a computer's page driver sees it: its size in pixels, which headless " + "is its window's, a PNG of it, and CDP input calls sent to it in order.")) async def browser_record(self) -> dict: return {**await self.control.record(), "launches": self._computer_app.launches()} + @extension(BROWSER, op="page/viewport", method="GET") + async def browser_page_viewport(self) -> dict: + window = self.settings.window + return {"width": window.width, "height": window.height} + + @extension(BROWSER, op="page/capture", method="GET") + async def browser_page_capture(self) -> dict: + png = await self.control.capture() + return {"png": png} if png is not None else {"blank": True} + + @extension(BROWSER, op="page/input") + async def browser_page_input(self, events: list[dict]) -> dict: + await self.control.dispatch(events) + return {} + def extension_params(self, uri: str) -> dict: if uri == BROWSER: params: dict[str, Any] = {"chromium": self.version, "clock": "real"} @@ -586,11 +651,8 @@ def browser_skill(self) -> tuple[str, str]: def _alone() -> bool: - return os.environ.get(ALONE_ENV) == "1" - - -def _open_web() -> bool: - return os.environ.get(OPEN_WEB_ENV) == "1" + """Whether it runs on a computer with no display, a browser computer's, alone and headless.""" + return computer_app.detect().display == "none" def _shm_bytes(path: str = "/dev/shm") -> int: @@ -601,11 +663,6 @@ def _shm_bytes(path: str = "/dev/shm") -> int: return stats.f_frsize * stats.f_blocks -def _headless() -> bool: - """Alone it runs headless, but headful on Xvfb for the open web, whose sites block headless browsers.""" - return _alone() and not _open_web() - - def _refuse_owned(name: str, launch: Launch) -> None: if owned := [arg for arg in launch.command if arg.startswith(OWNED_FLAGS)]: raise ValueError(f"{name}'s launch sets {', '.join(owned)}, which agentenv sets for every website browser") diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index c0196ffad..260bbe4d0 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -115,6 +115,7 @@ def test_unknown_predicates_are_refused(): ({"keys": ("enter",)}, "keys are W3C key values in their W3C case"), ({"keys": ("Return",)}, "keys: 'Return' is not a W3C key value"), ({"display_modes": ((10, 10),)}, "display_modes and @set_display come together"), + ({"screen_kind": "tablet"}, "screen_kind is desktop or page, got 'tablet'"), ]) def test_features_are_checked_when_the_class_is_defined(features, error): with pytest.raises(TypeError, match=error): @@ -169,7 +170,7 @@ async def test_the_card_declares_the_contract_the_driver_implements(): computer = extensions["urn:agentenv:computer/v1"]["params"] assert computer["buttons"] == ["left", "right", "middle"] and computer["max_click_count"] == 3 assert computer["keys"] == "all" and computer["platform"] == LINUX.as_dict() and computer["delivery"] == "foreground" - assert computer["default_data_paths"] == ["Desktop", "Documents", "Downloads"] + assert computer["default_data_paths"] == ["Desktop", "Documents", "Downloads"] and computer["screen"] == "desktop" assert "methods" not in computer and extensions["urn:agentenv:skill/v1"]["params"]["methods"]["get"]["method"] == "GET" assert extensions["urn:agentenv:computer-pointer/v1"]["params"]["buttons"] == ["left", "right", "middle"] assert set(extensions["urn:agentenv:computer-keyboard/v1"]["params"]["methods"]) == {"get", "set"} @@ -294,6 +295,21 @@ async def ready_checks(self): async with http_client(Raising()) as http: raised = (await http.get("/agentenv/ext/computer-ready/status")).json() assert raised["checks"][-1] == {"name": "driver", "ok": False, "detail": "RuntimeError: probe crashed"} + assert "failed" not in before and "failed" not in raised + + +@pytest.mark.asyncio +async def test_a_final_check_that_fails_has_the_status_say_the_computer_failed_and_why(): + class Ended(FakeDriver): + async def ready_checks(self): + return [ReadyCheck("browser", False, "it ended on its own", final=True), + ReadyCheck("session", True, "logged in", final=True)] + + async with http_client(Ended()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert status["ready"] is False and status["failed"] == "browser: it ended on its own" + assert status["checks"][-2:] == [{"name": "browser", "ok": False, "detail": "it ended on its own", "final": True}, + {"name": "session", "ok": True, "detail": "logged in", "final": True}] @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index c4ce5bbe8..eeadda2b3 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -19,10 +19,10 @@ from agentenv_protocol.computer import APP_SETTINGS_ENV, ComputerAppEnvironment, Launch, Ready, app, linux, session from agentenv_protocol import website_browser from agentenv_protocol.computer import app_runtime +from agentenv_protocol.computer.errors import DeviceUnavailable, InvalidArgument from agentenv_protocol.computer.platform import Platform -from agentenv_protocol.website_browser import (ALONE_BYPASS, ALONE_ENV, BROWSER, COMPUTER_BYPASS, OPEN_WEB_ENV, - POLICIES, SITE_PROXY_ENV, SKILL, BrowserControl, DevToolsError, - WebsiteBrowserEnvironment) +from agentenv_protocol.website_browser import (ALONE_BYPASS, BROWSER, COMPUTER_BYPASS, POLICIES, SITE_PROXY_ENV, SKILL, + BrowserControl, DevToolsError, WebsiteBrowserEnvironment) PROXY = "http://127.0.0.1:3128" OWNED = ["--user-data-dir", "--disk-cache-dir", "--remote-debugging-pipe", "--no-first-run", @@ -80,7 +80,7 @@ def browser_home(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path / "home")) (tmp_path / "home").mkdir() monkeypatch.setenv("ENVIRONMENT_NAME", "chromium") - for name in (ALONE_ENV, OPEN_WEB_ENV, SITE_PROXY_ENV, APP_SETTINGS_ENV): + for name in (SITE_PROXY_ENV, APP_SETTINGS_ENV): monkeypatch.delenv(name, raising=False) return tmp_path / "home" @@ -89,6 +89,11 @@ def _settings(monkeypatch, **values) -> None: monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"chromium": values})) +def _alone(monkeypatch) -> None: + """On a computer with no display, a browser computer's.""" + monkeypatch.setattr(app, "detect", lambda: Platform(os="linux", display="none")) + + def test_in_a_computer_its_flags_follow_its_command_then_its_start_tabs_and_its_window_is_waited_for(browser_home): launch = Browser().launch @@ -110,7 +115,7 @@ def test_in_a_computer_with_a_site_proxy_only_loopback_bypasses_it(monkeypatch): def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_waits_on_no_window(monkeypatch): - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) _settings(monkeypatch, window={"width": 1024, "height": 768}) @@ -130,7 +135,7 @@ def test_alone_its_launch_keeps_the_port_and_command_it_waits_on_and_their_timeo class Waiting(WebsiteBrowserEnvironment): pass - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) waiting = Waiting() @@ -140,18 +145,8 @@ class Waiting(WebsiteBrowserEnvironment): assert waiting._computer_app.timeout_s == 45 -def test_alone_on_the_open_web_it_runs_headful(monkeypatch): - monkeypatch.setenv(ALONE_ENV, "1") - monkeypatch.setenv(OPEN_WEB_ENV, "1") - monkeypatch.setenv(SITE_PROXY_ENV, PROXY) - - command = Browser().launch.command - - assert "--window-size=1280,800" in command and not any(flag.startswith("--headless") for flag in command) - - def test_alone_it_refuses_to_launch_without_a_site_proxy(monkeypatch): - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) with pytest.raises(ValueError, match=SITE_PROXY_ENV): Browser() @@ -215,7 +210,7 @@ def test_its_settings_give_its_start_tabs_language_and_time_zone_and_no_start_ur @pytest.mark.asyncio async def test_alone_its_server_launches_it_at_once_in_its_environment_with_no_display_wait_or_launcher( monkeypatch, tmp_path): - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) monkeypatch.delenv("DISPLAY", raising=False) monkeypatch.chdir(tmp_path) @@ -314,7 +309,7 @@ async def test_a_launch_while_it_runs_opens_a_window_over_its_pipe_and_starts_no async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_rest_over_its_pipe(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app", "https://github.com"]) browser = FakeBrowser() @@ -400,7 +395,7 @@ def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch): in_computer = Browser().extension_params(BROWSER) - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) alone = Browser().extension_params(BROWSER) @@ -503,6 +498,111 @@ async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to assert ("Page.enable", {}, "s-popup") in devtools.calls assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls + assert not any(method == "Emulation.setDeviceMetricsOverride" for method, _, _ in devtools.calls) + assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ + "waitForDebuggerOnStart"] is False + + +@pytest.mark.asyncio +async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_runs(tmp_path): + devtools = StubDevTools() + control = BrowserControl(devtools, "agent", (1024, 768)) + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0) + + sent = [method for method, _, session in devtools.calls if session == "s-popup"] + assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ + "waitForDebuggerOnStart"] is True + assert sent == ["Page.enable", "Emulation.setDeviceMetricsOverride", "Runtime.runIfWaitingForDebugger"] + assert ("Emulation.setDeviceMetricsOverride", {"width": 1024, "height": 768, "deviceScaleFactor": 1, + "mobile": False}, "s-popup") in devtools.calls + + +@pytest.mark.asyncio +async def test_a_page_it_couldnt_prepare_is_still_told_to_run_so_it_never_stays_held(): + def refused(params, session): + raise DevToolsError("Target closed") + + devtools = StubDevTools({"Page.enable": refused}) + control = BrowserControl(devtools, "agent", (1024, 768)) + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0) + + assert [method for method, _, session in devtools.calls if session == "s-popup"] == [ + "Page.enable", "Runtime.runIfWaitingForDebugger"] + + +class Painting(StubDevTools): + """A browser whose every tab has focus and paints, but those whose sessions are in ``blank``, which never answer a + capture; it records each capture's timeout.""" + + def __init__(self, blank=()): + super().__init__({"Runtime.evaluate": lambda params, session: {"result": {"value": ["visible", True]}}}) + self.blank, self.timeouts = set(blank), [] + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method != "Page.captureScreenshot": + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + self.timeouts.append(timeout_s) + if session_id in self.blank: + raise asyncio.TimeoutError + return {"data": f"png of {session_id}"} + + +async def _one_tab(devtools, url) -> BrowserControl: + control = BrowserControl(devtools, "agent") + await control.attach() + devtools.send("Target.targetCreated", _page("a", url)) + devtools.send("Target.attachedToTarget", {"sessionId": "s-a", **_page("a", url)}) + await asyncio.sleep(0) + return control + + +@pytest.mark.asyncio +@pytest.mark.parametrize("url, blank, png, timeout", [ + ("https://mail.test/", (), "png of s-a", 3.0), + ("https://mail.test/", ("s-a",), None, 3.0), + ("", ("s-a",), None, 1.0), +]) +async def test_its_capture_is_the_active_tabs_page_or_none_for_one_that_paints_nothing_sooner_with_no_url( + url, blank, png, timeout): + devtools = Painting(blank) + control = await _one_tab(devtools, url) + + assert await control.capture() == png and devtools.timeouts == [timeout] + + +@pytest.mark.asyncio +async def test_its_page_input_goes_to_the_active_tab_in_order_and_takes_only_input_calls(): + devtools = Painting() + control = await _one_tab(devtools, "https://mail.test/") + events = [{"method": "Input.dispatchMouseEvent", "params": {"type": "mousePressed", "x": 1, "y": 2}}, + {"method": "Input.insertText", "params": {"text": "hi"}}] + + await control.dispatch(events) + with pytest.raises(InvalidArgument, match="Page.navigate"): + await control.dispatch([*events, {"method": "Page.navigate", "params": {"url": "https://elsewhere.test/"}}]) + + sent = [(method, params, session) for method, params, session in devtools.calls if method.startswith("Input.")] + assert sent == [("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 1, "y": 2}, "s-a"), + ("Input.insertText", {"text": "hi"}, "s-a")] + + +@pytest.mark.asyncio +async def test_with_no_tab_open_its_capture_and_input_are_unavailable(): + control = BrowserControl(Painting(), "agent") + await control.attach() + + with pytest.raises(DeviceUnavailable, match="no tab open"): + await control.capture() + with pytest.raises(DeviceUnavailable, match="no tab open"): + await control.dispatch([]) + + +def test_its_declaration_says_it_is_a_website_browser_and_another_apps_says_nothing(): + assert Browser.declaration().browser is True and Mail.declaration().browser is None @pytest.mark.asyncio diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index 61d5cc773..52ed716f4 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -44,14 +44,14 @@ class LinuxOS(BaseModel): """A Linux base: its distribution and version (``rolling`` for one with no versions, such as Arch), and the display - server its desktop runs.""" + server its desktop runs, ``none`` for one with no desktop, whose driver's screen is something else, a page say.""" model_config = ConfigDict(extra="forbid") type: Literal["linux"] = "linux" distribution: str = Field(pattern=r"^[a-z][a-z0-9-]*$", description="os-release's ID: ubuntu, debian, fedora, arch") version: str = Field(min_length=1, description="os-release's VERSION_ID, or rolling") - display: Literal["x11", "wayland"] + display: Literal["x11", "wayland", "none"] build: str | None = None diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 43bb3783d..06c6bfea1 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -1,7 +1,8 @@ The website browsers that come with agent-env, today `chromium`, a `computer_app` env a computer env lists in its -`apps`. +`apps`. Deployed alone, `chromium` is a browser computer: the `ubuntu-24.04-headless` base, with no display, and the +`page` driver, whose screen is the browser's active tab, headless. -`envs/chromium/` is Chromium for an X11 desktop on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the +`envs/chromium/` is Chromium for an X11 desktop or a computer with no display on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists it is put and unpacked into `/opt/agentenv/apps/chromium/` by its install step. It's a website browser (`agentenv_protocol.website_browser.WebsiteBrowserEnvironment`): agentenv launches it on a debugging pipe with its own diff --git a/src/agent_env/browsers/envs/chromium/chromium.py b/src/agent_env/browsers/envs/chromium/chromium.py index 4fb265ae8..33331d9c5 100644 --- a/src/agent_env/browsers/envs/chromium/chromium.py +++ b/src/agent_env/browsers/envs/chromium/chromium.py @@ -25,7 +25,7 @@ PROFILE = ".config/chromium" -@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11", "none"]}, "arch": ["amd64", "arm64"]}]) @linux(packages=Packages(apt=PACKAGES), downloads=DOWNLOADS, paths=[PROFILE], launch=Launch(at="boot", command=["chromium/chrome/chrome", "--no-sandbox", f"--class={WINDOW_CLASS}"], window={"class": WINDOW_CLASS}, ready=Ready(window=True))) diff --git a/src/agent_env/browsers/envs/chromium/env.toml b/src/agent_env/browsers/envs/chromium/env.toml index 8a3a20ae1..05625f6a1 100644 --- a/src/agent_env/browsers/envs/chromium/env.toml +++ b/src/agent_env/browsers/envs/chromium/env.toml @@ -1,7 +1,7 @@ type = "computer_app" module = "chromium.py:Chromium" -computer = "ubuntu-24.04" -driver = "reference" +computer = "ubuntu-24.04-headless" +driver = "page" [settings] start_urls = ["about:blank"] diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 8ebce59df..3ee85568d 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -8,6 +8,14 @@ container form's image is built from the folder's `Dockerfile`, and a release pu building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04`. +`artifacts/ubuntu-24.04-headless/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop +(`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the +browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the +session user, among them a stand-in site proxy on `127.0.0.1:3128`, the browser's only route out, which forwards every +request to wherever it names, refusing nothing: the sandbox's network policy is all that limits it until the shared +site proxy replaces it. It has a container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by +its id, `@local/agentenv-framework/computers/ubuntu-24.04-headless`. + It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, and an ext4 root filesystem that is the image's own, with `/etc/agentenv/vm/image.env`, its entrypoint, command, diff --git a/src/agent_env/computers/aliases.toml b/src/agent_env/computers/aliases.toml index c33416836..5b6e5b26d 100644 --- a/src/agent_env/computers/aliases.toml +++ b/src/agent_env/computers/aliases.toml @@ -1,3 +1,4 @@ # Ids a task can name these computers by; a store's artifact of the id wins over an alias. [artifacts] "ubuntu-24.04" = ["ubuntu-24.04"] +"ubuntu-24.04-headless" = ["ubuntu-24.04-headless"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile new file mode 100644 index 000000000..694f80d78 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile @@ -0,0 +1,22 @@ +# The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), for one +# website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord runs +# every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. Composing +# a driver or an app onto it adds its services as programs in /etc/supervisor/conf.d. +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + supervisor tini python3 python3-venv fonts-dejavu-core tzdata ca-certificates procps \ + && rm -rf /var/lib/apt/lists/* \ + && userdel --remove ubuntu \ + && useradd --create-home --uid 1000 --shell /bin/bash user \ + && runuser -u user -- mkdir -p /home/user/Desktop /home/user/Documents /home/user/Downloads \ + && mkdir -p /etc/supervisor/conf.d /var/log/agentenv /run/agentenv \ + && chown user:user /var/log/agentenv /run/agentenv +COPY supervisord.conf /etc/supervisor/supervisord.conf +COPY start-computer /usr/local/bin/start-computer +COPY site-proxy.py /usr/local/bin/agentenv-site-proxy +USER user +WORKDIR /home/user +ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3128 +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/usr/local/bin/start-computer"] diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml new file mode 100644 index 000000000..dfcd4d670 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml @@ -0,0 +1,14 @@ +type = "computer" +description = "Ubuntu 24.04 with no display, for one website browser that runs headless and a page driver whose screen is its page; run by supervisord as the session user" +arch = ["amd64", "arm64"] +os = { type = "linux", distribution = "ubuntu", version = "24.04", display = "none" } +session = { user = "user" } + +[forms.container] +image = { dockerfile = "Dockerfile" } +init = "supervisord" +input_channel = "env" + +[config.timezone] +default = "UTC" +phase = "deploy" diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py new file mode 100644 index 000000000..5eacb8004 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py @@ -0,0 +1,74 @@ +"""A stand-in for the shared site proxy in this base: it forwards every request the browser sends it, absolute-form +HTTP and CONNECT tunnels alike, to wherever it names, refusing nothing, on 127.0.0.1:3128.""" +import http.server +import select +import socket +import socketserver +import urllib.parse + +PORT = 3128 +IDLE_S = 60 +HOP = {"connection", "keep-alive", "proxy-connection", "proxy-authorization"} + + +class Proxy(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_CONNECT(self): + host, _, port = self.path.rpartition(":") + host = host.removeprefix("[").removesuffix("]") + try: + upstream = socket.create_connection((host, int(port)), timeout=10) + except (OSError, ValueError): + self.send_error(502) + return + self.send_response(200, "Connection established") + self.end_headers() + self.close_connection = True + _pipe(self.connection, upstream) + + def forward(self): + url = urllib.parse.urlsplit(self.path) + try: + upstream = socket.create_connection((url.hostname, url.port or 80), timeout=10) + except (OSError, TypeError): + self.send_error(502) + return + target = (url.path or "/") + (f"?{url.query}" if url.query else "") + headers = "".join(f"{k}: {v}\r\n" for k, v in self.headers.items() if k.lower() not in HOP) + upstream.sendall(f"{self.command} {target} HTTP/1.1\r\n{headers}Connection: close\r\n\r\n".encode("latin-1")) + if length := int(self.headers.get("Content-Length") or 0): + upstream.sendall(self.rfile.read(length)) + self.close_connection = True + _pipe(self.connection, upstream) + + do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_HEAD = do_OPTIONS = forward + + def log_message(self, *args): + pass + + +def _pipe(a: socket.socket, b: socket.socket) -> None: + try: + while True: + ready, _, _ = select.select([a, b], [], [], IDLE_S) + if not ready: + return + for s in ready: + data = s.recv(65536) + if not data: + return + (b if s is a else a).sendall(data) + except OSError: + return + finally: + b.close() + + +class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +if __name__ == "__main__": + Server(("127.0.0.1", PORT), Proxy).serve_forever() diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer new file mode 100755 index 000000000..125e1d88c --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer @@ -0,0 +1,8 @@ +#!/bin/sh +# The base's start: the time zone a deploy passes as AGENTENV_SETTING_TIMEZONE applied for every program, then +# supervisord. +set -eu +timezone="${AGENTENV_SETTING_TIMEZONE:-UTC}" +[ -f "/usr/share/zoneinfo/$timezone" ] || { echo "timezone '$timezone' isn't in /usr/share/zoneinfo" >&2; exit 2; } +export TZ="$timezone" +exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf new file mode 100644 index 000000000..da2860323 --- /dev/null +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf @@ -0,0 +1,26 @@ +; The base computer's own program, the site proxy, run as the session user. A composed driver or app adds its own in +; conf.d. +[supervisord] +nodaemon=true +logfile=/var/log/agentenv/supervisord.log +pidfile=/run/agentenv/supervisord.pid +childlogdir=/var/log/agentenv + +[unix_http_server] +file=/run/agentenv/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/agentenv/supervisor.sock + +[program:site-proxy] +command=/usr/bin/python3 /usr/local/bin/agentenv-site-proxy +priority=10 +autorestart=true +redirect_stderr=true +stdout_logfile=/var/log/agentenv/site-proxy.log + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 893bcc807..0da83b5a8 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -7,6 +7,12 @@ pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer `x11grab` through PyAV at up to 30 fps. Name it from another bundle as `reference`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/reference`. +`envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `ubuntu-24.04-headless`: +its screen is the computer's one website browser's active tab, its `screen` feature `page`. A screenshot is the page, +and input reaches it as CDP input events, both through `browser/v1`'s `page/*` operations on the browser's own server; +a tab that paints nothing shows white. The browser ending on its own fails the computer for good: its ready check is +`final`, and its tools answer with why. Name it as `page`, or by its id, `@local/agentenv-framework/drivers/page`. + A run that names it writes it: its files, what its class declares for each platform it supports, read without `mss`, `python-xlib` or PyAV installed, and how a computer installs it. Its `env.toml`'s `[install.linux]` has a computer env's composer put the folder in `/opt/agentenv/drivers/reference/`, install `requirements.txt`, resolved and diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index 43b2f3510..daa0d6144 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -2,3 +2,4 @@ [envs] reference = ["reference"] cua-driver = ["cua-driver"] +page = ["page"] diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py new file mode 100644 index 000000000..e9dcdc60f --- /dev/null +++ b/src/agent_env/drivers/envs/page/driver.py @@ -0,0 +1,326 @@ +"""The page driver, for a computer with no display whose one website browser runs headless: its screen is the browser's +active tab. A screenshot is the page, and input reaches the page as CDP input events, both through the browser's own +server on its socket (``browser/v1``'s ``page/*`` operations), since only that server holds the browser's debugging pipe. +The browser ending on its own is an infrastructure failure, since nothing here launches it again. Needs +``agentenv-framework-protocol[computer]`` where it runs.""" +from __future__ import annotations + +import asyncio +import base64 +import math +import string +from pathlib import Path +from typing import Any, Optional + +import httpx + +from agentenv_protocol import client +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + DeviceUnavailable, + Display, + Image, + ReadyCheck, + Recording, + Unsupported, + capture, + click, + computer_driver, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + scroll, + type_text, +) +from agentenv_protocol.computer import apps +from agentenv_protocol.computer.app import SOCKET +from agentenv_protocol.website_browser import BROWSER + +BUTTONS = {"left": 1, "right": 2, "middle": 4, "back": 8, "forward": 16} +MODIFIERS = {"Alt": 1, "Control": 2, "Meta": 4, "Shift": 8} +#: Keys that aren't characters: each one's DOM code, its Windows virtual key code, and the text a press of it types. +NAMED = { + "Enter": ("Enter", 13, "\r"), "Tab": ("Tab", 9, ""), "Escape": ("Escape", 27, ""), + "Backspace": ("Backspace", 8, ""), "Delete": ("Delete", 46, ""), "Insert": ("Insert", 45, ""), + "ArrowUp": ("ArrowUp", 38, ""), "ArrowDown": ("ArrowDown", 40, ""), "ArrowLeft": ("ArrowLeft", 37, ""), + "ArrowRight": ("ArrowRight", 39, ""), "Home": ("Home", 36, ""), "End": ("End", 35, ""), + "PageUp": ("PageUp", 33, ""), "PageDown": ("PageDown", 34, ""), "Control": ("ControlLeft", 17, ""), + "Shift": ("ShiftLeft", 16, ""), "Alt": ("AltLeft", 18, ""), "Meta": ("MetaLeft", 91, ""), + " ": ("Space", 32, " "), **{f"F{n}": (f"F{n}", 111 + n, "") for n in range(1, 13)}, +} +#: The US layout's punctuation: each key's DOM code and virtual key code, and the character Shift makes of it. +PUNCTUATION = {"-": ("Minus", 189, "_"), "=": ("Equal", 187, "+"), "[": ("BracketLeft", 219, "{"), + "]": ("BracketRight", 221, "}"), "\\": ("Backslash", 220, "|"), ";": ("Semicolon", 186, ":"), + "'": ("Quote", 222, '"'), ",": ("Comma", 188, "<"), ".": ("Period", 190, ">"), + "/": ("Slash", 191, "?"), "`": ("Backquote", 192, "~")} +SHIFTED_DIGITS = dict(zip(")!@#$%^&*(", string.digits)) +SHIFTED = {**{shifted: key for key, (_, _, shifted) in PUNCTUATION.items()}, **SHIFTED_DIGITS} +TYPED = {"\n": "Enter", "\t": "Tab"} +#: How far one scroll step moves, and how far apart a drag's pointer moves are, in pixels. +WHEEL_PX = 100 +DRAG_STEP_PX = 20 +#: A pixel of a tab with nothing in it. +BLANK = b"\xff\xff\xff" +#: How long one call to the browser's server may take. +CALL_TIMEOUT_S = 30.0 + + +@computer_driver(name="page", supports=[{"os": {"type": "linux", "display": ["none"]}, "arch": ["amd64", "arm64"]}]) +class PageDriver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle", "back", "forward") + max_click_count = 3 + max_drag_points = 50 + keys = sorted({*NAMED, *string.ascii_letters, *string.digits, *PUNCTUATION, *SHIFTED}) + recording = Recording(max_fps=10) + screen_kind = "page" + notes = ("This computer is one web page: the browser's active tab, with no address bar, tab strip or desktop " + "around it. Its screen is the page's viewport.") + + def __init__(self) -> None: + super().__init__() + self._browser: Optional[tuple[Path, dict]] = None + self._pointer = (0.0, 0.0) + self._buttons = 0 + self._held: list[str] = [] + + @display + async def viewport(self) -> Display: + size = await self._call("page/viewport") + return Display(size["width"], size["height"], 1.0) + + @capture + async def page(self) -> Image: + """The page, or a blank one where the browser shows nothing, as a desktop browser shows a tab with nothing + in it.""" + shot = await self._call("page/capture") + if shot.get("blank"): + size = await self.viewport() + return Image(size.width, size.height, BLANK * (size.width * size.height), "RGB") + data = base64.b64decode(shot["png"]) + return Image(int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big"), data, "png") + + async def ready_checks(self) -> list[ReadyCheck]: + try: + folder, _ = await self._found() + except DeviceUnavailable as e: + return [ReadyCheck("browser", False, e.message)] + failed = apps.ended_on_its_own(folder) + return [ReadyCheck("browser", failed is None, failed or f"the page of the browser in {folder.name}", + final=failed is not None)] + + @click + async def click_at(self, x, y, button, count, modifiers) -> None: + mask = _modifiers(modifiers) | self._mask() + events = [_mouse("mouseMoved", x, y, modifiers=mask)] + for n in range(1, count + 1): + events += [_mouse("mousePressed", x, y, button, BUTTONS[button], n, mask), + _mouse("mouseReleased", x, y, button, 0, n, mask)] + self._pointer = (x, y) + await self._input(events) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers) -> None: + mask, held = _modifiers(modifiers) | self._mask(), BUTTONS[button] + steps = _steps(path) + pause = duration_ms / 1000 / len(steps) + await self._input([_mouse("mouseMoved", *path[0], modifiers=mask), + _mouse("mousePressed", *path[0], button, held, 1, mask)]) + try: + for x, y in steps: + await self._input([_mouse("mouseMoved", x, y, "none", held, 0, mask)]) + await asyncio.sleep(pause) + finally: + await self._input([_mouse("mouseReleased", *path[-1], button, 0, 1, mask)]) + self._pointer = tuple(path[-1]) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers) -> None: + dx = {"left": -1, "right": 1}.get(direction, 0) * amount * WHEEL_PX + dy = {"up": -1, "down": 1}.get(direction, 0) * amount * WHEEL_PX + self._pointer = (x, y) + await self._input([{"method": "Input.dispatchMouseEvent", "params": { + "type": "mouseWheel", "x": x, "y": y, "deltaX": dx, "deltaY": dy, + "modifiers": _modifiers(modifiers) | self._mask()}}]) + + @pointer_move + async def move(self, x, y) -> None: + self._pointer = (x, y) + await self._input([_mouse("mouseMoved", x, y, "none", self._buttons, 0, self._mask())]) + + @pointer_down + async def down(self, button) -> None: + self._buttons |= BUTTONS[button] + await self._input([_mouse("mousePressed", *self._pointer, button, self._buttons, 1, self._mask())]) + + @pointer_up + async def up(self, button) -> None: + self._buttons &= ~BUTTONS[button] + await self._input([_mouse("mouseReleased", *self._pointer, button, self._buttons, 1, self._mask())]) + + @pointer_position + async def where(self) -> tuple[int, int]: + return round(self._pointer[0]), round(self._pointer[1]) + + @type_text + async def type_in(self, text) -> None: + """Each character as the keystroke that types it, with the keys held down still held: Shift held types ``a`` as + ``A``, and Control held makes it a shortcut; a character no key types is inserted as text.""" + held, events = self._mask(), [] + for ch in text: + key = TYPED.get(ch, ch) + if key in NAMED or _printable(key): + mask = held | (MODIFIERS["Shift"] if key in SHIFTED or key.isupper() else 0) + events += _keystroke(_shifted(key) if mask & MODIFIERS["Shift"] else key, mask) + else: + events.append({"method": "Input.insertText", "params": {"text": ch}}) + await self._input(events) + + @press + async def press_keys(self, keys) -> None: + events, mask = [], self._mask() + for key in keys: + mask |= MODIFIERS.get(key, 0) + events.append(_key("keyDown", key, mask)) + for key in reversed(keys): + mask &= ~MODIFIERS.get(key, 0) + events.append(_key("keyUp", key, mask)) + await self._input(events) + + @key_down + async def keys_down(self, keys) -> None: + self._held += [key for key in keys if key not in self._held] + await self._input([_key("keyDown", key, self._mask()) for key in keys]) + + @key_up + async def keys_up(self, keys) -> None: + self._held = [key for key in self._held if key not in keys] + await self._input([_key("keyUp", key, self._mask()) for key in keys]) + + def _mask(self) -> int: + return _modifiers(self._held) + + async def _input(self, events: list[dict]) -> None: + await self._call("page/input", {"events": events}) + + async def _call(self, op: str, params: Optional[dict] = None) -> Any: + """``browser/v1``'s ``op`` on the browser's server; DeviceUnavailable when it can't answer or says why not, and + once the browser has ended on its own, why, even after its server has launched another.""" + folder, card = await self._found() + if (failed := _ended(folder)) is not None: + raise DeviceUnavailable(failed) + method = client.find_extension_method(card, BROWSER, op) + if method is None: + raise Unsupported(f"the browser in {folder.name} serves no {op}") + try: + async with _http(folder) as http: + answer = await http.request(method["method"], method["endpoint"], + **({"json": params} if method["method"] != "GET" else {})) + except (httpx.HTTPError, OSError) as e: + raise DeviceUnavailable(_ended(folder) or f"the browser's server isn't answering: {type(e).__name__}: {e}" + ) from None + if answer.status_code >= 400: + raise DeviceUnavailable(_ended(folder) or f"the browser couldn't {op}: {answer.text[:300]}") + return answer.json() + + async def _found(self) -> tuple[Path, dict]: + """The folder and card of the computer's one website browser, found once its server answers.""" + if self._browser is None: + for folder in self._computer.apps: + card = await apps.served_card(folder) + if card is not None and client.find_extension_method(card, BROWSER, "page/capture"): + self._browser = (folder, card) + break + else: + raise DeviceUnavailable("no website browser on this computer answers yet") + return self._browser + + +def _ended(folder: Path) -> Optional[str]: + if (failed := apps.ended_on_its_own(folder)) is not None: + return f"the browser in {folder.name} has failed: {failed}" + return None + + +def _http(folder: Path) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(uds=str(folder / SOCKET)), base_url="http://app", + timeout=CALL_TIMEOUT_S) + + +def _mouse(kind: str, x: float, y: float, button: str = "none", buttons: int = 0, count: int = 0, + modifiers: int = 0) -> dict: + return {"method": "Input.dispatchMouseEvent", "params": { + "type": kind, "x": x, "y": y, "button": button, "buttons": buttons, "clickCount": count, "modifiers": modifiers}} + + +def _keystroke(key: str, modifiers: int) -> list[dict]: + return [_key("keyDown", key, modifiers), _key("keyUp", key, modifiers)] + + +def _key(kind: str, key: str, modifiers: int) -> dict: + """A CDP key event for the W3C key ``key`` with ``modifiers`` held: one with Control, Alt or Meta held types + nothing, so it reaches the page as a shortcut.""" + code, vk, text = _codes(key) + if modifiers & (MODIFIERS["Control"] | MODIFIERS["Alt"] | MODIFIERS["Meta"]): + text = "" + params = {"type": ("keyDown" if text else "rawKeyDown") if kind == "keyDown" else "keyUp", "key": key, + "code": code, "windowsVirtualKeyCode": vk, "nativeVirtualKeyCode": vk, "modifiers": modifiers} + if kind == "keyDown" and text: + params.update(text=text, unmodifiedText=text) + return {"method": "Input.dispatchKeyEvent", "params": params} + + +def _codes(key: str) -> tuple[str, int, str]: + """The DOM code, virtual key code and text of ``key`` on a US layout.""" + if key in NAMED: + return NAMED[key] + if len(key) == 1 and key in string.ascii_letters: + return f"Key{key.upper()}", ord(key.upper()), key + if len(key) == 1 and key in string.digits: + return f"Digit{key}", ord(key), key + if key in PUNCTUATION: + code, vk, _ = PUNCTUATION[key] + return code, vk, key + if key in SHIFTED: + code, vk, _ = _codes(SHIFTED[key]) + return code, vk, key + raise Unsupported(f"this driver has no key {key!r}") + + +def _shifted(key: str) -> str: + """The key Shift makes of ``key`` on a US layout: a letter's capital, a digit's or punctuation's symbol.""" + if len(key) == 1 and key in string.ascii_lowercase: + return key.upper() + if key in PUNCTUATION: + return PUNCTUATION[key][2] + return {digit: symbol for symbol, digit in SHIFTED_DIGITS.items()}.get(key, key) + + +def _printable(key: str) -> bool: + return len(key) == 1 and (key in string.ascii_letters or key in string.digits or key in PUNCTUATION + or key in SHIFTED) + + +def _modifiers(keys) -> int: + mask = 0 + for key in keys: + mask |= MODIFIERS.get(key, 0) + return mask + + +def _steps(path: list) -> list[tuple[float, float]]: + """The points a drag moves through after its start, about every DRAG_STEP_PX pixels.""" + steps = [] + for (x0, y0), (x1, y1) in zip(path, path[1:]): + n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX)) + steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] + return steps + + +if __name__ == "__main__": + PageDriver().serve() diff --git a/src/agent_env/drivers/envs/page/env.toml b/src/agent_env/drivers/envs/page/env.toml new file mode 100644 index 000000000..6af38616c --- /dev/null +++ b/src/agent_env/drivers/envs/page/env.toml @@ -0,0 +1,7 @@ +type = "computer_driver" +module = "driver.py:PageDriver" +port = 18765 + +[install.linux] +python = "requirements.txt" +services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:PageDriver" }] diff --git a/src/agent_env/drivers/envs/page/requirements.txt b/src/agent_env/drivers/envs/page/requirements.txt new file mode 100644 index 000000000..6b36e62a6 --- /dev/null +++ b/src/agent_env/drivers/envs/page/requirements.txt @@ -0,0 +1,3 @@ +# What a composed computer installs into the driver's venv. A release pins agentenv-framework-protocol to the version +# it ships, which serves the driver with agentenv_protocol.computer.serve. +agentenv-framework-protocol[computer]>=0.1.381 diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index d57d0ba5a..36f374494 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -124,7 +124,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact elif computer is not None and not problems: _, refused = gate.check(computer, driver, environment_name=environment_name, apps=[], display=None, network=None, computer_settings={}) - problems += refused + gate.check_app(computer, driver, app) + problems += refused + gate.check_app(computer, driver, app) + gate.check_screen(driver, [app], "app") if problems: raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) app = get_env_store().put_document(app) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 5e36d9a8f..b1a3954a8 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -118,6 +118,7 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, apps, app_settings = list(apps), app_settings or {} values, problems = gate.check(computer, driver, environment_name=environment_name, apps=apps, display=display, network=network, computer_settings=computer_settings or {}) + problems += gate.check_screen(driver, apps) by_name = {app.environment_name: app for app in apps} for name, defaults in app_settings.items(): problems += ([f"apps.{name}.settings: {id} has no app {name!r}"] if name not in by_name else diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index 634638db6..8c9885b38 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -37,6 +37,10 @@ _POLL_S = 0.5 #: How a setting a record keeps shows a secret's or a generated one's value, which it never holds. WITHHELD = "" + + +class ComputerFailed(RuntimeError): + """The computer's ``computer-ready`` status says it failed for good, a check that can't pass again.""" #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 #: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in. @@ -168,12 +172,13 @@ def ready_timeout(apps: list[ComputerAppEnv], os_type: str) -> float: async def ready(url: str, timeout: float, token: Optional[str] = None) -> dict: """The card of the driver at ``url``, read once its ``computer-ready`` status, asked with its control ``token``, - says it's ready, so each app it fronts is on it as the app's own server serves it; raises TimeoutError, naming - what it last saw, when it isn't within ``timeout`` seconds.""" - last = "nothing answered" + says it's ready, so each app it fronts is on it as the app's own server serves it; raises ComputerFailed as soon + as the status says it failed, and TimeoutError, naming what it last saw, when it isn't ready within ``timeout`` + seconds.""" + last, failed = "nothing answered", None try: async with asyncio.timeout(timeout): - while True: + while failed is None: try: card = await protocol_v1.get_card(url, timeout=5) status = await protocol_v1.invoke_extension(url, card, COMPUTER_READY, None, _STATUS_TIMEOUT_S, @@ -181,10 +186,13 @@ async def ready(url: str, timeout: float, token: Optional[str] = None) -> dict: headers=protocol_v1.control_headers(token)) if status.get("ready"): return await protocol_v1.get_card(url, timeout=5) + failed = status.get("failed") last = "not ready: " + "; ".join(f"{check['name']}: {check.get('detail')}" for check in status.get("checks", []) if not check.get("ok")) except Exception as e: last = f"{type(e).__name__}: {e}" - await asyncio.sleep(_POLL_S) + if failed is None: + await asyncio.sleep(_POLL_S) except TimeoutError: raise TimeoutError(f"the computer at {url} wasn't ready within {timeout:.0f} s ({last})") from None + raise ComputerFailed(f"the computer at {url} failed: {failed}") diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index 056d78f19..f7326acb0 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -38,6 +38,18 @@ def check(base: ComputerArtifact, driver: ComputerDriverEnv, *, environment_name return values, problems + invalid +def check_screen(driver: ComputerDriverEnv, apps: Sequence[ComputerAppEnv], at: str = "apps") -> list[str]: + """Why ``apps`` don't fit ``driver`` when its screen is a website browser's page: that needs exactly one website + browser among them.""" + if not any(platform.features.get("screen") == "page" for platform in driver.declaration.platforms): + return [] + browsers = [app.id for app in apps if app.declaration.browser] + if len(browsers) == 1: + return [] + return [f"{at}: {driver.id}'s screen is a website browser's page, so its computer takes exactly one website " + f"browser; it has {len(browsers)}{': ' + ', '.join(browsers) if browsers else ''}"] + + def forms(base: ComputerArtifact) -> list[str]: """The forms a composition of ``base`` builds: every one the base has.""" return [form for form in FORMS if getattr(base.forms, form, None)] diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index 54a426c05..58e6773b3 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -71,18 +71,20 @@ def ref(self) -> str: # The example env `a2a-agent validate` deploys, so a remote sandbox pulls its image rather than building it. EXAMPLE_IMAGES = (PublishedImage("agentenv-mail-mcp", _EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile"),) # The base computers the computers bundle ships, so a run pulls a base rather than building its desktop. -COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"),) +COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"), + PublishedImage("agentenv-ubuntu-24.04-headless", + _COMPUTERS / "ubuntu-24.04-headless" / "Dockerfile")) # The shipped bundles' published images, whose requirements keep a protocol floor. BUNDLE_IMAGES = AGENT_IMAGES + EXAMPLE_IMAGES + COMPUTER_IMAGES # What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. DISK_TOOLS_IMAGE = PublishedImage("agentenv-disk-tools", DISK / "tools" / "Dockerfile") LAUNCHER_IMAGE = PublishedImage("agentenv-qemu-launcher", DISK / "launcher" / "Dockerfile") -# Each base computer's disk form: its container form's root filesystem on the boot recipe, at the size its [forms.disk] -# declares, so a run pulls it rather than building it from the two. +# Each base computer's disk form, for a base that has one: its container form's root filesystem on the boot recipe, at +# the size its [forms.disk] declares, so a run pulls it rather than building it from the two. DISK_IMAGES = tuple(PublishedImage( f"{image.name}-disk", DISK / "boot" / "Dockerfile", (("ROOTFS", image), ("TOOLS", DISK_TOOLS_IMAGE)), - (("SIZE_GB", str(tomllib.loads((image.dockerfile.parent / "artifact.toml").read_text())["forms"]["disk"][ - "min_disk_gb"])),)) for image in COMPUTER_IMAGES) + (("SIZE_GB", str(disk["min_disk_gb"])),)) for image in COMPUTER_IMAGES + if (disk := tomllib.loads((image.dockerfile.parent / "artifact.toml").read_text())["forms"].get("disk"))) def source_tag(digest: str) -> str: diff --git a/tst/integration/computer/browser-computer/tasks/chromium.json b/tst/integration/computer/browser-computer/tasks/chromium.json new file mode 100644 index 000000000..de36cdb48 --- /dev/null +++ b/tst/integration/computer/browser-computer/tasks/chromium.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "chromium"} +] diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py new file mode 100644 index 000000000..efe205fff --- /dev/null +++ b/tst/integration/computer/browser_computer_test.py @@ -0,0 +1,227 @@ +"""The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display and the page +driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; deployed +through the computer provider like any computer, its computer_* tools see and drive the page; nothing runs X and nothing +listens but the driver and the site proxy; a killed app server or driver is started again by supervisord and the +computer is ready again; and chromium killed fails it for good, computer-ready/v1 saying why and the tools answering +with it. Needs Docker and the local registry on localhost:5000.""" + +import asyncio +import json +import platform +import shlex +import time +import urllib.parse + +import httpx +import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client +from PIL import Image + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER_READY +from agentenv_protocol.computer.conformance.client import image +from agentenv_protocol.website_browser import BROWSER + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] +#: How long an effect may take to reach the page, or the computer to be ready again; how often it's asked meanwhile. +WAIT_S, POLL_S = 120.0, 0.5 +# A page whose title says, as JSON, what reached it: clicks and double clicks on its red box, its input's value, how far +# it has scrolled and the last keys pressed. +PAGE = """loading +
+""" +PAGE_URL = "data:text/html," + urllib.parse.quote(PAGE) +# Every TCP port something in the computer listens on, and whether any process is an X server. +LISTENING = """import glob +ports = set() +for table in ("/proc/net/tcp", "/proc/net/tcp6"): + for line in open(table).read().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A": + ports.add(int(fields[1].rsplit(":", 1)[1], 16)) +x = [p for p in glob.glob("/proc/[0-9]*/comm") if open(p).read().strip() in ("Xvfb", "Xorg", "xfce4-session")] +print(sorted(ports), bool(x)) +""" + + +def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver(browser_computer): + results = browser_computer.metadata["composed"]["container"] + probe = results[ARCH]["probe"] + + with namespace_routing(): + report = json.loads(FileArtifact.get(probe["report"]["id"], probe["report"]["version"]).load()) + print(json.dumps({"counts": probe["checks"], "extensions": probe["extensions"], + "not passed": [(c["check"], c["name"], c["status"], c["detail"][:160]) for c in report["checks"] + if c["status"] != "passed"]}, indent=1)) + assert results[ARCH]["status"] == "ready" and probe["passed"], results[ARCH] + assert [app.environment_name for app in browser_computer.composed_apps()] == ["chromium"] + + +@pytest.mark.asyncio +async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + await _state(deployed, lambda s: True) + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + shot = image(await session.call_tool("computer_screenshot", {"space": "frame"})) + await _tool(session, "computer_click", {"x": 200, "y": 150, "space": "frame"}) + clicked = await _state(deployed, lambda s: s["c"] == 1) + await _tool(session, "computer_click", {"x": 200, "y": 150, "count": 2, "space": "frame"}) + doubled = await _state(deployed, lambda s: s["d"] == 1) + await _tool(session, "computer_click", {"x": 300, "y": 320, "space": "frame"}) + await _tool(session, "computer_type", {"text": "Hello, World!"}) + typed = await _state(deployed, lambda s: s["v"] == "Hello, World!") + await _tool(session, "computer_key", {"keys": ["Control", "a"]}) + await _tool(session, "computer_type", {"text": "x"}) + replaced = await _state(deployed, lambda s: s["v"] == "x") + await _tool(session, "computer_key", {"keys": ["Enter"]}) + entered = await _state(deployed, lambda s: "Enter" in s["k"]) + await _tool(session, "computer_scroll", {"x": 640, "y": 600, "direction": "down", "amount": 3, + "space": "frame"}) + scrolled = await _state(deployed, lambda s: s["y"] >= 300) + after = image(await session.call_tool("computer_screenshot", {"space": "frame"})) + listening, x_server = (await inside.run(deployed, ["python3", "-c", LISTENING], timeout_s=30))[ + "stdout"].rsplit(" ", 1) + command = (await inside.run(deployed, ["sh", "-c", "tr '\\0' ' ' < /proc/$(pgrep -o -f -- " + "'[-]-remote-debugging-pipe')/cmdline"], timeout_s=30))["stdout"] + finally: + report = await teardown_run(context) + + assert shot.size == (1280, 800) and shot.getpixel((200, 150)) == (255, 0, 0) + assert shot.getpixel((640, 700)) == (255, 255, 255) + assert clicked["c"] == 1 and doubled["d"] == 1 and typed["v"] == "Hello, World!" and replaced["v"] == "x" + assert "C-a" in replaced["k"] and "Enter" in entered["k"] and scrolled["y"] >= 300 + assert after.getpixel((200, 150)) != (255, 0, 0) + assert json.loads(listening) == [3128, 18765] and x_server.strip() == "False" + assert "--headless=new" in command and "--window-size=1280,800" in command + assert not report.failed + + +@pytest.mark.asyncio +async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fails_it_for_good(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + await _kill(deployed, "[s]erve_app") + server_back = await _ready(deployed) + launches = (await _record(deployed))["launches"] + await _kill(deployed, "[c]omputer.serve driver.py:PageDriver") + driver_back = await _ready(deployed) + shot = await _screenshot(deployed) + await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) + failed = await _until(deployed, lambda status: "failed" in status) + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + refused = await session.call_tool("computer_screenshot", {"space": "frame"}) + finally: + report = await teardown_run(context) + + assert server_back["ready"] and driver_back["ready"] and shot.size == (1280, 800) + assert [entry.get("cause") for entry in launches if entry["event"] == "launch"] == ["boot", "server_restart"] + browser = next(check for check in failed["checks"] if check["name"] == "browser") + assert failed["ready"] is False and browser["final"] is True and failed["failed"] == f"browser: {browser['detail']}" + assert "by signal SIGKILL" in browser["detail"] and "infrastructure failure" in browser["detail"] + assert refused.isError and "has failed" in refused.content[0].text and "SIGKILL" in refused.content[0].text + assert not report.failed + + +async def _tool(session: ClientSession, name: str, args: dict) -> None: + result = await session.call_tool(name, args) + assert not result.isError, f"{name} {args}: {result.content[0].text}" + + +async def _record(deployed) -> dict: + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + app = next(child for child in card["children_environments"] if child["name"] == "chromium") + path = protocol_v1.find_extension_method(app, BROWSER, "record")["endpoint"] + return (await _get(deployed, path, headers=deployed.control_headers)).json() + + +async def _state(deployed, holds) -> dict: + """The page's state, from its title, once ``holds`` says it has what's waited for, within ``WAIT_S``.""" + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + try: + last = json.loads((await _record(deployed))["tabs"][0]["title"]) + if holds(last): + return last + except (httpx.HTTPError, ValueError, KeyError, IndexError): + pass + await asyncio.sleep(POLL_S) + raise AssertionError(f"the page didn't get there within {WAIT_S:g} s; it last said {last!r}") + + +async def _status(deployed) -> dict: + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + path = protocol_v1.find_extension_method(card, COMPUTER_READY, "status")["endpoint"] + return (await _get(deployed, path, headers=deployed.control_headers)).json() + + +async def _until(deployed, holds) -> dict: + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + try: + last = await _status(deployed) + if holds(last): + return last + except (httpx.HTTPError, ValueError, KeyError): + pass + await asyncio.sleep(POLL_S) + raise AssertionError(f"computer-ready didn't get there within {WAIT_S:g} s; it last said {last!r}") + + +async def _ready(deployed) -> dict: + await asyncio.sleep(2) + return await _until(deployed, lambda status: status["ready"]) + + +async def _kill(deployed, pattern: str, oldest: bool = False) -> None: + """SIGKILL the newest, or oldest, process matching ``pattern``, a second after computer-exec has answered, since the + driver that runs the command may be the one killed.""" + pick = "-o" if oldest else "-n" + killer = f"sleep 1; kill -9 $(pgrep {pick} -f -- '{pattern}')" + await inside.run(deployed, ["sh", "-c", f"setsid sh -c {shlex.quote(killer)} >/dev/null 2>&1 < /dev/null &"], + timeout_s=30) + await asyncio.sleep(2) + + +async def _screenshot(deployed) -> Image.Image: + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + return image(await session.call_tool("computer_screenshot", {"space": "frame"})) + + +async def _get(deployed, path: str, **kwargs) -> httpx.Response: + async with httpx.AsyncClient(timeout=30) as http: + return await http.get(f"{deployed.environment_url}{path}", **kwargs) diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index 5d14ba95b..fc33fb45a 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -1,6 +1,6 @@ """The ubuntu-24.04 base computer, built from its folder once for the tests here and tagged for the local registry; the -shipped desktop example's computer env, the cua-driver test bundle's, the chromium test bundle's and the website-browser -test bundle's, each composed once for a module's tests; +shipped desktop example's computer env, the cua-driver test bundle's, the chromium test bundle's, the website-browser +test bundle's and agentenv's chromium deployed alone, each composed once for a module's tests; ``state``, local stores for a test that runs a bundle; and ``docker``, which runs the docker CLI for them.""" import logging @@ -28,6 +28,8 @@ CUA_DRIVER = Path(__file__).parent / "cua-driver" CHROMIUM = Path(__file__).parent / "chromium" WEBSITE_BROWSER = Path(__file__).parent / "website-browser" +BROWSER_COMPUTER = Path(__file__).parent / "browser-computer" +CHROMIUM_ALONE = "@local/agentenv-framework/browsers/chromium" @pytest.fixture(scope="session") @@ -67,7 +69,15 @@ def website_browser_desktop(tmp_path_factory): yield from _composed(tmp_path_factory.mktemp("website-browser-desktop"), parse_bundle(WEBSITE_BROWSER)) -def _composed(root: Path, bundle): +@pytest.fixture(scope="module") +def browser_computer(tmp_path_factory): + """agentenv's chromium, which deploys alone as a browser computer: the ubuntu-24.04-headless base, with no display, and the + page driver, whose screen is the browser's page; composed, through the browser-computer test bundle's task that + deploys it, as ``desktop`` is.""" + yield from _composed(tmp_path_factory.mktemp("browser-computer"), parse_bundle(BROWSER_COMPUTER), CHROMIUM_ALONE) + + +def _composed(root: Path, bundle, env_id: str | None = None): patch = pytest.MonkeyPatch() patch.chdir(root) patch.setenv("XDG_STATE_HOME", str(root / "state")) @@ -78,7 +88,7 @@ def _composed(root: Path, bundle): with namespace_routing(): plan = plan_bundle(resolve_bundle(bundle), installed=installed_bundles) materialize(plan, build_here=frozenset(w.id for w in plan.writes if isinstance(w.source, BuiltImage))) - yield Env.get(f"{bundle.id_root}/desktop") + yield Env.get(env_id or f"{bundle.id_root}/desktop") reset_artifact_store() reset_config() patch.undo() diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index 2155b86d7..a2e440229 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -130,7 +130,8 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( policies["stdout"]).items() params = protocol_v1.extension_params(card, BROWSER) - assert (params["chromium"], params["clock"], list(params["methods"])) == ("156.0.8078.4", "real", ["record"]) + assert (params["chromium"], params["clock"], sorted(params["methods"])) == ( + "156.0.8078.4", "real", ["page/capture", "page/input", "page/viewport", "record"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 assert relaunched_readiness["ready"] is True, relaunched_readiness diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 9f3bd8274..39f75fc69 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -59,7 +59,8 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri assert table.refused == {} and table.refused_claims == {} assert {alias: named.target_id for alias, named in table.aliases.items()} == { "mail": "@local/agentenv-framework/mail-mcp/mail", "reference": "@local/agentenv-framework/drivers/reference", - "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "chromium": "@local/agentenv-framework/browsers/chromium"} + "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "page": "@local/agentenv-framework/drivers/page", + "chromium": "@local/agentenv-framework/browsers/chromium"} assert A2AAgent.VALIDATION_ENV_ID == "mail" def test_a_task_naming_no_agent_plans_with_the_shipped_default(tmp_path, monkeypatch, local_stores, cli_routing): diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py new file mode 100644 index 000000000..e99c7f4ef --- /dev/null +++ b/tst/unit/drivers/page_driver_test.py @@ -0,0 +1,165 @@ +"""The page driver against a fake browser server on the app's socket: it passes the definition checks with a page +screen, turns tool calls into the CDP input they mean, shows a tab that paints nothing as white, and once the browser +has ended on its own, its ready check fails for good and its calls say why.""" +from __future__ import annotations + +import base64 +import importlib.util +import json +from pathlib import Path + +import httpx +import pytest + +import agent_env +from agentenv_protocol.computer import DeviceUnavailable, Platform +from agentenv_protocol.computer import driver as computer_driver +from agentenv_protocol.computer.app_runtime import LAUNCHES +from agentenv_protocol.website_browser import BROWSER + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "page" / "driver.py" +NO_DISPLAY = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="none") +PAGE_OPS = ("page/viewport", "page/capture", "page/input") +CARD = {"name": "chromium", "capabilities": {"extensions": [{"uri": BROWSER, "params": {"methods": { + op: {"method": "POST" if op == "page/input" else "GET", "endpoint": f"/agentenv/ext/browser/{op}"} + for op in PAGE_OPS}}}]}} +#: A 2×1 PNG's first 24 bytes: its signature, then its IHDR's length, type, width and height. +PNG = b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + (2).to_bytes(4, "big") + (1).to_bytes(4, "big") + + +def _load(): + spec = importlib.util.spec_from_file_location("page_driver", DRIVER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def page(monkeypatch, tmp_path): + """The driver, on a computer with no display, its one browser in ``tmp_path``, whose server answers ``served``, + a dict of answers by op, and records each request it gets in ``sent``.""" + module = _load() + monkeypatch.setattr(computer_driver, "detect", lambda: NO_DISPLAY) + driver = module.PageDriver() + driver._browser = (tmp_path, CARD) + driver.sent, driver.served = [], {"page/viewport": {"width": 1280, "height": 800}} + + def handler(request: httpx.Request) -> httpx.Response: + op = request.url.path.removeprefix("/agentenv/ext/browser/") + driver.sent.append((op, json.loads(request.content) if request.content else None)) + answer = driver.served.get(op, {}) + return answer if isinstance(answer, httpx.Response) else httpx.Response(200, json=answer) + + monkeypatch.setattr(module, "_http", lambda folder: httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://app")) + return driver + + +def _ended(folder: Path) -> None: + (folder / LAUNCHES).parent.mkdir(parents=True, exist_ok=True) + (folder / LAUNCHES).write_text(json.dumps({"event": "launch", "cause": "boot", "at": "t0"}) + "\n" + + json.dumps({"event": "exit", "signal": "SIGKILL", "at": "t1"}) + "\n") + + +def _events(page) -> list[dict]: + return [event for op, body in page.sent if op == "page/input" for event in body["events"]] + + +def test_its_screen_is_a_page_on_a_computer_with_no_display(): + module = _load() + + assert module.PageDriver.screen_kind == "page" + assert module.PageDriver.supports == [{"os": {"type": "linux", "display": ["none"]}, "arch": ["amd64", "arm64"]}] + + +@pytest.mark.asyncio +async def test_a_key_with_control_held_types_nothing_so_it_reaches_the_page_as_a_shortcut(page): + await page.press_keys(["Control", "a"]) + + assert [(e["params"]["type"], e["params"]["key"], e["params"]["modifiers"], e["params"].get("text")) + for e in _events(page)] == [("rawKeyDown", "Control", 2, None), ("rawKeyDown", "a", 2, None), + ("keyUp", "a", 2, None), ("keyUp", "Control", 0, None)] + + +@pytest.mark.asyncio +async def test_a_key_held_down_then_let_go_is_released_with_its_own_modifier_off(page): + await page.keys_down(["Shift"]) + await page.keys_up(["Shift"]) + + assert [(e["params"]["type"], e["params"]["modifiers"]) for e in _events(page)] == [("rawKeyDown", 8), + ("keyUp", 0)] + + +@pytest.mark.asyncio +async def test_with_shift_held_typed_text_is_shifted_and_with_control_held_a_click_carries_it(page): + await page.keys_down(["Shift"]) + await page.type_in("a1;") + await page.keys_up(["Shift"]) + await page.keys_down(["Control"]) + await page.click_at(10, 20, "left", 1, []) + await page.scroll_at(10, 20, "down", 1, ["Alt"]) + + events = _events(page) + typed = [(e["params"]["key"], e["params"]["modifiers"], e["params"]["text"]) for e in events + if e["params"].get("type") == "keyDown"] + assert typed == [("A", 8, "A"), ("!", 8, "!"), (":", 8, ":")] + assert {e["params"]["modifiers"] for e in events if e["params"]["type"] in ("mousePressed", "mouseReleased")} == {2} + assert events[-1]["params"]["type"] == "mouseWheel" and events[-1]["params"]["modifiers"] == 3 + + +@pytest.mark.asyncio +async def test_typed_text_is_keystrokes_with_shift_where_it_takes_one_and_inserted_text_for_the_rest(page): + await page.type_in("Hi!\né") + + events = _events(page) + downs = [(e["params"]["key"], e["params"]["modifiers"], e["params"]["text"]) for e in events + if e["params"].get("type") == "keyDown"] + assert downs == [("H", 8, "H"), ("i", 0, "i"), ("!", 8, "!"), ("Enter", 0, "\r")] + assert events[-1] == {"method": "Input.insertText", "params": {"text": "é"}} + + +@pytest.mark.asyncio +async def test_a_double_click_is_one_move_then_two_presses_and_releases_counted(page): + await page.click_at(10, 20, "left", 2, []) + + assert [(e["params"]["type"], e["params"]["clickCount"]) for e in _events(page)] == [ + ("mouseMoved", 0), ("mousePressed", 1), ("mouseReleased", 1), ("mousePressed", 2), ("mouseReleased", 2)] + + +@pytest.mark.asyncio +async def test_its_capture_is_the_pages_png_and_a_tab_that_paints_nothing_is_white(page): + page.served["page/capture"] = {"png": base64.b64encode(PNG).decode()} + shot = await page.page() + page.served["page/capture"] = {"blank": True} + blank = await page.page() + + assert (shot.width, shot.height, shot.mode, shot.data) == (2, 1, "png", PNG) + assert (blank.width, blank.height, blank.mode) == (1280, 800, "RGB") and set(blank.data) == {0xff} + + +@pytest.mark.asyncio +async def test_its_browser_check_passes_until_the_browser_ends_on_its_own_then_fails_for_good(page, tmp_path): + before = await page.ready_checks() + _ended(tmp_path) + after = await page.ready_checks() + + assert [(c.name, c.ok, c.final) for c in before] == [("browser", True, False)] + assert [(c.name, c.ok, c.final) for c in after] == [("browser", False, True)] + assert "by signal SIGKILL" in after[0].detail and "infrastructure failure" in after[0].detail + + +@pytest.mark.asyncio +async def test_once_the_browser_has_ended_its_calls_say_why_and_send_nothing_though_its_server_answers(page, tmp_path): + page.served["page/capture"] = httpx.Response(503, text="the browser has no tab open") + with pytest.raises(DeviceUnavailable, match="couldn't page/capture: the browser has no tab open"): + await page.page() + _ended(tmp_path) + + with pytest.raises(DeviceUnavailable, match=f"the browser in {tmp_path.name} has failed: .*by signal SIGKILL"): + await page.page() + page.served["page/capture"] = {"blank": True} + sent = len(page.sent) + + with pytest.raises(DeviceUnavailable, match="has failed"): + await page.click_at(1, 2, "left", 1, []) + assert len(page.sent) == sent diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 0b8c34fb6..3dbc49012 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -201,7 +201,8 @@ def _base(image, **changes): "init": "supervisord", "input_channel": "env"}}, **changes}) -def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, install=True, extra=None): +def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, install=True, extra=None, + features=None): folder.mkdir(parents=True, exist_ok=True) (folder / "driver.py").write_text("class Desk: ...\n") (folder / "requirements.txt").write_text(requirements) @@ -211,7 +212,7 @@ def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, i files = FileArtifactUniverse.put(files.id, file_artifacts={**files.get_file_artifacts(), **extra}) supports = supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] platforms = [{"platform": {"os": "linux", "arch": arch, "display": "x11"}, "provides": [], "tools": TOOLS, - "features": {}} for arch in ("amd64", "arm64")] + "features": features or {}} for arch in ("amd64", "arm64")] services = services or [Service(name="driver", command=SERVE)] return ComputerDriverEnv.put( id="@local/~/kit/desk", module="driver.py:Desk", files=files, @@ -352,11 +353,12 @@ async def create_vm(self, **size): def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, name="mail", paths=None, defaults=None, - **changes): + browser=None, **changes): """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of its build settings, ``linux_settings``, that of its settings on Linux, ``name``, its id's last part and its - environment name, ``paths``, where it keeps its data on Linux, and ``defaults``, its env's ``[settings]``.""" + environment name, ``paths``, where it keeps its data on Linux, ``defaults``, its env's ``[settings]``, and + ``browser``, whether it declares itself a website browser.""" folder.mkdir(parents=True, exist_ok=True) (folder / "app.py").write_text("class Mail: ...\n") (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") @@ -369,7 +371,7 @@ def _app(base, driver, folder, compose=True, supports=None, install=None, settin "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], "permissions": [], "settings": settings, "build_settings": build, "skill": None, "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], - "ready_checks": [], "os": {"linux": { + "ready_checks": [], **({"browser": browser} if browser is not None else {}), "os": {"linux": { "launch": launch, "settings": linux_settings, "paths": paths, "install": install or { "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) return ComputerAppEnv.put(**{"id": f"@local/~/kit/{name}", "environment_name": name, "module": "app.py:Mail", diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 2cc8bba9b..72dc6d70a 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -423,6 +423,40 @@ def test_a_driver_that_cant_compose_with_the_base_is_refused(base, docker, tmp_p assert problem in str(refused.value) +@pytest.mark.parametrize("names, problem", [ + ((), "apps: @local/~/kit/desk's screen is a website browser's page, so its computer takes exactly one website " + "browser; it has 0"), + (("first", "second"), "it has 2: @local/~/kit/first, @local/~/kit/second"), +]) +def test_a_driver_whose_screen_is_a_page_takes_exactly_one_website_browser_and_any_other_apps(base, docker, tmp_path, + names, problem): + page = _driver(tmp_path / "driver", features={"screen": "page"}) + mail = _app(None, None, tmp_path / "mail", compose=False) + browsers = [_app(None, None, tmp_path / name, compose=False, name=name, browser=True) for name in names] + + with pytest.raises(ValueError) as refused: + _put(base, page, compose=False, apps=[*browsers, mail]) + alone = _put(base, page, compose=False, apps=[_app(None, None, tmp_path / "one", compose=False, name="one", + browser=True), mail]) + desktop = _put(base, _driver(tmp_path / "desktop-driver"), compose=False, id="@local/~/kit/two", + apps=[_app(None, None, tmp_path / "a", compose=False, name="a", browser=True), + _app(None, None, tmp_path / "b", compose=False, name="b", browser=True)]) + + assert problem in str(refused.value) + assert [app.environment_name for app in alone.apps] == ["one", "mail"] and len(desktop.apps) == 2 + + +def test_an_app_deployed_alone_on_a_driver_whose_screen_is_a_page_is_one_website_browser(base, docker, tmp_path): + page = _driver(tmp_path / "driver", features={"screen": "page"}) + + with pytest.raises(ValueError, match="app: @local/~/kit/desk's screen is a website browser's page, so its " + "computer takes exactly one website browser; it has 0"): + _app(base, page, tmp_path / "mail", compose=False) + browser = _app(base, page, tmp_path / "browser", compose=False, name="browser", browser=True) + + assert browser.declaration.browser is True + + def test_a_base_and_driver_with_no_arch_in_common_and_an_image_that_cant_be_got_are_refused(docker, tmp_path): image = get_artifact_store().put_document(DockerImageArtifact(id="base-image", description="", image_name="local/base:1")) @@ -740,6 +774,27 @@ async def status(url, card, uri, params, timeout, method, headers): assert sent == [{"Authorization": "Bearer t0ken"}] * 2 +@pytest.mark.asyncio +async def test_a_boot_whose_status_says_the_computer_failed_raises_at_once_with_why(monkeypatch): + asked = [] + + async def card(url, timeout): + return {"name": "desk"} + + async def status(url, card, uri, params, timeout, method, headers): + asked.append(method) + return {"ready": False, "failed": "browser: it ended on its own", "checks": [ + {"name": "browser", "ok": False, "detail": "it ended on its own", "final": True}]} + + monkeypatch.setattr(boot.protocol_v1, "get_card", card) + monkeypatch.setattr(boot.protocol_v1, "invoke_extension", status) + monkeypatch.setattr(boot, "_POLL_S", 0) + + with pytest.raises(boot.ComputerFailed, match="the computer at http://desk failed: browser: it ended on its own"): + await boot.ready("http://desk", 5, "t0ken") + assert asked == ["status"] + + def test_a_boot_mints_a_generated_setting_resolves_a_secret_its_record_withholds_both_and_a_secret_is_a_reference( base, monkeypatch): monkeypatch.setenv("DESK_KEY", "s3cret") diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index 7dfd67121..46dcde31f 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -13,7 +13,6 @@ from agent_env.env import bootstrap, published_images from agent_env.env.published_images import ( AGENT_IMAGES, - COMPUTER_IMAGES, DISK, DISK_IMAGES, DISK_TOOLS_IMAGE, @@ -41,7 +40,7 @@ def test_every_shipped_dockerfile_is_published_once_under_a_name_of_its_own(): shipped.append(published_images._EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile") shipped += published_images._COMPUTERS.glob("*/Dockerfile") shipped += DISK.glob("*/Dockerfile") - shipped += [DISK / "boot" / "Dockerfile"] * (len(COMPUTER_IMAGES) - 1) + shipped += [DISK / "boot" / "Dockerfile"] * (len(DISK_IMAGES) - 1) assert sorted(image.dockerfile for image in bootstrap.PUBLISHED_IMAGES) == sorted(shipped) assert len({image.name for image in bootstrap.PUBLISHED_IMAGES}) == len(bootstrap.PUBLISHED_IMAGES) @@ -178,7 +177,8 @@ def test_a_shipped_agents_sources_whose_image_isnt_published_find_none(monkeypat def test_a_disk_forms_tag_is_of_its_boot_recipe_its_bases_sources_and_the_disk_tools_and_it_is_found_by_its_bases( tmp_path, monkeypatch): - (disk,), (base,) = DISK_IMAGES, COMPUTER_IMAGES + (disk,) = DISK_IMAGES + base = dict(disk.args)["ROOTFS"] recipe = BuildContext.of(disk.dockerfile.parent, disk.dockerfile).source_digest(None).removeprefix("sha256:") served = {} monkeypatch.setattr(published_images, "_registry_digest", served.get) diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index 71be6dd6a..f1b7be2ab 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -76,7 +76,7 @@ def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(sc assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", "agentenv-pydantic-ai", - "agentenv-mail-mcp", "agentenv-ubuntu-24.04"] + "agentenv-mail-mcp", "agentenv-ubuntu-24.04", "agentenv-ubuntu-24.04-headless"] def test_plan_missing_lists_the_images_the_registry_lacks_or_refuses_to_show(script, monkeypatch, capsys):