From 1205bc6c1315dc9d9a03c33ff200930faf6f1043 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 16:20:14 -0800 Subject: [PATCH 1/4] feat(computer): a website browser running alone is a computer, on the browser-24.04 base with the page driver chromium deployed alone now runs on browser-24.04, a base with no display, with the page driver, whose screen is its active tab, over browser/v1's new page/viewport, page/capture and page/input. A ready check can be final: while one fails, computer-ready/v1 says the computer failed and why, and boot.ready() raises at once. A website browser runs headless exactly when its computer has no display. A driver's card says its screen, and put refuses a page computer unless it has exactly one website browser, which its AppDeclaration now says it is. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 18 +- .../src/agentenv_protocol/computer/app.py | 3 +- .../src/agentenv_protocol/computer/apps.py | 36 +- .../src/agentenv_protocol/computer/card.py | 2 +- .../src/agentenv_protocol/computer/checks.py | 2 + .../src/agentenv_protocol/computer/driver.py | 5 +- .../agentenv_protocol/computer/extensions.py | 8 +- .../agentenv_protocol/computer/operations.py | 2 + .../agentenv_protocol/computer/platform.py | 5 +- .../src/agentenv_protocol/website_browser.py | 95 ++++-- .../tests/test_computer_driver.py | 18 +- .../tests/test_website_browser.py | 121 +++++-- src/agent_env/artifact/artifacts/computer.py | 4 +- src/agent_env/browsers/README.md | 5 +- .../browsers/envs/chromium/chromium.py | 2 +- src/agent_env/browsers/envs/chromium/env.toml | 4 +- src/agent_env/computers/README.md | 8 + src/agent_env/computers/aliases.toml | 1 + .../artifacts/browser-24.04/Dockerfile | 22 ++ .../artifacts/browser-24.04/artifact.toml | 14 + .../artifacts/browser-24.04/site-proxy.py | 73 ++++ .../artifacts/browser-24.04/start-computer | 8 + .../artifacts/browser-24.04/supervisord.conf | 26 ++ src/agent_env/drivers/README.md | 6 + src/agent_env/drivers/aliases.toml | 1 + src/agent_env/drivers/envs/page/driver.py | 311 ++++++++++++++++++ src/agent_env/drivers/envs/page/env.toml | 7 + .../drivers/envs/page/requirements.txt | 3 + src/agent_env/env/envs/computer/app_env.py | 2 +- src/agent_env/env/envs/computer/env.py | 1 + src/agent_env/env/envs/computer/utils/boot.py | 18 +- src/agent_env/env/envs/computer/utils/gate.py | 12 + .../browser-computer/tasks/chromium.json | 3 + .../computer/browser_computer_test.py | 227 +++++++++++++ tst/integration/computer/conftest.py | 18 +- .../computer/website_browser_test.py | 3 +- tst/unit/drivers/page_driver_test.py | 142 ++++++++ tst/unit/env/envs/_computer.py | 12 +- tst/unit/env/envs/test_computer_env.py | 55 ++++ 39 files changed, 1220 insertions(+), 83 deletions(-) create mode 100644 src/agent_env/computers/artifacts/browser-24.04/Dockerfile create mode 100644 src/agent_env/computers/artifacts/browser-24.04/artifact.toml create mode 100644 src/agent_env/computers/artifacts/browser-24.04/site-proxy.py create mode 100755 src/agent_env/computers/artifacts/browser-24.04/start-computer create mode 100644 src/agent_env/computers/artifacts/browser-24.04/supervisord.conf create mode 100644 src/agent_env/drivers/envs/page/driver.py create mode 100644 src/agent_env/drivers/envs/page/env.toml create mode 100644 src/agent_env/drivers/envs/page/requirements.txt create mode 100644 tst/integration/computer/browser-computer/tasks/chromium.json create mode 100644 tst/integration/computer/browser_computer_test.py create mode 100644 tst/unit/drivers/page_driver_test.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index b6cc8c649..33b09b2fe 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -505,7 +505,8 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope - **The input queue.** Input calls run one at a time in arrival order; one that waits past `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. - **The computer skill.** `computer/v1` declares the seven tools, its params the driver's - features. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base + features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website + browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base class): a SKILL.md written from the computer as it booted (its platform, session user and home, the live screen size, what its input takes, its OS's shortcuts) and ending with the driver class's `notes`, markdown for what its card can't say. A driver whose software ships a skill of @@ -513,7 +514,9 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope computer skill leads with it, the computer's part following. - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a capture, the driver's own `ready_checks()`, and one per app on the computer, asked of the app's - own server (see Computer apps); tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool + own server (see Computer apps); tools refuse `device_unavailable` until they pass once; a check + that can't pass again is `final`, and while one fails the status carries `failed`, saying why, + which `boot.ready()` raises at once rather than waiting out its timeout), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and, with `recording`, `computer-recording/v1` (below). @@ -742,7 +745,7 @@ its card lists each one's card as a child environment, rooted at `/svc//`, requests to the app's server. `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` -env stores, and `python -m agentenv_protocol.computer._declare ` prints it as +env stores (`browser: true` for a website browser), and `python -m agentenv_protocol.computer._declare ` prints it as JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed. ### Website browsers @@ -769,8 +772,8 @@ are the profile's template. The base class adds: `AGENTENV_SITE_PROXY` set, it goes out through that proxy, with WebRTC's non-proxied UDP off. Alone it bypasses nothing, loopback included; in a computer it bypasses loopback (`127.0.0.1`, `[::1]`, `localhost`), so the apps beside it stay reachable while `*.localhost` names still go - through. Alone (`AGENTENV_BROWSER_ALONE=1`) it also takes `--window-size` and `--headless=new`, - except on the open web (`AGENTENV_BROWSER_OPEN_WEB=1`), and refuses to start with no proxy. + through. Alone, on a computer with no display (`XDG_SESSION_TYPE=none`, a browser computer's), it + also takes `--window-size` and `--headless=new`, and refuses to start with no proxy. `declaration()` refuses a subclass whose command carries any of these, or `--remote-allow-origins`, or any other way to open a debugging port. - **Before each launch** it empties its cache, so no launch inherits another's, and writes the deploy's @@ -803,7 +806,10 @@ are the profile's template. The base class adds: launch (with its cause: `boot`, `server_restart`, `launch`, `data/add` or `data/reset`), each stop it made and each exit it didn't, with its exit `code` or the `signal` that ended it, since the computer booted. Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra - failure, its ready check false. + failure, its ready check false. `page/viewport`, `page/capture` and `page/input`, framework-only, are + its active tab's page as a page driver sees it: its size, which headless is its window's; a PNG of it, + or `blank` for a tab that paints nothing within 3 s (1 s with no URL yet); and CDP input calls + (`Input.dispatchMouseEvent`, `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in `~/Downloads`, folders it makes. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index 22877b7b2..5cdef1cb0 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -236,7 +236,7 @@ class AppDeclaration(BaseModel): """What a ComputerAppEnvironment class declares, as a ``computer_app`` env stores it: the platforms it ``supports``, as a driver names them; ``os``, what it declares for each OS type; ``settings`` and ``build_settings``, the JSON schemas of its ``Settings`` and of its ``BuildSettings`` when it declares any; its ``skill``; and the ``data`` - methods, ``install_steps`` and ``ready_checks`` it serves, by name.""" + methods, ``install_steps`` and ``ready_checks`` it serves, by name; and ``browser``, true for a website browser.""" model_config = ConfigDict(extra="forbid") @@ -249,6 +249,7 @@ class AppDeclaration(BaseModel): data: list[str] install_steps: list[InstallStep] ready_checks: list[AppReadyCheck] + browser: Optional[bool] = None @model_validator(mode="before") @classmethod diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index 9de569ed5..3f638d986 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -8,6 +8,7 @@ import asyncio import copy +import json from pathlib import Path from typing import Optional @@ -16,7 +17,7 @@ from .. import client from ..types import RPC_PATH, WELL_KNOWN_PATH, EnvironmentCard from .app import COMPUTER_APP, FAILURE, SOCKET -from .app_runtime import CHECK_TIMEOUT_S, AppReadiness +from .app_runtime import CHECK_TIMEOUT_S, LAUNCHES, AppReadiness from .operations import ReadyCheck #: Where a composed computer installs its apps, a folder each. @@ -54,7 +55,7 @@ async def check(folder: Path) -> ReadyCheck: async def children(folders: list[Path]) -> list[EnvironmentCard]: """Each app in ``folders`` as a child environment of its computer: its server's card, rooted at ``/svc//``, or, while its server isn't answering, its name and route alone.""" - cards = await asyncio.gather(*(_card(folder) for folder in folders)) + cards = await asyncio.gather(*(served_card(folder) for folder in folders)) return [EnvironmentCard.model_validate(child_card(folder.name, card)) if card is not None else EnvironmentCard(name=folder.name, url=f"/svc/{folder.name}{RPC_PATH}") for folder, card in zip(folders, cards)] @@ -85,7 +86,7 @@ def rooted(path: object) -> object: async def owned_paths(folders: list[Path]) -> list[str]: """The paths under the session user's home that the apps in ``folders`` keep their data in, as their cards say.""" - cards = await asyncio.gather(*(_card(folder) for folder in folders)) + cards = await asyncio.gather(*(served_card(folder) for folder in folders)) return [path for card in cards if card is not None for path in client.extension_params(card, COMPUTER_APP).get("paths") or []] @@ -104,7 +105,7 @@ def endpoint(card: dict, method: str) -> str: return client.find_extension_method(card, COMPUTER_APP, method)["endpoint"] -async def _card(folder: Path) -> Optional[dict]: +async def served_card(folder: Path) -> Optional[dict]: """The card the app in ``folder`` serves, or None while its server isn't answering.""" try: async with _client(folder / SOCKET) as http: @@ -113,6 +114,33 @@ async def _card(folder: Path) -> Optional[dict]: return None +def ended_on_its_own(folder: Path) -> Optional[str]: + """Why the app in ``folder`` can't be ready again where nothing launches it but its server: its launch log says it + ended on its own, an infrastructure failure; None while it hasn't.""" + for entry in launch_log(folder): + if entry.get("event") == "exit": + how = (f"by signal {entry['signal']}" if "signal" in entry else + f"with code {entry['code']}" if "code" in entry else "in a way that wasn't recorded") + return (f"it ended on its own at {entry.get('at')}, {how}, and nothing launches it again: an " + "infrastructure failure") + return None + + +def launch_log(folder: Path) -> list[dict]: + """Each entry of the launch log of the app in ``folder`` that's whole, in order.""" + try: + lines = (folder / LAUNCHES).read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + return [] + entries = [] + for line in lines: + try: + entries.append(json.loads(line)) + except ValueError: + continue + return entries + + def _unanswered(folder: Path) -> str: try: return f"its server failed to start: {(folder / FAILURE).read_text()}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py index a8453b7e2..20e46fd21 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -89,7 +89,7 @@ def computer_features(driver: ComputerDriverEnvironment) -> dict: "max_wait_ms": driver.max_wait_ms, "max_drag_points": driver.max_drag_points, "max_duration_ms": driver.max_duration_ms, "max_key_repeat": driver.max_key_repeat, "max_scroll_amount": driver.max_scroll_amount, "delivery": "foreground", "platform": driver.platform.as_dict(), - "supports": list(driver.supports), "default_data_paths": list(DEFAULT_DATA_PATHS), + "supports": list(driver.supports), "default_data_paths": list(DEFAULT_DATA_PATHS), "screen": driver.screen_kind, } diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py index 6d0736eb6..a36266a4c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/checks.py @@ -37,6 +37,8 @@ def check_features(cls: type) -> None: problems.append("keys are W3C key values in their W3C case") except AgentEnvError as e: problems.append(f"keys: {e.message}") + if cls.screen_kind not in ("desktop", "page"): + problems.append(f"screen_kind is desktop or page, got {cls.screen_kind!r}") if ("set_display" in declared(cls)) != bool(cls.display_modes): problems.append("display_modes and @set_display come together") problems += check_recording(cls.recording) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 6177a4aac..a5b8f90b9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -4,7 +4,7 @@ from __future__ import annotations import asyncio -from typing import Any, Callable, ClassVar +from typing import Any, Callable, ClassVar, Literal import httpx from pydantic import BaseModel, ConfigDict @@ -79,6 +79,9 @@ class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironmen recording: ClassVar[Recording | None] = Recording() #: What an agent should know about this driver's computer beyond what its card says, as markdown: the end of its skill. notes: ClassVar[str] = "" + #: What the computer's screen shows, its card's ``screen``: a ``desktop``, or a website browser's ``page``, its + #: active tab. + screen_kind: ClassVar[Literal["desktop", "page"]] = "desktop" tool_access = True #: What the base class keeps for the driver, built by ``__init__``. _computer: ComputerRuntime | None = None diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py index ff12c3b36..a569d094f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/extensions.py @@ -45,8 +45,12 @@ async def _computer_skill(self) -> tuple[str, str, dict[str, str]]: description="Whether the computer is usable now, by real checks; never sends input.") async def _computer_ready_status(self) -> dict: checks = await self._computer.ready_checks() - return {"ready": all(c.ok for c in checks), - "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks]} + status = {"ready": all(c.ok for c in checks), + "checks": [{"name": c.name, "ok": c.ok, "detail": c.detail, **({"final": True} if c.final else {})} + for c in checks]} + if failed := [c for c in checks if c.final and not c.ok]: + status["failed"] = "; ".join(f"{c.name}: {c.detail}" for c in failed) + return status @extension(COMPUTER_DISPLAY, op="get", method="GET", description="Get or set the display's resolution and scale.") async def _computer_display_get(self) -> dict: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index 1e9936ec1..b6ead0141 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -74,6 +74,8 @@ class ReadyCheck: name: str ok: bool detail: str = "" + #: Failed for good: the check can't pass again, so the computer has failed. + final: bool = False @dataclass(frozen=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py index 3f2587e79..241b5dd30 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/platform.py @@ -28,14 +28,15 @@ def as_dict(self) -> dict: def detect(environ: dict | None = None, os_release: Path = Path("/etc/os-release")) -> Platform: """The running platform: the OS, ``/etc/os-release``'s ID and VERSION_ID on Linux, the machine, and the - display from ``XDG_SESSION_TYPE``, else ``DISPLAY`` (x11) or ``WAYLAND_DISPLAY`` (wayland).""" + display from ``XDG_SESSION_TYPE``, ``none`` for a computer with no display, else ``DISPLAY`` (x11) or + ``WAYLAND_DISPLAY`` (wayland).""" environ = os.environ if environ is None else environ name = _OS.get(sys.platform, sys.platform) release = _os_release(os_release) if name == "linux" else {} display = None if name == "linux": session = (environ.get("XDG_SESSION_TYPE") or "").strip().lower() - if session in ("x11", "wayland"): + if session in ("x11", "wayland", "none"): display = session elif environ.get("DISPLAY"): display = "x11" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index a5a435caf..00159dbd7 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -2,8 +2,9 @@ its build installs and starts; this class adds what agentenv owns, the same in both: its flags after the subclass's command, the unattended-use policies, a CDP connection over the browser's debugging pipe, the settings every browser takes, a fresh cache and the deploy's prompt answers at every launch, another window for a launch while it runs, its -skill and ``browser/v1`` on its card. Whether it runs alone, on the open web and through which site proxy come from -variables agentenv sets, never from its settings.""" +skill and ``browser/v1`` on its card. It runs alone, headless, on a computer with no display, a browser computer's, +whose screen is its page; which site proxy it goes through comes from a variable agentenv sets, never from its +settings.""" from __future__ import annotations import asyncio @@ -23,11 +24,11 @@ from .agent_env_environment import agent_skill, extension from .computer import AppDeclaration, ComputerAppEnvironment, Launch, install_step, ready_check +from .computer import app as computer_app +from .computer.errors import DeviceUnavailable, InvalidArgument logger = logging.getLogger(__name__) -ALONE_ENV = "AGENTENV_BROWSER_ALONE" -OPEN_WEB_ENV = "AGENTENV_BROWSER_OPEN_WEB" SITE_PROXY_ENV = "AGENTENV_SITE_PROXY" BROWSER = "urn:agentenv:browser/v1" #: The flags agentenv sets, by prefix, which a subclass's command may not carry: among them every way to open a debugging @@ -51,6 +52,9 @@ AGENT = "agent" #: The targets agentenv's connection follows. PAGES = [{"type": "page"}] +#: The CDP input a computer's page driver may send to the active tab, and how long its screenshot may take. +PAGE_INPUT = frozenset({"Input.dispatchMouseEvent", "Input.dispatchKeyEvent", "Input.insertText"}) +CAPTURE_TIMEOUT_S, UNNAVIGATED_CAPTURE_TIMEOUT_S = 3.0, 1.0 #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" @@ -187,8 +191,8 @@ class BrowserControl: ``dialogs`` says to: always leaving a page that asks before it's left, so no answer of agentenv's blocks a navigation.""" - def __init__(self, devtools: DevToolsPipe, dialogs: str) -> None: - self.devtools, self.dialogs = devtools, dialogs + def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tuple[int, int]] = None) -> None: + self.devtools, self.dialogs, self.viewport = devtools, dialogs, viewport #: When this record starts: a server that restarts starts another. self.since = _now() self.tabs: dict[str, dict] = {} @@ -268,6 +272,36 @@ async def record(self) -> dict: "closed_tabs": list(self.closed_tabs), "downloads": list(self.downloads.values()), "dialogs": list(self.dialog_log)} + async def active_session(self) -> str: + """The session of the active tab, else of the first open one; DeviceUnavailable with no tab open.""" + active = await self.active_tab() or next(iter(self.tabs), None) + session = next((session for session, target in self._sessions.items() if target == active), None) + if session is None: + raise DeviceUnavailable("the browser has no tab open") + return session + + async def capture(self) -> Optional[str]: + """The active tab's page as it shows now, a PNG, base64; None for a tab that paints nothing within + ``CAPTURE_TIMEOUT_S``, which a desktop browser shows blank, or sooner for a tab with no URL yet, such as a new + tab whose navigation the browser blocked, which may never paint.""" + session = await self.active_session() + navigated = self.tabs.get(self._sessions[session], {}).get("url") + try: + shot = await self.devtools.call("Page.captureScreenshot", {"format": "png", "optimizeForSpeed": True}, + session_id=session, + timeout_s=CAPTURE_TIMEOUT_S if navigated else UNNAVIGATED_CAPTURE_TIMEOUT_S) + except asyncio.TimeoutError: + return None + return shot["data"] + + async def dispatch(self, events: list[dict]) -> None: + """Each of ``events``, a CDP input call, sent to the active tab in order; InvalidArgument for any other call.""" + if refused := [event.get("method") for event in events if event.get("method") not in PAGE_INPUT]: + raise InvalidArgument(f"page input takes {', '.join(sorted(PAGE_INPUT))}, not {refused}") + session = await self.active_session() + for event in events: + await self.devtools.call(event["method"], event.get("params") or {}, session_id=session) + async def _refresh(self, target: Optional[str] = None) -> None: """The URL and title of the tab ``target``, or of every open tab, as the browser has them now: it sends no event when a page's title changes.""" @@ -308,6 +342,11 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N if params["targetInfo"].get("type") == "page": self._sessions[params["sessionId"]] = params["targetInfo"]["targetId"] self._spawn(self.devtools.call("Page.enable", session_id=params["sessionId"])) + if self.viewport is not None: + width, height = self.viewport + self._spawn(self.devtools.call("Emulation.setDeviceMetricsOverride", { + "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, + session_id=params["sessionId"])) # A page attached as it opens, a popup say, waits for this before it navigates. self._spawn(self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=params["sessionId"])) @@ -433,16 +472,19 @@ def __init__(self) -> None: for folder in (UPLOADS, DOWNLOADS): (Path.home() / folder).mkdir(exist_ok=True) #: agentenv's control of it over that connection, and the record it keeps. - self.control = BrowserControl(self.devtools, self.settings.dialogs) + self.control = BrowserControl(self.devtools, self.settings.dialogs, + (self.settings.window.width, self.settings.window.height) if _alone() + else None) self.devtools.on_close(self._pipe_closed) @classmethod def declaration(cls) -> AppDeclaration: - """What the class declares; ValueError for a launch whose command carries a flag agentenv owns.""" + """What the class declares, a website browser; ValueError for a launch whose command carries a flag agentenv + owns.""" for declared in cls._installs.values(): if declared.launch is not None: _refuse_owned(cls.__name__, declared.launch) - return super().declaration() + return super().declaration().model_copy(update={"browser": True}) def profile_dir(self) -> Path: return Path.home() / self.profile @@ -469,7 +511,7 @@ def agentenv_flags(self) -> list[str]: "names none") if alone: flags.append(f"--window-size={settings.window.width},{settings.window.height}") - if _headless(): + if alone: flags.append("--headless=new") return flags @@ -491,7 +533,7 @@ def _on_desktop(self) -> bool: def _command_line_tabs(self) -> list[str]: """The start tabs its command line opens: headless Chromium takes one, and the rest open over its pipe.""" - return self.settings.start_urls[:1] if _headless() else self.settings.start_urls + return self.settings.start_urls[:1] if _alone() else self.settings.start_urls async def _launch_fds(self) -> dict[int, int]: """Before each launch, an empty cache and the deploy's prompt answers in its profile; and a new debugging pipe: @@ -568,10 +610,27 @@ async def devtools_answers(self) -> bool: "A website browser agentenv runs: its build, the clock its pages see and, alone, its window. record returns " "what agentenv's own connection to it saw: its open tabs, each with whether it's active, the tabs closed and " "who closed them, its downloads with their size and sha256, its JavaScript dialogs and who answered each, and " - "each launch, stop and exit of it since the computer booted, with its cause.")) + "each launch, stop and exit of it since the computer booted, with its cause. page/viewport, page/capture and " + "page/input are its active tab's page as a computer's page driver sees it: its size in pixels, which headless " + "is its window's, a PNG of it, and CDP input calls sent to it in order.")) async def browser_record(self) -> dict: return {**await self.control.record(), "launches": self._computer_app.launches()} + @extension(BROWSER, op="page/viewport", method="GET") + async def browser_page_viewport(self) -> dict: + window = self.settings.window + return {"width": window.width, "height": window.height} + + @extension(BROWSER, op="page/capture", method="GET") + async def browser_page_capture(self) -> dict: + png = await self.control.capture() + return {"png": png} if png is not None else {"blank": True} + + @extension(BROWSER, op="page/input") + async def browser_page_input(self, events: list[dict]) -> dict: + await self.control.dispatch(events) + return {} + def extension_params(self, uri: str) -> dict: if uri == BROWSER: params: dict[str, Any] = {"chromium": self.version, "clock": "real"} @@ -586,11 +645,8 @@ def browser_skill(self) -> tuple[str, str]: def _alone() -> bool: - return os.environ.get(ALONE_ENV) == "1" - - -def _open_web() -> bool: - return os.environ.get(OPEN_WEB_ENV) == "1" + """Whether it runs on a computer with no display, a browser computer's, alone and headless.""" + return computer_app.detect().display == "none" def _shm_bytes(path: str = "/dev/shm") -> int: @@ -601,11 +657,6 @@ def _shm_bytes(path: str = "/dev/shm") -> int: return stats.f_frsize * stats.f_blocks -def _headless() -> bool: - """Alone it runs headless, but headful on Xvfb for the open web, whose sites block headless browsers.""" - return _alone() and not _open_web() - - def _refuse_owned(name: str, launch: Launch) -> None: if owned := [arg for arg in launch.command if arg.startswith(OWNED_FLAGS)]: raise ValueError(f"{name}'s launch sets {', '.join(owned)}, which agentenv sets for every website browser") diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index c0196ffad..260bbe4d0 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -115,6 +115,7 @@ def test_unknown_predicates_are_refused(): ({"keys": ("enter",)}, "keys are W3C key values in their W3C case"), ({"keys": ("Return",)}, "keys: 'Return' is not a W3C key value"), ({"display_modes": ((10, 10),)}, "display_modes and @set_display come together"), + ({"screen_kind": "tablet"}, "screen_kind is desktop or page, got 'tablet'"), ]) def test_features_are_checked_when_the_class_is_defined(features, error): with pytest.raises(TypeError, match=error): @@ -169,7 +170,7 @@ async def test_the_card_declares_the_contract_the_driver_implements(): computer = extensions["urn:agentenv:computer/v1"]["params"] assert computer["buttons"] == ["left", "right", "middle"] and computer["max_click_count"] == 3 assert computer["keys"] == "all" and computer["platform"] == LINUX.as_dict() and computer["delivery"] == "foreground" - assert computer["default_data_paths"] == ["Desktop", "Documents", "Downloads"] + assert computer["default_data_paths"] == ["Desktop", "Documents", "Downloads"] and computer["screen"] == "desktop" assert "methods" not in computer and extensions["urn:agentenv:skill/v1"]["params"]["methods"]["get"]["method"] == "GET" assert extensions["urn:agentenv:computer-pointer/v1"]["params"]["buttons"] == ["left", "right", "middle"] assert set(extensions["urn:agentenv:computer-keyboard/v1"]["params"]["methods"]) == {"get", "set"} @@ -294,6 +295,21 @@ async def ready_checks(self): async with http_client(Raising()) as http: raised = (await http.get("/agentenv/ext/computer-ready/status")).json() assert raised["checks"][-1] == {"name": "driver", "ok": False, "detail": "RuntimeError: probe crashed"} + assert "failed" not in before and "failed" not in raised + + +@pytest.mark.asyncio +async def test_a_final_check_that_fails_has_the_status_say_the_computer_failed_and_why(): + class Ended(FakeDriver): + async def ready_checks(self): + return [ReadyCheck("browser", False, "it ended on its own", final=True), + ReadyCheck("session", True, "logged in", final=True)] + + async with http_client(Ended()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + assert status["ready"] is False and status["failed"] == "browser: it ended on its own" + assert status["checks"][-2:] == [{"name": "browser", "ok": False, "detail": "it ended on its own", "final": True}, + {"name": "session", "ok": True, "detail": "logged in", "final": True}] @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index c4ce5bbe8..3a7b7c8c4 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -19,10 +19,10 @@ from agentenv_protocol.computer import APP_SETTINGS_ENV, ComputerAppEnvironment, Launch, Ready, app, linux, session from agentenv_protocol import website_browser from agentenv_protocol.computer import app_runtime +from agentenv_protocol.computer.errors import DeviceUnavailable, InvalidArgument from agentenv_protocol.computer.platform import Platform -from agentenv_protocol.website_browser import (ALONE_BYPASS, ALONE_ENV, BROWSER, COMPUTER_BYPASS, OPEN_WEB_ENV, - POLICIES, SITE_PROXY_ENV, SKILL, BrowserControl, DevToolsError, - WebsiteBrowserEnvironment) +from agentenv_protocol.website_browser import (ALONE_BYPASS, BROWSER, COMPUTER_BYPASS, POLICIES, SITE_PROXY_ENV, SKILL, + BrowserControl, DevToolsError, WebsiteBrowserEnvironment) PROXY = "http://127.0.0.1:3128" OWNED = ["--user-data-dir", "--disk-cache-dir", "--remote-debugging-pipe", "--no-first-run", @@ -80,7 +80,7 @@ def browser_home(monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path / "home")) (tmp_path / "home").mkdir() monkeypatch.setenv("ENVIRONMENT_NAME", "chromium") - for name in (ALONE_ENV, OPEN_WEB_ENV, SITE_PROXY_ENV, APP_SETTINGS_ENV): + for name in (SITE_PROXY_ENV, APP_SETTINGS_ENV): monkeypatch.delenv(name, raising=False) return tmp_path / "home" @@ -89,6 +89,11 @@ def _settings(monkeypatch, **values) -> None: monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"chromium": values})) +def _alone(monkeypatch) -> None: + """On a computer with no display, a browser computer's.""" + monkeypatch.setattr(app, "detect", lambda: Platform(os="linux", display="none")) + + def test_in_a_computer_its_flags_follow_its_command_then_its_start_tabs_and_its_window_is_waited_for(browser_home): launch = Browser().launch @@ -110,7 +115,7 @@ def test_in_a_computer_with_a_site_proxy_only_loopback_bypasses_it(monkeypatch): def test_alone_it_goes_through_its_proxy_headless_at_its_window_and_its_launch_waits_on_no_window(monkeypatch): - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) _settings(monkeypatch, window={"width": 1024, "height": 768}) @@ -130,7 +135,7 @@ def test_alone_its_launch_keeps_the_port_and_command_it_waits_on_and_their_timeo class Waiting(WebsiteBrowserEnvironment): pass - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) waiting = Waiting() @@ -140,18 +145,8 @@ class Waiting(WebsiteBrowserEnvironment): assert waiting._computer_app.timeout_s == 45 -def test_alone_on_the_open_web_it_runs_headful(monkeypatch): - monkeypatch.setenv(ALONE_ENV, "1") - monkeypatch.setenv(OPEN_WEB_ENV, "1") - monkeypatch.setenv(SITE_PROXY_ENV, PROXY) - - command = Browser().launch.command - - assert "--window-size=1280,800" in command and not any(flag.startswith("--headless") for flag in command) - - def test_alone_it_refuses_to_launch_without_a_site_proxy(monkeypatch): - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) with pytest.raises(ValueError, match=SITE_PROXY_ENV): Browser() @@ -215,7 +210,7 @@ def test_its_settings_give_its_start_tabs_language_and_time_zone_and_no_start_ur @pytest.mark.asyncio async def test_alone_its_server_launches_it_at_once_in_its_environment_with_no_display_wait_or_launcher( monkeypatch, tmp_path): - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) monkeypatch.delenv("DISPLAY", raising=False) monkeypatch.chdir(tmp_path) @@ -314,7 +309,7 @@ async def test_a_launch_while_it_runs_opens_a_window_over_its_pipe_and_starts_no async def test_headless_it_opens_one_start_tab_from_its_command_line_and_the_rest_over_its_pipe(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) _settings(monkeypatch, start_urls=["https://slack.com", "https://linear.app", "https://github.com"]) browser = FakeBrowser() @@ -400,7 +395,7 @@ def test_its_policies_are_agentenvs_with_its_own_over_them_where_its_build_reads def test_its_card_declares_browser_v1_with_its_build_and_clock_and_alone_its_window(monkeypatch): in_computer = Browser().extension_params(BROWSER) - monkeypatch.setenv(ALONE_ENV, "1") + _alone(monkeypatch) monkeypatch.setenv(SITE_PROXY_ENV, PROXY) alone = Browser().extension_params(BROWSER) @@ -503,6 +498,92 @@ async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to assert ("Page.enable", {}, "s-popup") in devtools.calls assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls + assert not any(method == "Emulation.setDeviceMetricsOverride" for method, _, _ in devtools.calls) + + +@pytest.mark.asyncio +async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_runs(tmp_path): + devtools = StubDevTools() + control = BrowserControl(devtools, "agent", (1024, 768)) + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0) + + sent = [method for method, _, session in devtools.calls if session == "s-popup"] + assert sent == ["Page.enable", "Emulation.setDeviceMetricsOverride", "Runtime.runIfWaitingForDebugger"] + assert ("Emulation.setDeviceMetricsOverride", {"width": 1024, "height": 768, "deviceScaleFactor": 1, + "mobile": False}, "s-popup") in devtools.calls + + +class Painting(StubDevTools): + """A browser whose every tab has focus and paints, but those whose sessions are in ``blank``, which never answer a + capture; it records each capture's timeout.""" + + def __init__(self, blank=()): + super().__init__({"Runtime.evaluate": lambda params, session: {"result": {"value": ["visible", True]}}}) + self.blank, self.timeouts = set(blank), [] + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method != "Page.captureScreenshot": + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + self.timeouts.append(timeout_s) + if session_id in self.blank: + raise asyncio.TimeoutError + return {"data": f"png of {session_id}"} + + +async def _one_tab(devtools, url) -> BrowserControl: + control = BrowserControl(devtools, "agent") + await control.attach() + devtools.send("Target.targetCreated", _page("a", url)) + devtools.send("Target.attachedToTarget", {"sessionId": "s-a", **_page("a", url)}) + await asyncio.sleep(0) + return control + + +@pytest.mark.asyncio +@pytest.mark.parametrize("url, blank, png, timeout", [ + ("https://mail.test/", (), "png of s-a", 3.0), + ("https://mail.test/", ("s-a",), None, 3.0), + ("", ("s-a",), None, 1.0), +]) +async def test_its_capture_is_the_active_tabs_page_or_none_for_one_that_paints_nothing_sooner_with_no_url( + url, blank, png, timeout): + devtools = Painting(blank) + control = await _one_tab(devtools, url) + + assert await control.capture() == png and devtools.timeouts == [timeout] + + +@pytest.mark.asyncio +async def test_its_page_input_goes_to_the_active_tab_in_order_and_takes_only_input_calls(): + devtools = Painting() + control = await _one_tab(devtools, "https://mail.test/") + events = [{"method": "Input.dispatchMouseEvent", "params": {"type": "mousePressed", "x": 1, "y": 2}}, + {"method": "Input.insertText", "params": {"text": "hi"}}] + + await control.dispatch(events) + with pytest.raises(InvalidArgument, match="Page.navigate"): + await control.dispatch([*events, {"method": "Page.navigate", "params": {"url": "https://elsewhere.test/"}}]) + + sent = [(method, params, session) for method, params, session in devtools.calls if method.startswith("Input.")] + assert sent == [("Input.dispatchMouseEvent", {"type": "mousePressed", "x": 1, "y": 2}, "s-a"), + ("Input.insertText", {"text": "hi"}, "s-a")] + + +@pytest.mark.asyncio +async def test_with_no_tab_open_its_capture_and_input_are_unavailable(): + control = BrowserControl(Painting(), "agent") + await control.attach() + + with pytest.raises(DeviceUnavailable, match="no tab open"): + await control.capture() + with pytest.raises(DeviceUnavailable, match="no tab open"): + await control.dispatch([]) + + +def test_its_declaration_says_it_is_a_website_browser_and_another_apps_says_nothing(): + assert Browser.declaration().browser is True and Mail.declaration().browser is None @pytest.mark.asyncio diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index 61d5cc773..52ed716f4 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -44,14 +44,14 @@ class LinuxOS(BaseModel): """A Linux base: its distribution and version (``rolling`` for one with no versions, such as Arch), and the display - server its desktop runs.""" + server its desktop runs, ``none`` for one with no desktop, whose driver's screen is something else, a page say.""" model_config = ConfigDict(extra="forbid") type: Literal["linux"] = "linux" distribution: str = Field(pattern=r"^[a-z][a-z0-9-]*$", description="os-release's ID: ubuntu, debian, fedora, arch") version: str = Field(min_length=1, description="os-release's VERSION_ID, or rolling") - display: Literal["x11", "wayland"] + display: Literal["x11", "wayland", "none"] build: str | None = None diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 43bb3783d..186a16cf0 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -1,7 +1,8 @@ The website browsers that come with agent-env, today `chromium`, a `computer_app` env a computer env lists in its -`apps`. +`apps`. Deployed alone, `chromium` is a browser computer: the `browser-24.04` base, with no display, and the `page` +driver, whose screen is the browser's active tab, headless. -`envs/chromium/` is Chromium for an X11 desktop on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the +`envs/chromium/` is Chromium for an X11 desktop or a computer with no display on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists it is put and unpacked into `/opt/agentenv/apps/chromium/` by its install step. It's a website browser (`agentenv_protocol.website_browser.WebsiteBrowserEnvironment`): agentenv launches it on a debugging pipe with its own diff --git a/src/agent_env/browsers/envs/chromium/chromium.py b/src/agent_env/browsers/envs/chromium/chromium.py index 4fb265ae8..33331d9c5 100644 --- a/src/agent_env/browsers/envs/chromium/chromium.py +++ b/src/agent_env/browsers/envs/chromium/chromium.py @@ -25,7 +25,7 @@ PROFILE = ".config/chromium" -@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11", "none"]}, "arch": ["amd64", "arm64"]}]) @linux(packages=Packages(apt=PACKAGES), downloads=DOWNLOADS, paths=[PROFILE], launch=Launch(at="boot", command=["chromium/chrome/chrome", "--no-sandbox", f"--class={WINDOW_CLASS}"], window={"class": WINDOW_CLASS}, ready=Ready(window=True))) diff --git a/src/agent_env/browsers/envs/chromium/env.toml b/src/agent_env/browsers/envs/chromium/env.toml index 8a3a20ae1..8e8054f0d 100644 --- a/src/agent_env/browsers/envs/chromium/env.toml +++ b/src/agent_env/browsers/envs/chromium/env.toml @@ -1,7 +1,7 @@ type = "computer_app" module = "chromium.py:Chromium" -computer = "ubuntu-24.04" -driver = "reference" +computer = "browser-24.04" +driver = "page" [settings] start_urls = ["about:blank"] diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 8ebce59df..f1fc1608f 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -8,6 +8,14 @@ container form's image is built from the folder's `Dockerfile`, and a release pu building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04`. +`artifacts/browser-24.04/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop +(`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the +browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the +session user, among them a stand-in site proxy on `127.0.0.1:3128`, the browser's only route out, which forwards every +request to wherever it names, refusing nothing: the sandbox's network policy is all that limits it until the shared +site proxy replaces it. It has a container form only. Name it from another bundle as `browser-24.04`, or by its id, +`@local/agentenv-framework/computers/browser-24.04`. + It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, and an ext4 root filesystem that is the image's own, with `/etc/agentenv/vm/image.env`, its entrypoint, command, diff --git a/src/agent_env/computers/aliases.toml b/src/agent_env/computers/aliases.toml index c33416836..f8c608e16 100644 --- a/src/agent_env/computers/aliases.toml +++ b/src/agent_env/computers/aliases.toml @@ -1,3 +1,4 @@ # Ids a task can name these computers by; a store's artifact of the id wins over an alias. [artifacts] "ubuntu-24.04" = ["ubuntu-24.04"] +"browser-24.04" = ["browser-24.04"] diff --git a/src/agent_env/computers/artifacts/browser-24.04/Dockerfile b/src/agent_env/computers/artifacts/browser-24.04/Dockerfile new file mode 100644 index 000000000..5a1b01c6e --- /dev/null +++ b/src/agent_env/computers/artifacts/browser-24.04/Dockerfile @@ -0,0 +1,22 @@ +# The browser-24.04 base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), for one +# website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord runs +# every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. Composing +# a driver or an app onto it adds its services as programs in /etc/supervisor/conf.d. +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + supervisor tini python3 python3-venv fonts-dejavu-core tzdata ca-certificates procps \ + && rm -rf /var/lib/apt/lists/* \ + && userdel --remove ubuntu \ + && useradd --create-home --uid 1000 --shell /bin/bash user \ + && runuser -u user -- mkdir -p /home/user/Desktop /home/user/Documents /home/user/Downloads \ + && mkdir -p /etc/supervisor/conf.d /var/log/agentenv /run/agentenv \ + && chown user:user /var/log/agentenv /run/agentenv +COPY supervisord.conf /etc/supervisor/supervisord.conf +COPY start-computer /usr/local/bin/start-computer +COPY site-proxy.py /usr/local/bin/agentenv-site-proxy +USER user +WORKDIR /home/user +ENV LANG=C.UTF-8 XDG_SESSION_TYPE=none AGENTENV_SITE_PROXY=http://127.0.0.1:3128 +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["/usr/local/bin/start-computer"] diff --git a/src/agent_env/computers/artifacts/browser-24.04/artifact.toml b/src/agent_env/computers/artifacts/browser-24.04/artifact.toml new file mode 100644 index 000000000..dfcd4d670 --- /dev/null +++ b/src/agent_env/computers/artifacts/browser-24.04/artifact.toml @@ -0,0 +1,14 @@ +type = "computer" +description = "Ubuntu 24.04 with no display, for one website browser that runs headless and a page driver whose screen is its page; run by supervisord as the session user" +arch = ["amd64", "arm64"] +os = { type = "linux", distribution = "ubuntu", version = "24.04", display = "none" } +session = { user = "user" } + +[forms.container] +image = { dockerfile = "Dockerfile" } +init = "supervisord" +input_channel = "env" + +[config.timezone] +default = "UTC" +phase = "deploy" diff --git a/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py b/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py new file mode 100644 index 000000000..0f448fed4 --- /dev/null +++ b/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py @@ -0,0 +1,73 @@ +"""A stand-in for the shared site proxy in this base: it forwards every request the browser sends it, absolute-form +HTTP and CONNECT tunnels alike, to wherever it names, refusing nothing, on 127.0.0.1:3128.""" +import http.server +import select +import socket +import socketserver +import urllib.parse + +PORT = 3128 +IDLE_S = 60 +HOP = {"connection", "keep-alive", "proxy-connection", "proxy-authorization"} + + +class Proxy(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_CONNECT(self): + host, _, port = self.path.rpartition(":") + try: + upstream = socket.create_connection((host, int(port)), timeout=10) + except (OSError, ValueError): + self.send_error(502) + return + self.send_response(200, "Connection established") + self.end_headers() + self.close_connection = True + _pipe(self.connection, upstream) + + def forward(self): + url = urllib.parse.urlsplit(self.path) + try: + upstream = socket.create_connection((url.hostname, url.port or 80), timeout=10) + except (OSError, TypeError): + self.send_error(502) + return + target = (url.path or "/") + (f"?{url.query}" if url.query else "") + headers = "".join(f"{k}: {v}\r\n" for k, v in self.headers.items() if k.lower() not in HOP) + upstream.sendall(f"{self.command} {target} HTTP/1.1\r\n{headers}Connection: close\r\n\r\n".encode("latin-1")) + if length := int(self.headers.get("Content-Length") or 0): + upstream.sendall(self.rfile.read(length)) + self.close_connection = True + _pipe(self.connection, upstream) + + do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_HEAD = do_OPTIONS = forward + + def log_message(self, *args): + pass + + +def _pipe(a: socket.socket, b: socket.socket) -> None: + try: + while True: + ready, _, _ = select.select([a, b], [], [], IDLE_S) + if not ready: + return + for s in ready: + data = s.recv(65536) + if not data: + return + (b if s is a else a).sendall(data) + except OSError: + return + finally: + b.close() + + +class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +if __name__ == "__main__": + Server(("127.0.0.1", PORT), Proxy).serve_forever() diff --git a/src/agent_env/computers/artifacts/browser-24.04/start-computer b/src/agent_env/computers/artifacts/browser-24.04/start-computer new file mode 100755 index 000000000..125e1d88c --- /dev/null +++ b/src/agent_env/computers/artifacts/browser-24.04/start-computer @@ -0,0 +1,8 @@ +#!/bin/sh +# The base's start: the time zone a deploy passes as AGENTENV_SETTING_TIMEZONE applied for every program, then +# supervisord. +set -eu +timezone="${AGENTENV_SETTING_TIMEZONE:-UTC}" +[ -f "/usr/share/zoneinfo/$timezone" ] || { echo "timezone '$timezone' isn't in /usr/share/zoneinfo" >&2; exit 2; } +export TZ="$timezone" +exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/src/agent_env/computers/artifacts/browser-24.04/supervisord.conf b/src/agent_env/computers/artifacts/browser-24.04/supervisord.conf new file mode 100644 index 000000000..da2860323 --- /dev/null +++ b/src/agent_env/computers/artifacts/browser-24.04/supervisord.conf @@ -0,0 +1,26 @@ +; The base computer's own program, the site proxy, run as the session user. A composed driver or app adds its own in +; conf.d. +[supervisord] +nodaemon=true +logfile=/var/log/agentenv/supervisord.log +pidfile=/run/agentenv/supervisord.pid +childlogdir=/var/log/agentenv + +[unix_http_server] +file=/run/agentenv/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/agentenv/supervisor.sock + +[program:site-proxy] +command=/usr/bin/python3 /usr/local/bin/agentenv-site-proxy +priority=10 +autorestart=true +redirect_stderr=true +stdout_logfile=/var/log/agentenv/site-proxy.log + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 893bcc807..c059cf8e5 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -7,6 +7,12 @@ pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer `x11grab` through PyAV at up to 30 fps. Name it from another bundle as `reference`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/reference`. +`envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `browser-24.04`: +its screen is the computer's one website browser's active tab, its `screen` feature `page`. A screenshot is the page, +and input reaches it as CDP input events, both through `browser/v1`'s `page/*` operations on the browser's own server; +a tab that paints nothing shows white. The browser ending on its own fails the computer for good: its ready check is +`final`, and its tools answer with why. Name it as `page`, or by its id, `@local/agentenv-framework/drivers/page`. + A run that names it writes it: its files, what its class declares for each platform it supports, read without `mss`, `python-xlib` or PyAV installed, and how a computer installs it. Its `env.toml`'s `[install.linux]` has a computer env's composer put the folder in `/opt/agentenv/drivers/reference/`, install `requirements.txt`, resolved and diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index 43b2f3510..daa0d6144 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -2,3 +2,4 @@ [envs] reference = ["reference"] cua-driver = ["cua-driver"] +page = ["page"] diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py new file mode 100644 index 000000000..d331083b2 --- /dev/null +++ b/src/agent_env/drivers/envs/page/driver.py @@ -0,0 +1,311 @@ +"""The page driver, for a computer with no display whose one website browser runs headless: its screen is the browser's +active tab. A screenshot is the page, and input reaches the page as CDP input events, both through the browser's own +server on its socket (``browser/v1``'s ``page/*`` operations), since only that server holds the browser's debugging pipe. +The browser ending on its own is an infrastructure failure, since nothing here launches it again. Needs +``agentenv-framework-protocol[computer]`` where it runs.""" +from __future__ import annotations + +import asyncio +import base64 +import math +import string +from pathlib import Path +from typing import Any, Optional + +import httpx + +from agentenv_protocol import client +from agentenv_protocol.computer import ( + ComputerDriverEnvironment, + DeviceUnavailable, + Display, + Image, + ReadyCheck, + Recording, + Unsupported, + capture, + click, + computer_driver, + display, + drag, + key_down, + key_up, + pointer_down, + pointer_move, + pointer_position, + pointer_up, + press, + scroll, + type_text, +) +from agentenv_protocol.computer import apps +from agentenv_protocol.computer.app import SOCKET +from agentenv_protocol.website_browser import BROWSER + +BUTTONS = {"left": 1, "right": 2, "middle": 4, "back": 8, "forward": 16} +MODIFIERS = {"Alt": 1, "Control": 2, "Meta": 4, "Shift": 8} +#: Keys that aren't characters: each one's DOM code, its Windows virtual key code, and the text a press of it types. +NAMED = { + "Enter": ("Enter", 13, "\r"), "Tab": ("Tab", 9, ""), "Escape": ("Escape", 27, ""), + "Backspace": ("Backspace", 8, ""), "Delete": ("Delete", 46, ""), "Insert": ("Insert", 45, ""), + "ArrowUp": ("ArrowUp", 38, ""), "ArrowDown": ("ArrowDown", 40, ""), "ArrowLeft": ("ArrowLeft", 37, ""), + "ArrowRight": ("ArrowRight", 39, ""), "Home": ("Home", 36, ""), "End": ("End", 35, ""), + "PageUp": ("PageUp", 33, ""), "PageDown": ("PageDown", 34, ""), "Control": ("ControlLeft", 17, ""), + "Shift": ("ShiftLeft", 16, ""), "Alt": ("AltLeft", 18, ""), "Meta": ("MetaLeft", 91, ""), + " ": ("Space", 32, " "), **{f"F{n}": (f"F{n}", 111 + n, "") for n in range(1, 13)}, +} +#: The US layout's punctuation: each key's DOM code and virtual key code, and the character Shift makes of it. +PUNCTUATION = {"-": ("Minus", 189, "_"), "=": ("Equal", 187, "+"), "[": ("BracketLeft", 219, "{"), + "]": ("BracketRight", 221, "}"), "\\": ("Backslash", 220, "|"), ";": ("Semicolon", 186, ":"), + "'": ("Quote", 222, '"'), ",": ("Comma", 188, "<"), ".": ("Period", 190, ">"), + "/": ("Slash", 191, "?"), "`": ("Backquote", 192, "~")} +SHIFTED_DIGITS = dict(zip(")!@#$%^&*(", string.digits)) +SHIFTED = {**{shifted: key for key, (_, _, shifted) in PUNCTUATION.items()}, **SHIFTED_DIGITS} +TYPED = {"\n": "Enter", "\t": "Tab"} +#: How far one scroll step moves, and how far apart a drag's pointer moves are, in pixels. +WHEEL_PX = 100 +DRAG_STEP_PX = 20 +#: A pixel of a tab with nothing in it. +BLANK = b"\xff\xff\xff" +#: How long one call to the browser's server may take. +CALL_TIMEOUT_S = 30.0 + + +@computer_driver(name="page", supports=[{"os": {"type": "linux", "display": ["none"]}, "arch": ["amd64", "arm64"]}]) +class PageDriver(ComputerDriverEnvironment): + buttons = ("left", "right", "middle", "back", "forward") + max_click_count = 3 + max_drag_points = 50 + keys = sorted({*NAMED, *string.ascii_letters, *string.digits, *PUNCTUATION, *SHIFTED}) + recording = Recording(max_fps=10) + screen_kind = "page" + notes = ("This computer is one web page: the browser's active tab, with no address bar, tab strip or desktop " + "around it. Its screen is the page's viewport.") + + def __init__(self) -> None: + super().__init__() + self._browser: Optional[tuple[Path, dict]] = None + self._pointer = (0.0, 0.0) + self._buttons = 0 + self._held: list[str] = [] + + @display + async def viewport(self) -> Display: + size = await self._call("page/viewport") + return Display(size["width"], size["height"], 1.0) + + @capture + async def page(self) -> Image: + """The page, or a blank one where the browser shows nothing, as a desktop browser shows a tab with nothing + in it.""" + shot = await self._call("page/capture") + if shot.get("blank"): + size = await self.viewport() + return Image(size.width, size.height, BLANK * (size.width * size.height), "RGB") + data = base64.b64decode(shot["png"]) + return Image(int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big"), data, "png") + + async def ready_checks(self) -> list[ReadyCheck]: + try: + folder, _ = await self._found() + except DeviceUnavailable as e: + return [ReadyCheck("browser", False, e.message)] + failed = apps.ended_on_its_own(folder) + return [ReadyCheck("browser", failed is None, failed or f"the page of the browser in {folder.name}", + final=failed is not None)] + + @click + async def click_at(self, x, y, button, count, modifiers) -> None: + mask = _modifiers(modifiers) + events = [_mouse("mouseMoved", x, y, modifiers=mask)] + for n in range(1, count + 1): + events += [_mouse("mousePressed", x, y, button, BUTTONS[button], n, mask), + _mouse("mouseReleased", x, y, button, 0, n, mask)] + self._pointer = (x, y) + await self._input(events) + + @drag + async def drag_along(self, path, button, duration_ms, modifiers) -> None: + mask, held = _modifiers(modifiers), BUTTONS[button] + steps = _steps(path) + pause = duration_ms / 1000 / len(steps) + await self._input([_mouse("mouseMoved", *path[0], modifiers=mask), + _mouse("mousePressed", *path[0], button, held, 1, mask)]) + try: + for x, y in steps: + await self._input([_mouse("mouseMoved", x, y, "none", held, 0, mask)]) + await asyncio.sleep(pause) + finally: + await self._input([_mouse("mouseReleased", *path[-1], button, 0, 1, mask)]) + self._pointer = tuple(path[-1]) + + @scroll + async def scroll_at(self, x, y, direction, amount, modifiers) -> None: + dx = {"left": -1, "right": 1}.get(direction, 0) * amount * WHEEL_PX + dy = {"up": -1, "down": 1}.get(direction, 0) * amount * WHEEL_PX + self._pointer = (x, y) + await self._input([{"method": "Input.dispatchMouseEvent", "params": { + "type": "mouseWheel", "x": x, "y": y, "deltaX": dx, "deltaY": dy, "modifiers": _modifiers(modifiers)}}]) + + @pointer_move + async def move(self, x, y) -> None: + self._pointer = (x, y) + await self._input([_mouse("mouseMoved", x, y, "none", self._buttons, 0, self._mask())]) + + @pointer_down + async def down(self, button) -> None: + self._buttons |= BUTTONS[button] + await self._input([_mouse("mousePressed", *self._pointer, button, self._buttons, 1, self._mask())]) + + @pointer_up + async def up(self, button) -> None: + self._buttons &= ~BUTTONS[button] + await self._input([_mouse("mouseReleased", *self._pointer, button, self._buttons, 1, self._mask())]) + + @pointer_position + async def where(self) -> tuple[int, int]: + return round(self._pointer[0]), round(self._pointer[1]) + + @type_text + async def type_in(self, text) -> None: + events = [] + for ch in text: + key = TYPED.get(ch, ch) + if key in NAMED or _printable(key): + events += _keystroke(key, MODIFIERS["Shift"] if key in SHIFTED or key.isupper() else 0) + else: + events.append({"method": "Input.insertText", "params": {"text": ch}}) + await self._input(events) + + @press + async def press_keys(self, keys) -> None: + events, mask = [], self._mask() + for key in keys: + mask |= MODIFIERS.get(key, 0) + events.append(_key("keyDown", key, mask)) + for key in reversed(keys): + mask &= ~MODIFIERS.get(key, 0) + events.append(_key("keyUp", key, mask)) + await self._input(events) + + @key_down + async def keys_down(self, keys) -> None: + self._held += [key for key in keys if key not in self._held] + await self._input([_key("keyDown", key, self._mask()) for key in keys]) + + @key_up + async def keys_up(self, keys) -> None: + self._held = [key for key in self._held if key not in keys] + await self._input([_key("keyUp", key, self._mask()) for key in keys]) + + def _mask(self) -> int: + return _modifiers(self._held) + + async def _input(self, events: list[dict]) -> None: + await self._call("page/input", {"events": events}) + + async def _call(self, op: str, params: Optional[dict] = None) -> Any: + """``browser/v1``'s ``op`` on the browser's server; DeviceUnavailable when it can't answer or says why not, and + why the browser can't come back once it has ended on its own.""" + folder, card = await self._found() + method = client.find_extension_method(card, BROWSER, op) + if method is None: + raise Unsupported(f"the browser in {folder.name} serves no {op}") + try: + async with _http(folder) as http: + answer = await http.request(method["method"], method["endpoint"], + **({"json": params} if method["method"] != "GET" else {})) + except (httpx.HTTPError, OSError) as e: + raise DeviceUnavailable(_ended(folder) or f"the browser's server isn't answering: {type(e).__name__}: {e}" + ) from None + if answer.status_code >= 400: + raise DeviceUnavailable(_ended(folder) or f"the browser couldn't {op}: {answer.text[:300]}") + return answer.json() + + async def _found(self) -> tuple[Path, dict]: + """The folder and card of the computer's one website browser, found once its server answers.""" + if self._browser is None: + for folder in self._computer.apps: + card = await apps.served_card(folder) + if card is not None and client.find_extension_method(card, BROWSER, "page/capture"): + self._browser = (folder, card) + break + else: + raise DeviceUnavailable("no website browser on this computer answers yet") + return self._browser + + +def _ended(folder: Path) -> Optional[str]: + if (failed := apps.ended_on_its_own(folder)) is not None: + return f"the browser in {folder.name} has failed: {failed}" + return None + + +def _http(folder: Path) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(uds=str(folder / SOCKET)), base_url="http://app", + timeout=CALL_TIMEOUT_S) + + +def _mouse(kind: str, x: float, y: float, button: str = "none", buttons: int = 0, count: int = 0, + modifiers: int = 0) -> dict: + return {"method": "Input.dispatchMouseEvent", "params": { + "type": kind, "x": x, "y": y, "button": button, "buttons": buttons, "clickCount": count, "modifiers": modifiers}} + + +def _keystroke(key: str, modifiers: int) -> list[dict]: + return [_key("keyDown", key, modifiers), _key("keyUp", key, modifiers)] + + +def _key(kind: str, key: str, modifiers: int) -> dict: + """A CDP key event for the W3C key ``key`` with ``modifiers`` held: one with Control, Alt or Meta held types + nothing, so it reaches the page as a shortcut.""" + code, vk, text = _codes(key) + if modifiers & (MODIFIERS["Control"] | MODIFIERS["Alt"] | MODIFIERS["Meta"]): + text = "" + params = {"type": ("keyDown" if text else "rawKeyDown") if kind == "keyDown" else "keyUp", "key": key, + "code": code, "windowsVirtualKeyCode": vk, "nativeVirtualKeyCode": vk, "modifiers": modifiers} + if kind == "keyDown" and text: + params.update(text=text, unmodifiedText=text) + return {"method": "Input.dispatchKeyEvent", "params": params} + + +def _codes(key: str) -> tuple[str, int, str]: + """The DOM code, virtual key code and text of ``key`` on a US layout.""" + if key in NAMED: + return NAMED[key] + if len(key) == 1 and key in string.ascii_letters: + return f"Key{key.upper()}", ord(key.upper()), key + if len(key) == 1 and key in string.digits: + return f"Digit{key}", ord(key), key + if key in PUNCTUATION: + code, vk, _ = PUNCTUATION[key] + return code, vk, key + if key in SHIFTED: + code, vk, _ = _codes(SHIFTED[key]) + return code, vk, key + raise Unsupported(f"this driver has no key {key!r}") + + +def _printable(key: str) -> bool: + return len(key) == 1 and (key in string.ascii_letters or key in string.digits or key in PUNCTUATION + or key in SHIFTED) + + +def _modifiers(keys) -> int: + mask = 0 + for key in keys: + mask |= MODIFIERS.get(key, 0) + return mask + + +def _steps(path: list) -> list[tuple[float, float]]: + """The points a drag moves through after its start, about every DRAG_STEP_PX pixels.""" + steps = [] + for (x0, y0), (x1, y1) in zip(path, path[1:]): + n = max(2, int(math.hypot(x1 - x0, y1 - y0) / DRAG_STEP_PX)) + steps += [(x0 + (x1 - x0) * k / n, y0 + (y1 - y0) * k / n) for k in range(1, n + 1)] + return steps + + +if __name__ == "__main__": + PageDriver().serve() diff --git a/src/agent_env/drivers/envs/page/env.toml b/src/agent_env/drivers/envs/page/env.toml new file mode 100644 index 000000000..6af38616c --- /dev/null +++ b/src/agent_env/drivers/envs/page/env.toml @@ -0,0 +1,7 @@ +type = "computer_driver" +module = "driver.py:PageDriver" +port = 18765 + +[install.linux] +python = "requirements.txt" +services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:PageDriver" }] diff --git a/src/agent_env/drivers/envs/page/requirements.txt b/src/agent_env/drivers/envs/page/requirements.txt new file mode 100644 index 000000000..6b36e62a6 --- /dev/null +++ b/src/agent_env/drivers/envs/page/requirements.txt @@ -0,0 +1,3 @@ +# What a composed computer installs into the driver's venv. A release pins agentenv-framework-protocol to the version +# it ships, which serves the driver with agentenv_protocol.computer.serve. +agentenv-framework-protocol[computer]>=0.1.381 diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index d57d0ba5a..36f374494 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -124,7 +124,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact elif computer is not None and not problems: _, refused = gate.check(computer, driver, environment_name=environment_name, apps=[], display=None, network=None, computer_settings={}) - problems += refused + gate.check_app(computer, driver, app) + problems += refused + gate.check_app(computer, driver, app) + gate.check_screen(driver, [app], "app") if problems: raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) app = get_env_store().put_document(app) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 5e36d9a8f..b1a3954a8 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -118,6 +118,7 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, apps, app_settings = list(apps), app_settings or {} values, problems = gate.check(computer, driver, environment_name=environment_name, apps=apps, display=display, network=network, computer_settings=computer_settings or {}) + problems += gate.check_screen(driver, apps) by_name = {app.environment_name: app for app in apps} for name, defaults in app_settings.items(): problems += ([f"apps.{name}.settings: {id} has no app {name!r}"] if name not in by_name else diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index 634638db6..8c9885b38 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -37,6 +37,10 @@ _POLL_S = 0.5 #: How a setting a record keeps shows a secret's or a generated one's value, which it never holds. WITHHELD = "" + + +class ComputerFailed(RuntimeError): + """The computer's ``computer-ready`` status says it failed for good, a check that can't pass again.""" #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 #: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in. @@ -168,12 +172,13 @@ def ready_timeout(apps: list[ComputerAppEnv], os_type: str) -> float: async def ready(url: str, timeout: float, token: Optional[str] = None) -> dict: """The card of the driver at ``url``, read once its ``computer-ready`` status, asked with its control ``token``, - says it's ready, so each app it fronts is on it as the app's own server serves it; raises TimeoutError, naming - what it last saw, when it isn't within ``timeout`` seconds.""" - last = "nothing answered" + says it's ready, so each app it fronts is on it as the app's own server serves it; raises ComputerFailed as soon + as the status says it failed, and TimeoutError, naming what it last saw, when it isn't ready within ``timeout`` + seconds.""" + last, failed = "nothing answered", None try: async with asyncio.timeout(timeout): - while True: + while failed is None: try: card = await protocol_v1.get_card(url, timeout=5) status = await protocol_v1.invoke_extension(url, card, COMPUTER_READY, None, _STATUS_TIMEOUT_S, @@ -181,10 +186,13 @@ async def ready(url: str, timeout: float, token: Optional[str] = None) -> dict: headers=protocol_v1.control_headers(token)) if status.get("ready"): return await protocol_v1.get_card(url, timeout=5) + failed = status.get("failed") last = "not ready: " + "; ".join(f"{check['name']}: {check.get('detail')}" for check in status.get("checks", []) if not check.get("ok")) except Exception as e: last = f"{type(e).__name__}: {e}" - await asyncio.sleep(_POLL_S) + if failed is None: + await asyncio.sleep(_POLL_S) except TimeoutError: raise TimeoutError(f"the computer at {url} wasn't ready within {timeout:.0f} s ({last})") from None + raise ComputerFailed(f"the computer at {url} failed: {failed}") diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index 056d78f19..f7326acb0 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -38,6 +38,18 @@ def check(base: ComputerArtifact, driver: ComputerDriverEnv, *, environment_name return values, problems + invalid +def check_screen(driver: ComputerDriverEnv, apps: Sequence[ComputerAppEnv], at: str = "apps") -> list[str]: + """Why ``apps`` don't fit ``driver`` when its screen is a website browser's page: that needs exactly one website + browser among them.""" + if not any(platform.features.get("screen") == "page" for platform in driver.declaration.platforms): + return [] + browsers = [app.id for app in apps if app.declaration.browser] + if len(browsers) == 1: + return [] + return [f"{at}: {driver.id}'s screen is a website browser's page, so its computer takes exactly one website " + f"browser; it has {len(browsers)}{': ' + ', '.join(browsers) if browsers else ''}"] + + def forms(base: ComputerArtifact) -> list[str]: """The forms a composition of ``base`` builds: every one the base has.""" return [form for form in FORMS if getattr(base.forms, form, None)] diff --git a/tst/integration/computer/browser-computer/tasks/chromium.json b/tst/integration/computer/browser-computer/tasks/chromium.json new file mode 100644 index 000000000..de36cdb48 --- /dev/null +++ b/tst/integration/computer/browser-computer/tasks/chromium.json @@ -0,0 +1,3 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "chromium"} +] diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py new file mode 100644 index 000000000..e81bac6c7 --- /dev/null +++ b/tst/integration/computer/browser_computer_test.py @@ -0,0 +1,227 @@ +"""The browser as a computer: agentenv's chromium deployed alone, on the browser-24.04 base with no display and the page +driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; deployed +through the computer provider like any computer, its computer_* tools see and drive the page; nothing runs X and nothing +listens but the driver and the site proxy; a killed app server or driver is started again by supervisord and the +computer is ready again; and chromium killed fails it for good, computer-ready/v1 saying why and the tools answering +with it. Needs Docker and the local registry on localhost:5000.""" + +import asyncio +import json +import platform +import shlex +import time +import urllib.parse + +import httpx +import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client +from PIL import Image + +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER_READY +from agentenv_protocol.computer.conformance.client import image +from agentenv_protocol.website_browser import BROWSER + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.env import Env +from agent_env.env.envs.computer.utils import inside +from agent_env.store.routing import namespace_routing +from agent_env.task.teardown import teardown_run +from agent_env.task_step.context import TaskStepContext + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] +#: How long an effect may take to reach the page, or the computer to be ready again; how often it's asked meanwhile. +WAIT_S, POLL_S = 120.0, 0.5 +# A page whose title says, as JSON, what reached it: clicks and double clicks on its red box, its input's value, how far +# it has scrolled and the last keys pressed. +PAGE = """loading +
+""" +PAGE_URL = "data:text/html," + urllib.parse.quote(PAGE) +# Every TCP port something in the computer listens on, and whether any process is an X server. +LISTENING = """import glob +ports = set() +for table in ("/proc/net/tcp", "/proc/net/tcp6"): + for line in open(table).read().splitlines()[1:]: + fields = line.split() + if fields[3] == "0A": + ports.add(int(fields[1].rsplit(":", 1)[1], 16)) +x = [p for p in glob.glob("/proc/[0-9]*/comm") if open(p).read().strip() in ("Xvfb", "Xorg", "xfce4-session")] +print(sorted(ports), bool(x)) +""" + + +def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver(browser_computer): + results = browser_computer.metadata["composed"]["container"] + probe = results[ARCH]["probe"] + + with namespace_routing(): + report = json.loads(FileArtifact.get(probe["report"]["id"], probe["report"]["version"]).load()) + print(json.dumps({"counts": probe["checks"], "extensions": probe["extensions"], + "not passed": [(c["check"], c["name"], c["status"], c["detail"][:160]) for c in report["checks"] + if c["status"] != "passed"]}, indent=1)) + assert results[ARCH]["status"] == "ready" and probe["passed"], results[ARCH] + assert [app.environment_name for app in browser_computer.composed_apps()] == ["chromium"] + + +@pytest.mark.asyncio +async def test_its_tools_see_and_drive_the_page_that_is_its_screen(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + await _state(deployed, lambda s: True) + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + shot = image(await session.call_tool("computer_screenshot", {"space": "frame"})) + await _tool(session, "computer_click", {"x": 200, "y": 150, "space": "frame"}) + clicked = await _state(deployed, lambda s: s["c"] == 1) + await _tool(session, "computer_click", {"x": 200, "y": 150, "count": 2, "space": "frame"}) + doubled = await _state(deployed, lambda s: s["d"] == 1) + await _tool(session, "computer_click", {"x": 300, "y": 320, "space": "frame"}) + await _tool(session, "computer_type", {"text": "Hello, World!"}) + typed = await _state(deployed, lambda s: s["v"] == "Hello, World!") + await _tool(session, "computer_key", {"keys": ["Control", "a"]}) + await _tool(session, "computer_type", {"text": "x"}) + replaced = await _state(deployed, lambda s: s["v"] == "x") + await _tool(session, "computer_key", {"keys": ["Enter"]}) + entered = await _state(deployed, lambda s: "Enter" in s["k"]) + await _tool(session, "computer_scroll", {"x": 640, "y": 600, "direction": "down", "amount": 3, + "space": "frame"}) + scrolled = await _state(deployed, lambda s: s["y"] >= 300) + after = image(await session.call_tool("computer_screenshot", {"space": "frame"})) + listening, x_server = (await inside.run(deployed, ["python3", "-c", LISTENING], timeout_s=30))[ + "stdout"].rsplit(" ", 1) + command = (await inside.run(deployed, ["sh", "-c", "tr '\\0' ' ' < /proc/$(pgrep -o -f -- " + "'[-]-remote-debugging-pipe')/cmdline"], timeout_s=30))["stdout"] + finally: + report = await teardown_run(context) + + assert shot.size == (1280, 800) and shot.getpixel((200, 150)) == (255, 0, 0) + assert shot.getpixel((640, 700)) == (255, 255, 255) + assert clicked["c"] == 1 and doubled["d"] == 1 and typed["v"] == "Hello, World!" and replaced["v"] == "x" + assert "C-a" in replaced["k"] and "Enter" in entered["k"] and scrolled["y"] >= 300 + assert after.getpixel((200, 150)) != (255, 0, 0) + assert json.loads(listening) == [3128, 18765] and x_server.strip() == "False" + assert "--headless=new" in command and "--window-size=1280,800" in command + assert not report.failed + + +@pytest.mark.asyncio +async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fails_it_for_good(browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [PAGE_URL]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + await _kill(deployed, "[s]erve_app") + server_back = await _ready(deployed) + launches = (await _record(deployed))["launches"] + await _kill(deployed, "[c]omputer.serve driver.py:PageDriver") + driver_back = await _ready(deployed) + shot = await _screenshot(deployed) + await _kill(deployed, "[-]-remote-debugging-pipe", oldest=True) + failed = await _until(deployed, lambda status: "failed" in status) + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + refused = await session.call_tool("computer_screenshot", {"space": "frame"}) + finally: + report = await teardown_run(context) + + assert server_back["ready"] and driver_back["ready"] and shot.size == (1280, 800) + assert [entry.get("cause") for entry in launches if entry["event"] == "launch"] == ["boot", "server_restart"] + browser = next(check for check in failed["checks"] if check["name"] == "browser") + assert failed["ready"] is False and browser["final"] is True and failed["failed"] == f"browser: {browser['detail']}" + assert "by signal SIGKILL" in browser["detail"] and "infrastructure failure" in browser["detail"] + assert refused.isError and "has failed" in refused.content[0].text and "SIGKILL" in refused.content[0].text + assert not report.failed + + +async def _tool(session: ClientSession, name: str, args: dict) -> None: + result = await session.call_tool(name, args) + assert not result.isError, f"{name} {args}: {result.content[0].text}" + + +async def _record(deployed) -> dict: + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + app = next(child for child in card["children_environments"] if child["name"] == "chromium") + path = protocol_v1.find_extension_method(app, BROWSER, "record")["endpoint"] + return (await _get(deployed, path, headers=deployed.control_headers)).json() + + +async def _state(deployed, holds) -> dict: + """The page's state, from its title, once ``holds`` says it has what's waited for, within ``WAIT_S``.""" + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + try: + last = json.loads((await _record(deployed))["tabs"][0]["title"]) + if holds(last): + return last + except (httpx.HTTPError, ValueError, KeyError, IndexError): + pass + await asyncio.sleep(POLL_S) + raise AssertionError(f"the page didn't get there within {WAIT_S:g} s; it last said {last!r}") + + +async def _status(deployed) -> dict: + card = (await _get(deployed, "/.well-known/agent-env.json")).json() + path = protocol_v1.find_extension_method(card, COMPUTER_READY, "status")["endpoint"] + return (await _get(deployed, path, headers=deployed.control_headers)).json() + + +async def _until(deployed, holds) -> dict: + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + try: + last = await _status(deployed) + if holds(last): + return last + except (httpx.HTTPError, ValueError, KeyError): + pass + await asyncio.sleep(POLL_S) + raise AssertionError(f"computer-ready didn't get there within {WAIT_S:g} s; it last said {last!r}") + + +async def _ready(deployed) -> dict: + await asyncio.sleep(2) + return await _until(deployed, lambda status: status["ready"]) + + +async def _kill(deployed, pattern: str, oldest: bool = False) -> None: + """SIGKILL the newest, or oldest, process matching ``pattern``, a second after computer-exec has answered, since the + driver that runs the command may be the one killed.""" + pick = "-o" if oldest else "-n" + killer = f"sleep 1; kill -9 $(pgrep {pick} -f -- '{pattern}')" + await inside.run(deployed, ["sh", "-c", f"setsid sh -c {shlex.quote(killer)} >/dev/null 2>&1 < /dev/null &"], + timeout_s=30) + await asyncio.sleep(2) + + +async def _screenshot(deployed) -> Image.Image: + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + return image(await session.call_tool("computer_screenshot", {"space": "frame"})) + + +async def _get(deployed, path: str, **kwargs) -> httpx.Response: + async with httpx.AsyncClient(timeout=30) as http: + return await http.get(f"{deployed.environment_url}{path}", **kwargs) diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index 5d14ba95b..8166fefe7 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -1,6 +1,6 @@ """The ubuntu-24.04 base computer, built from its folder once for the tests here and tagged for the local registry; the -shipped desktop example's computer env, the cua-driver test bundle's, the chromium test bundle's and the website-browser -test bundle's, each composed once for a module's tests; +shipped desktop example's computer env, the cua-driver test bundle's, the chromium test bundle's, the website-browser +test bundle's and agentenv's chromium deployed alone, each composed once for a module's tests; ``state``, local stores for a test that runs a bundle; and ``docker``, which runs the docker CLI for them.""" import logging @@ -28,6 +28,8 @@ CUA_DRIVER = Path(__file__).parent / "cua-driver" CHROMIUM = Path(__file__).parent / "chromium" WEBSITE_BROWSER = Path(__file__).parent / "website-browser" +BROWSER_COMPUTER = Path(__file__).parent / "browser-computer" +CHROMIUM_ALONE = "@local/agentenv-framework/browsers/chromium" @pytest.fixture(scope="session") @@ -67,7 +69,15 @@ def website_browser_desktop(tmp_path_factory): yield from _composed(tmp_path_factory.mktemp("website-browser-desktop"), parse_bundle(WEBSITE_BROWSER)) -def _composed(root: Path, bundle): +@pytest.fixture(scope="module") +def browser_computer(tmp_path_factory): + """agentenv's chromium, which deploys alone as a browser computer: the browser-24.04 base, with no display, and the + page driver, whose screen is the browser's page; composed, through the browser-computer test bundle's task that + deploys it, as ``desktop`` is.""" + yield from _composed(tmp_path_factory.mktemp("browser-computer"), parse_bundle(BROWSER_COMPUTER), CHROMIUM_ALONE) + + +def _composed(root: Path, bundle, env_id: str | None = None): patch = pytest.MonkeyPatch() patch.chdir(root) patch.setenv("XDG_STATE_HOME", str(root / "state")) @@ -78,7 +88,7 @@ def _composed(root: Path, bundle): with namespace_routing(): plan = plan_bundle(resolve_bundle(bundle), installed=installed_bundles) materialize(plan, build_here=frozenset(w.id for w in plan.writes if isinstance(w.source, BuiltImage))) - yield Env.get(f"{bundle.id_root}/desktop") + yield Env.get(env_id or f"{bundle.id_root}/desktop") reset_artifact_store() reset_config() patch.undo() diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index 2155b86d7..a2e440229 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -130,7 +130,8 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun assert {"BrowserSignin": 0, "DownloadDirectory": "${user_home}/Downloads"}.items() <= json.loads( policies["stdout"]).items() params = protocol_v1.extension_params(card, BROWSER) - assert (params["chromium"], params["clock"], list(params["methods"])) == ("156.0.8078.4", "real", ["record"]) + assert (params["chromium"], params["clock"], sorted(params["methods"])) == ( + "156.0.8078.4", "real", ["page/capture", "page/input", "page/viewport", "record"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 assert relaunched_readiness["ready"] is True, relaunched_readiness diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py new file mode 100644 index 000000000..610bb6d36 --- /dev/null +++ b/tst/unit/drivers/page_driver_test.py @@ -0,0 +1,142 @@ +"""The page driver against a fake browser server on the app's socket: it passes the definition checks with a page +screen, turns tool calls into the CDP input they mean, shows a tab that paints nothing as white, and once the browser +has ended on its own, its ready check fails for good and its calls say why.""" +from __future__ import annotations + +import base64 +import importlib.util +import json +from pathlib import Path + +import httpx +import pytest + +import agent_env +from agentenv_protocol.computer import DeviceUnavailable, Platform +from agentenv_protocol.computer import driver as computer_driver +from agentenv_protocol.computer.app_runtime import LAUNCHES +from agentenv_protocol.website_browser import BROWSER + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "page" / "driver.py" +NO_DISPLAY = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="none") +PAGE_OPS = ("page/viewport", "page/capture", "page/input") +CARD = {"name": "chromium", "capabilities": {"extensions": [{"uri": BROWSER, "params": {"methods": { + op: {"method": "POST" if op == "page/input" else "GET", "endpoint": f"/agentenv/ext/browser/{op}"} + for op in PAGE_OPS}}}]}} +#: A 2×1 PNG's first 24 bytes: its signature, then its IHDR's length, type, width and height. +PNG = b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + (2).to_bytes(4, "big") + (1).to_bytes(4, "big") + + +def _load(): + spec = importlib.util.spec_from_file_location("page_driver", DRIVER) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def page(monkeypatch, tmp_path): + """The driver, on a computer with no display, its one browser in ``tmp_path``, whose server answers ``served``, + a dict of answers by op, and records each request it gets in ``sent``.""" + module = _load() + monkeypatch.setattr(computer_driver, "detect", lambda: NO_DISPLAY) + driver = module.PageDriver() + driver._browser = (tmp_path, CARD) + driver.sent, driver.served = [], {"page/viewport": {"width": 1280, "height": 800}} + + def handler(request: httpx.Request) -> httpx.Response: + op = request.url.path.removeprefix("/agentenv/ext/browser/") + driver.sent.append((op, json.loads(request.content) if request.content else None)) + answer = driver.served.get(op, {}) + return answer if isinstance(answer, httpx.Response) else httpx.Response(200, json=answer) + + monkeypatch.setattr(module, "_http", lambda folder: httpx.AsyncClient( + transport=httpx.MockTransport(handler), base_url="http://app")) + return driver + + +def _ended(folder: Path) -> None: + (folder / LAUNCHES).parent.mkdir(parents=True, exist_ok=True) + (folder / LAUNCHES).write_text(json.dumps({"event": "launch", "cause": "boot", "at": "t0"}) + "\n" + + json.dumps({"event": "exit", "signal": "SIGKILL", "at": "t1"}) + "\n") + + +def _events(page) -> list[dict]: + return [event for op, body in page.sent if op == "page/input" for event in body["events"]] + + +def test_its_screen_is_a_page_on_a_computer_with_no_display(): + module = _load() + + assert module.PageDriver.screen_kind == "page" + assert module.PageDriver.supports == [{"os": {"type": "linux", "display": ["none"]}, "arch": ["amd64", "arm64"]}] + + +@pytest.mark.asyncio +async def test_a_key_with_control_held_types_nothing_so_it_reaches_the_page_as_a_shortcut(page): + await page.press_keys(["Control", "a"]) + + assert [(e["params"]["type"], e["params"]["key"], e["params"]["modifiers"], e["params"].get("text")) + for e in _events(page)] == [("rawKeyDown", "Control", 2, None), ("rawKeyDown", "a", 2, None), + ("keyUp", "a", 2, None), ("keyUp", "Control", 0, None)] + + +@pytest.mark.asyncio +async def test_a_key_held_down_then_let_go_is_released_with_its_own_modifier_off(page): + await page.keys_down(["Shift"]) + await page.keys_up(["Shift"]) + + assert [(e["params"]["type"], e["params"]["modifiers"]) for e in _events(page)] == [("rawKeyDown", 8), + ("keyUp", 0)] + + +@pytest.mark.asyncio +async def test_typed_text_is_keystrokes_with_shift_where_it_takes_one_and_inserted_text_for_the_rest(page): + await page.type_in("Hi!\né") + + events = _events(page) + downs = [(e["params"]["key"], e["params"]["modifiers"], e["params"]["text"]) for e in events + if e["params"].get("type") == "keyDown"] + assert downs == [("H", 8, "H"), ("i", 0, "i"), ("!", 8, "!"), ("Enter", 0, "\r")] + assert events[-1] == {"method": "Input.insertText", "params": {"text": "é"}} + + +@pytest.mark.asyncio +async def test_a_double_click_is_one_move_then_two_presses_and_releases_counted(page): + await page.click_at(10, 20, "left", 2, []) + + assert [(e["params"]["type"], e["params"]["clickCount"]) for e in _events(page)] == [ + ("mouseMoved", 0), ("mousePressed", 1), ("mouseReleased", 1), ("mousePressed", 2), ("mouseReleased", 2)] + + +@pytest.mark.asyncio +async def test_its_capture_is_the_pages_png_and_a_tab_that_paints_nothing_is_white(page): + page.served["page/capture"] = {"png": base64.b64encode(PNG).decode()} + shot = await page.page() + page.served["page/capture"] = {"blank": True} + blank = await page.page() + + assert (shot.width, shot.height, shot.mode, shot.data) == (2, 1, "png", PNG) + assert (blank.width, blank.height, blank.mode) == (1280, 800, "RGB") and set(blank.data) == {0xff} + + +@pytest.mark.asyncio +async def test_its_browser_check_passes_until_the_browser_ends_on_its_own_then_fails_for_good(page, tmp_path): + before = await page.ready_checks() + _ended(tmp_path) + after = await page.ready_checks() + + assert [(c.name, c.ok, c.final) for c in before] == [("browser", True, False)] + assert [(c.name, c.ok, c.final) for c in after] == [("browser", False, True)] + assert "by signal SIGKILL" in after[0].detail and "infrastructure failure" in after[0].detail + + +@pytest.mark.asyncio +async def test_once_the_browser_has_ended_its_calls_say_why_rather_than_what_its_server_answered(page, tmp_path): + page.served["page/capture"] = httpx.Response(503, text="the browser has no tab open") + with pytest.raises(DeviceUnavailable, match="couldn't page/capture: the browser has no tab open"): + await page.page() + _ended(tmp_path) + + with pytest.raises(DeviceUnavailable, match=f"the browser in {tmp_path.name} has failed: .*by signal SIGKILL"): + await page.page() diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 0b8c34fb6..3dbc49012 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -201,7 +201,8 @@ def _base(image, **changes): "init": "supervisord", "input_channel": "env"}}, **changes}) -def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, install=True, extra=None): +def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, install=True, extra=None, + features=None): folder.mkdir(parents=True, exist_ok=True) (folder / "driver.py").write_text("class Desk: ...\n") (folder / "requirements.txt").write_text(requirements) @@ -211,7 +212,7 @@ def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, i files = FileArtifactUniverse.put(files.id, file_artifacts={**files.get_file_artifacts(), **extra}) supports = supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] platforms = [{"platform": {"os": "linux", "arch": arch, "display": "x11"}, "provides": [], "tools": TOOLS, - "features": {}} for arch in ("amd64", "arm64")] + "features": features or {}} for arch in ("amd64", "arm64")] services = services or [Service(name="driver", command=SERVE)] return ComputerDriverEnv.put( id="@local/~/kit/desk", module="driver.py:Desk", files=files, @@ -352,11 +353,12 @@ async def create_vm(self, **size): def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, name="mail", paths=None, defaults=None, - **changes): + browser=None, **changes): """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of its build settings, ``linux_settings``, that of its settings on Linux, ``name``, its id's last part and its - environment name, ``paths``, where it keeps its data on Linux, and ``defaults``, its env's ``[settings]``.""" + environment name, ``paths``, where it keeps its data on Linux, ``defaults``, its env's ``[settings]``, and + ``browser``, whether it declares itself a website browser.""" folder.mkdir(parents=True, exist_ok=True) (folder / "app.py").write_text("class Mail: ...\n") (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") @@ -369,7 +371,7 @@ def _app(base, driver, folder, compose=True, supports=None, install=None, settin "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], "permissions": [], "settings": settings, "build_settings": build, "skill": None, "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], - "ready_checks": [], "os": {"linux": { + "ready_checks": [], **({"browser": browser} if browser is not None else {}), "os": {"linux": { "launch": launch, "settings": linux_settings, "paths": paths, "install": install or { "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) return ComputerAppEnv.put(**{"id": f"@local/~/kit/{name}", "environment_name": name, "module": "app.py:Mail", diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 2cc8bba9b..72dc6d70a 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -423,6 +423,40 @@ def test_a_driver_that_cant_compose_with_the_base_is_refused(base, docker, tmp_p assert problem in str(refused.value) +@pytest.mark.parametrize("names, problem", [ + ((), "apps: @local/~/kit/desk's screen is a website browser's page, so its computer takes exactly one website " + "browser; it has 0"), + (("first", "second"), "it has 2: @local/~/kit/first, @local/~/kit/second"), +]) +def test_a_driver_whose_screen_is_a_page_takes_exactly_one_website_browser_and_any_other_apps(base, docker, tmp_path, + names, problem): + page = _driver(tmp_path / "driver", features={"screen": "page"}) + mail = _app(None, None, tmp_path / "mail", compose=False) + browsers = [_app(None, None, tmp_path / name, compose=False, name=name, browser=True) for name in names] + + with pytest.raises(ValueError) as refused: + _put(base, page, compose=False, apps=[*browsers, mail]) + alone = _put(base, page, compose=False, apps=[_app(None, None, tmp_path / "one", compose=False, name="one", + browser=True), mail]) + desktop = _put(base, _driver(tmp_path / "desktop-driver"), compose=False, id="@local/~/kit/two", + apps=[_app(None, None, tmp_path / "a", compose=False, name="a", browser=True), + _app(None, None, tmp_path / "b", compose=False, name="b", browser=True)]) + + assert problem in str(refused.value) + assert [app.environment_name for app in alone.apps] == ["one", "mail"] and len(desktop.apps) == 2 + + +def test_an_app_deployed_alone_on_a_driver_whose_screen_is_a_page_is_one_website_browser(base, docker, tmp_path): + page = _driver(tmp_path / "driver", features={"screen": "page"}) + + with pytest.raises(ValueError, match="app: @local/~/kit/desk's screen is a website browser's page, so its " + "computer takes exactly one website browser; it has 0"): + _app(base, page, tmp_path / "mail", compose=False) + browser = _app(base, page, tmp_path / "browser", compose=False, name="browser", browser=True) + + assert browser.declaration.browser is True + + def test_a_base_and_driver_with_no_arch_in_common_and_an_image_that_cant_be_got_are_refused(docker, tmp_path): image = get_artifact_store().put_document(DockerImageArtifact(id="base-image", description="", image_name="local/base:1")) @@ -740,6 +774,27 @@ async def status(url, card, uri, params, timeout, method, headers): assert sent == [{"Authorization": "Bearer t0ken"}] * 2 +@pytest.mark.asyncio +async def test_a_boot_whose_status_says_the_computer_failed_raises_at_once_with_why(monkeypatch): + asked = [] + + async def card(url, timeout): + return {"name": "desk"} + + async def status(url, card, uri, params, timeout, method, headers): + asked.append(method) + return {"ready": False, "failed": "browser: it ended on its own", "checks": [ + {"name": "browser", "ok": False, "detail": "it ended on its own", "final": True}]} + + monkeypatch.setattr(boot.protocol_v1, "get_card", card) + monkeypatch.setattr(boot.protocol_v1, "invoke_extension", status) + monkeypatch.setattr(boot, "_POLL_S", 0) + + with pytest.raises(boot.ComputerFailed, match="the computer at http://desk failed: browser: it ended on its own"): + await boot.ready("http://desk", 5, "t0ken") + assert asked == ["status"] + + def test_a_boot_mints_a_generated_setting_resolves_a_secret_its_record_withholds_both_and_a_secret_is_a_reference( base, monkeypatch): monkeypatch.setenv("DESK_KEY", "s3cret") From f339e9beafe365a479dd1879dbeaff800c4831d7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 16:27:06 -0800 Subject: [PATCH 2/4] build(computer): publish the browser-24.04 base as the release publishes ubuntu-24.04's, its digest bumped by Dependabot A base with no disk form gets no disk image. The shipped driver aliases now include page. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 1 + .github/scripts/infra_build.py | 6 ++++++ src/agent_env/env/published_images.py | 11 ++++++----- tst/unit/bundle/shipped_bundles_test.py | 3 ++- tst/unit/env/published_images_test.py | 6 +++--- tst/unit/test_infra_build_script.py | 2 +- tst/unit/test_infra_images_script.py | 2 +- 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 49d3f28dd..41ab2b41c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,6 +56,7 @@ updates: - "/src/agent_env/agents/agents/pydantic-ai" - "/src/agent_env/examples/mail-mcp/envs/mail" - "/src/agent_env/computers/artifacts/ubuntu-24.04" + - "/src/agent_env/computers/artifacts/browser-24.04" - "/src/agent_env/computers/disk/linux/boot" - "/src/agent_env/computers/disk/linux/tools" - "/src/agent_env/computers/disk/linux/launcher" diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 15d6e69a1..04e97e86f 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -56,6 +56,7 @@ "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile", "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile", "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile", + "agentenv-browser-24.04": "src/agent_env/computers/artifacts/browser-24.04/Dockerfile", "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile", "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", @@ -118,6 +119,11 @@ " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], + "agentenv-browser-24.04": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "start-computer & for _ in $(seq 60); do sleep 1;" + " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" + " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" + " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], "agentenv-disk-tools": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", "command -v disk image-env mkfs.ext4 mkfs.vfat mcopy sfdisk qemu-img"], "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index 54a426c05..8443bb98a 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -71,18 +71,19 @@ def ref(self) -> str: # The example env `a2a-agent validate` deploys, so a remote sandbox pulls its image rather than building it. EXAMPLE_IMAGES = (PublishedImage("agentenv-mail-mcp", _EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile"),) # The base computers the computers bundle ships, so a run pulls a base rather than building its desktop. -COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"),) +COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"), + PublishedImage("agentenv-browser-24.04", _COMPUTERS / "browser-24.04" / "Dockerfile")) # The shipped bundles' published images, whose requirements keep a protocol floor. BUNDLE_IMAGES = AGENT_IMAGES + EXAMPLE_IMAGES + COMPUTER_IMAGES # What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. DISK_TOOLS_IMAGE = PublishedImage("agentenv-disk-tools", DISK / "tools" / "Dockerfile") LAUNCHER_IMAGE = PublishedImage("agentenv-qemu-launcher", DISK / "launcher" / "Dockerfile") -# Each base computer's disk form: its container form's root filesystem on the boot recipe, at the size its [forms.disk] -# declares, so a run pulls it rather than building it from the two. +# Each base computer's disk form, for a base that has one: its container form's root filesystem on the boot recipe, at +# the size its [forms.disk] declares, so a run pulls it rather than building it from the two. DISK_IMAGES = tuple(PublishedImage( f"{image.name}-disk", DISK / "boot" / "Dockerfile", (("ROOTFS", image), ("TOOLS", DISK_TOOLS_IMAGE)), - (("SIZE_GB", str(tomllib.loads((image.dockerfile.parent / "artifact.toml").read_text())["forms"]["disk"][ - "min_disk_gb"])),)) for image in COMPUTER_IMAGES) + (("SIZE_GB", str(disk["min_disk_gb"])),)) for image in COMPUTER_IMAGES + if (disk := tomllib.loads((image.dockerfile.parent / "artifact.toml").read_text())["forms"].get("disk"))) def source_tag(digest: str) -> str: diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 9f3bd8274..39f75fc69 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -59,7 +59,8 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri assert table.refused == {} and table.refused_claims == {} assert {alias: named.target_id for alias, named in table.aliases.items()} == { "mail": "@local/agentenv-framework/mail-mcp/mail", "reference": "@local/agentenv-framework/drivers/reference", - "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "chromium": "@local/agentenv-framework/browsers/chromium"} + "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "page": "@local/agentenv-framework/drivers/page", + "chromium": "@local/agentenv-framework/browsers/chromium"} assert A2AAgent.VALIDATION_ENV_ID == "mail" def test_a_task_naming_no_agent_plans_with_the_shipped_default(tmp_path, monkeypatch, local_stores, cli_routing): diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index 7dfd67121..46dcde31f 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -13,7 +13,6 @@ from agent_env.env import bootstrap, published_images from agent_env.env.published_images import ( AGENT_IMAGES, - COMPUTER_IMAGES, DISK, DISK_IMAGES, DISK_TOOLS_IMAGE, @@ -41,7 +40,7 @@ def test_every_shipped_dockerfile_is_published_once_under_a_name_of_its_own(): shipped.append(published_images._EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile") shipped += published_images._COMPUTERS.glob("*/Dockerfile") shipped += DISK.glob("*/Dockerfile") - shipped += [DISK / "boot" / "Dockerfile"] * (len(COMPUTER_IMAGES) - 1) + shipped += [DISK / "boot" / "Dockerfile"] * (len(DISK_IMAGES) - 1) assert sorted(image.dockerfile for image in bootstrap.PUBLISHED_IMAGES) == sorted(shipped) assert len({image.name for image in bootstrap.PUBLISHED_IMAGES}) == len(bootstrap.PUBLISHED_IMAGES) @@ -178,7 +177,8 @@ def test_a_shipped_agents_sources_whose_image_isnt_published_find_none(monkeypat def test_a_disk_forms_tag_is_of_its_boot_recipe_its_bases_sources_and_the_disk_tools_and_it_is_found_by_its_bases( tmp_path, monkeypatch): - (disk,), (base,) = DISK_IMAGES, COMPUTER_IMAGES + (disk,) = DISK_IMAGES + base = dict(disk.args)["ROOTFS"] recipe = BuildContext.of(disk.dockerfile.parent, disk.dockerfile).source_digest(None).removeprefix("sha256:") served = {} monkeypatch.setattr(published_images, "_registry_digest", served.get) diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index da1246a5c..7933685ff 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -254,7 +254,7 @@ def test_verify_record_refuses_a_digest_the_registry_doesnt_serve_under_its_tag( @pytest.mark.parametrize("record, problem", [ - ({"images": []}, "records no agentenv-claude-code, agentenv-codex, agentenv-db-mcp"), + ({"images": []}, "records no agentenv-browser-24.04, agentenv-claude-code, agentenv-codex"), ({"images": [], "extra": 1}, "it holds ['extra', 'images']"), ({"images": [{"repository": "ghcr.io/someone/thing", "tag": _TAG, "digest": _DIGEST}]}, "'ghcr.io/someone/thing' isn't an image agent-env ships"), diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index 71be6dd6a..f7d7f3838 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -76,7 +76,7 @@ def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(sc assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", "agentenv-pydantic-ai", - "agentenv-mail-mcp", "agentenv-ubuntu-24.04"] + "agentenv-mail-mcp", "agentenv-ubuntu-24.04", "agentenv-browser-24.04"] def test_plan_missing_lists_the_images_the_registry_lacks_or_refuses_to_show(script, monkeypatch, capsys): From e6c927328d1eb4dd90d67fcfccd641d915659397 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 16:29:36 -0800 Subject: [PATCH 3/4] fix(computer): the page driver keeps held keys on what it types and clicks, a failed browser stays failed, and headless pages wait for their size Shift held types a as A, and a held modifier carries to clicks, drags and scrolls. Once the browser has ended on its own, every page call is refused with why, even after its server launches another. Headless, each new page is held until it is sized to the window, then told to run, and told to run even if preparing it fails. The stand-in proxy takes IPv6 literals in CONNECT. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/website_browser.py | 20 +++++++++----- .../tests/test_website_browser.py | 19 +++++++++++++ .../artifacts/browser-24.04/site-proxy.py | 1 + src/agent_env/drivers/envs/page/driver.py | 27 ++++++++++++++----- tst/unit/drivers/page_driver_test.py | 25 ++++++++++++++++- 5 files changed, 78 insertions(+), 14 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 00159dbd7..c24cdd2a9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -213,7 +213,8 @@ async def attach(self) -> None: downloads unattended: into ``~/Downloads``, a name downloaded again kept beside the first.""" self.ended(STOP) await self.devtools.call("Target.setDiscoverTargets", {"discover": True, "filter": PAGES}) - await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, "waitForDebuggerOnStart": False, + await self.devtools.call("Target.setAutoAttach", {"autoAttach": True, + "waitForDebuggerOnStart": self.viewport is not None, "flatten": True, "filter": PAGES}) await self.devtools.call("Browser.setDownloadBehavior", {"behavior": "default", "eventsEnabled": True}) @@ -341,14 +342,19 @@ def _on_Target_targetDestroyed(self, params: dict, session: Optional[str]) -> No def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> None: if params["targetInfo"].get("type") == "page": self._sessions[params["sessionId"]] = params["targetInfo"]["targetId"] - self._spawn(self.devtools.call("Page.enable", session_id=params["sessionId"])) + self._spawn(self._let_run(params["sessionId"])) + + async def _let_run(self, session: str) -> None: + """A page it attached to, enabled and, headless, held to its window's size, then told to run: a page attached as + it opens, a popup say, or any new page headless, waits for that before it runs.""" + try: + await self.devtools.call("Page.enable", session_id=session) if self.viewport is not None: width, height = self.viewport - self._spawn(self.devtools.call("Emulation.setDeviceMetricsOverride", { - "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, - session_id=params["sessionId"])) - # A page attached as it opens, a popup say, waits for this before it navigates. - self._spawn(self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=params["sessionId"])) + await self.devtools.call("Emulation.setDeviceMetricsOverride", { + "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) + finally: + await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: self._sessions.pop(params["sessionId"], None) diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 3a7b7c8c4..eeadda2b3 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -499,6 +499,8 @@ async def test_each_page_it_attaches_to_is_told_to_run_so_a_popup_it_attached_to assert ("Page.enable", {}, "s-popup") in devtools.calls assert ("Runtime.runIfWaitingForDebugger", {}, "s-popup") in devtools.calls assert not any(method == "Emulation.setDeviceMetricsOverride" for method, _, _ in devtools.calls) + assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ + "waitForDebuggerOnStart"] is False @pytest.mark.asyncio @@ -510,11 +512,28 @@ async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_ru await asyncio.sleep(0) sent = [method for method, _, session in devtools.calls if session == "s-popup"] + assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ + "waitForDebuggerOnStart"] is True assert sent == ["Page.enable", "Emulation.setDeviceMetricsOverride", "Runtime.runIfWaitingForDebugger"] assert ("Emulation.setDeviceMetricsOverride", {"width": 1024, "height": 768, "deviceScaleFactor": 1, "mobile": False}, "s-popup") in devtools.calls +@pytest.mark.asyncio +async def test_a_page_it_couldnt_prepare_is_still_told_to_run_so_it_never_stays_held(): + def refused(params, session): + raise DevToolsError("Target closed") + + devtools = StubDevTools({"Page.enable": refused}) + control = BrowserControl(devtools, "agent", (1024, 768)) + await control.attach() + devtools.send("Target.attachedToTarget", {"sessionId": "s-popup", "waitingForDebugger": True, **_page("popup")}) + await asyncio.sleep(0) + + assert [method for method, _, session in devtools.calls if session == "s-popup"] == [ + "Page.enable", "Runtime.runIfWaitingForDebugger"] + + class Painting(StubDevTools): """A browser whose every tab has focus and paints, but those whose sessions are in ``blank``, which never answer a capture; it records each capture's timeout.""" diff --git a/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py b/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py index 0f448fed4..5eacb8004 100644 --- a/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py +++ b/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py @@ -16,6 +16,7 @@ class Proxy(http.server.BaseHTTPRequestHandler): def do_CONNECT(self): host, _, port = self.path.rpartition(":") + host = host.removeprefix("[").removesuffix("]") try: upstream = socket.create_connection((host, int(port)), timeout=10) except (OSError, ValueError): diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index d331083b2..e9dcdc60f 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -116,7 +116,7 @@ async def ready_checks(self) -> list[ReadyCheck]: @click async def click_at(self, x, y, button, count, modifiers) -> None: - mask = _modifiers(modifiers) + mask = _modifiers(modifiers) | self._mask() events = [_mouse("mouseMoved", x, y, modifiers=mask)] for n in range(1, count + 1): events += [_mouse("mousePressed", x, y, button, BUTTONS[button], n, mask), @@ -126,7 +126,7 @@ async def click_at(self, x, y, button, count, modifiers) -> None: @drag async def drag_along(self, path, button, duration_ms, modifiers) -> None: - mask, held = _modifiers(modifiers), BUTTONS[button] + mask, held = _modifiers(modifiers) | self._mask(), BUTTONS[button] steps = _steps(path) pause = duration_ms / 1000 / len(steps) await self._input([_mouse("mouseMoved", *path[0], modifiers=mask), @@ -145,7 +145,8 @@ async def scroll_at(self, x, y, direction, amount, modifiers) -> None: dy = {"up": -1, "down": 1}.get(direction, 0) * amount * WHEEL_PX self._pointer = (x, y) await self._input([{"method": "Input.dispatchMouseEvent", "params": { - "type": "mouseWheel", "x": x, "y": y, "deltaX": dx, "deltaY": dy, "modifiers": _modifiers(modifiers)}}]) + "type": "mouseWheel", "x": x, "y": y, "deltaX": dx, "deltaY": dy, + "modifiers": _modifiers(modifiers) | self._mask()}}]) @pointer_move async def move(self, x, y) -> None: @@ -168,11 +169,14 @@ async def where(self) -> tuple[int, int]: @type_text async def type_in(self, text) -> None: - events = [] + """Each character as the keystroke that types it, with the keys held down still held: Shift held types ``a`` as + ``A``, and Control held makes it a shortcut; a character no key types is inserted as text.""" + held, events = self._mask(), [] for ch in text: key = TYPED.get(ch, ch) if key in NAMED or _printable(key): - events += _keystroke(key, MODIFIERS["Shift"] if key in SHIFTED or key.isupper() else 0) + mask = held | (MODIFIERS["Shift"] if key in SHIFTED or key.isupper() else 0) + events += _keystroke(_shifted(key) if mask & MODIFIERS["Shift"] else key, mask) else: events.append({"method": "Input.insertText", "params": {"text": ch}}) await self._input(events) @@ -206,8 +210,10 @@ async def _input(self, events: list[dict]) -> None: async def _call(self, op: str, params: Optional[dict] = None) -> Any: """``browser/v1``'s ``op`` on the browser's server; DeviceUnavailable when it can't answer or says why not, and - why the browser can't come back once it has ended on its own.""" + once the browser has ended on its own, why, even after its server has launched another.""" folder, card = await self._found() + if (failed := _ended(folder)) is not None: + raise DeviceUnavailable(failed) method = client.find_extension_method(card, BROWSER, op) if method is None: raise Unsupported(f"the browser in {folder.name} serves no {op}") @@ -286,6 +292,15 @@ def _codes(key: str) -> tuple[str, int, str]: raise Unsupported(f"this driver has no key {key!r}") +def _shifted(key: str) -> str: + """The key Shift makes of ``key`` on a US layout: a letter's capital, a digit's or punctuation's symbol.""" + if len(key) == 1 and key in string.ascii_lowercase: + return key.upper() + if key in PUNCTUATION: + return PUNCTUATION[key][2] + return {digit: symbol for symbol, digit in SHIFTED_DIGITS.items()}.get(key, key) + + def _printable(key: str) -> bool: return len(key) == 1 and (key in string.ascii_letters or key in string.digits or key in PUNCTUATION or key in SHIFTED) diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py index 610bb6d36..e99c7f4ef 100644 --- a/tst/unit/drivers/page_driver_test.py +++ b/tst/unit/drivers/page_driver_test.py @@ -90,6 +90,23 @@ async def test_a_key_held_down_then_let_go_is_released_with_its_own_modifier_off ("keyUp", 0)] +@pytest.mark.asyncio +async def test_with_shift_held_typed_text_is_shifted_and_with_control_held_a_click_carries_it(page): + await page.keys_down(["Shift"]) + await page.type_in("a1;") + await page.keys_up(["Shift"]) + await page.keys_down(["Control"]) + await page.click_at(10, 20, "left", 1, []) + await page.scroll_at(10, 20, "down", 1, ["Alt"]) + + events = _events(page) + typed = [(e["params"]["key"], e["params"]["modifiers"], e["params"]["text"]) for e in events + if e["params"].get("type") == "keyDown"] + assert typed == [("A", 8, "A"), ("!", 8, "!"), (":", 8, ":")] + assert {e["params"]["modifiers"] for e in events if e["params"]["type"] in ("mousePressed", "mouseReleased")} == {2} + assert events[-1]["params"]["type"] == "mouseWheel" and events[-1]["params"]["modifiers"] == 3 + + @pytest.mark.asyncio async def test_typed_text_is_keystrokes_with_shift_where_it_takes_one_and_inserted_text_for_the_rest(page): await page.type_in("Hi!\né") @@ -132,7 +149,7 @@ async def test_its_browser_check_passes_until_the_browser_ends_on_its_own_then_f @pytest.mark.asyncio -async def test_once_the_browser_has_ended_its_calls_say_why_rather_than_what_its_server_answered(page, tmp_path): +async def test_once_the_browser_has_ended_its_calls_say_why_and_send_nothing_though_its_server_answers(page, tmp_path): page.served["page/capture"] = httpx.Response(503, text="the browser has no tab open") with pytest.raises(DeviceUnavailable, match="couldn't page/capture: the browser has no tab open"): await page.page() @@ -140,3 +157,9 @@ async def test_once_the_browser_has_ended_its_calls_say_why_rather_than_what_its with pytest.raises(DeviceUnavailable, match=f"the browser in {tmp_path.name} has failed: .*by signal SIGKILL"): await page.page() + page.served["page/capture"] = {"blank": True} + sent = len(page.sent) + + with pytest.raises(DeviceUnavailable, match="has failed"): + await page.click_at(1, 2, "left", 1, []) + assert len(page.sent) == sent From 72530bed3bb5baae6f740357124f23a7b29c32e1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 16:36:43 -0800 Subject: [PATCH 4/4] refactor(computer): name the headless base ubuntu-24.04-headless, beside ubuntu-24.04, for what sets it apart Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 2 +- .github/scripts/infra_build.py | 13 +++++++------ src/agent_env/browsers/README.md | 4 ++-- src/agent_env/browsers/envs/chromium/env.toml | 2 +- src/agent_env/computers/README.md | 6 +++--- src/agent_env/computers/aliases.toml | 2 +- .../Dockerfile | 2 +- .../artifact.toml | 0 .../site-proxy.py | 0 .../start-computer | 0 .../supervisord.conf | 0 src/agent_env/drivers/README.md | 2 +- src/agent_env/env/published_images.py | 3 ++- tst/integration/computer/browser_computer_test.py | 2 +- tst/integration/computer/conftest.py | 2 +- tst/unit/test_infra_build_script.py | 2 +- tst/unit/test_infra_images_script.py | 2 +- 17 files changed, 23 insertions(+), 21 deletions(-) rename src/agent_env/computers/artifacts/{browser-24.04 => ubuntu-24.04-headless}/Dockerfile (91%) rename src/agent_env/computers/artifacts/{browser-24.04 => ubuntu-24.04-headless}/artifact.toml (100%) rename src/agent_env/computers/artifacts/{browser-24.04 => ubuntu-24.04-headless}/site-proxy.py (100%) rename src/agent_env/computers/artifacts/{browser-24.04 => ubuntu-24.04-headless}/start-computer (100%) rename src/agent_env/computers/artifacts/{browser-24.04 => ubuntu-24.04-headless}/supervisord.conf (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 41ab2b41c..5ea55bd29 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,7 +56,7 @@ updates: - "/src/agent_env/agents/agents/pydantic-ai" - "/src/agent_env/examples/mail-mcp/envs/mail" - "/src/agent_env/computers/artifacts/ubuntu-24.04" - - "/src/agent_env/computers/artifacts/browser-24.04" + - "/src/agent_env/computers/artifacts/ubuntu-24.04-headless" - "/src/agent_env/computers/disk/linux/boot" - "/src/agent_env/computers/disk/linux/tools" - "/src/agent_env/computers/disk/linux/launcher" diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 04e97e86f..fd0b10d57 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -56,7 +56,7 @@ "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile", "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile", "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile", - "agentenv-browser-24.04": "src/agent_env/computers/artifacts/browser-24.04/Dockerfile", + "agentenv-ubuntu-24.04-headless": "src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile", "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile", "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", @@ -119,11 +119,12 @@ " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], - "agentenv-browser-24.04": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", - "start-computer & for _ in $(seq 60); do sleep 1;" - " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" - " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" - " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], + "agentenv-ubuntu-24.04-headless": [ + "docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "start-computer & for _ in $(seq 60); do sleep 1;" + " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" + " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" + " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], "agentenv-disk-tools": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", "command -v disk image-env mkfs.ext4 mkfs.vfat mcopy sfdisk qemu-img"], "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 186a16cf0..06c6bfea1 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -1,6 +1,6 @@ The website browsers that come with agent-env, today `chromium`, a `computer_app` env a computer env lists in its -`apps`. Deployed alone, `chromium` is a browser computer: the `browser-24.04` base, with no display, and the `page` -driver, whose screen is the browser's active tab, headless. +`apps`. Deployed alone, `chromium` is a browser computer: the `ubuntu-24.04-headless` base, with no display, and the +`page` driver, whose screen is the browser's active tab, headless. `envs/chromium/` is Chromium for an X11 desktop or a computer with no display on Linux (`amd64` and `arm64`): Chrome for Testing 156.0.8078.4, the build Playwright 1.64.1 pins, fetched by sha256 when a computer env that lists it is put and unpacked into diff --git a/src/agent_env/browsers/envs/chromium/env.toml b/src/agent_env/browsers/envs/chromium/env.toml index 8e8054f0d..05625f6a1 100644 --- a/src/agent_env/browsers/envs/chromium/env.toml +++ b/src/agent_env/browsers/envs/chromium/env.toml @@ -1,6 +1,6 @@ type = "computer_app" module = "chromium.py:Chromium" -computer = "browser-24.04" +computer = "ubuntu-24.04-headless" driver = "page" [settings] diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index f1fc1608f..3ee85568d 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -8,13 +8,13 @@ container form's image is built from the folder's `Dockerfile`, and a release pu building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04`. -`artifacts/browser-24.04/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop +`artifacts/ubuntu-24.04-headless/` is Ubuntu 24.04 (`amd64` and `arm64`) with no display server or desktop (`display = "none"`), for one website browser, which runs headless there, and the `page` driver, whose screen is the browser's page: a browser computer, which agentenv's `chromium` deploys alone as. supervisord runs every program as the session user, among them a stand-in site proxy on `127.0.0.1:3128`, the browser's only route out, which forwards every request to wherever it names, refusing nothing: the sandbox's network policy is all that limits it until the shared -site proxy replaces it. It has a container form only. Name it from another bundle as `browser-24.04`, or by its id, -`@local/agentenv-framework/computers/browser-24.04`. +site proxy replaces it. It has a container form only. Name it from another bundle as `ubuntu-24.04-headless`, or by +its id, `@local/agentenv-framework/computers/ubuntu-24.04-headless`. It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, diff --git a/src/agent_env/computers/aliases.toml b/src/agent_env/computers/aliases.toml index f8c608e16..5b6e5b26d 100644 --- a/src/agent_env/computers/aliases.toml +++ b/src/agent_env/computers/aliases.toml @@ -1,4 +1,4 @@ # Ids a task can name these computers by; a store's artifact of the id wins over an alias. [artifacts] "ubuntu-24.04" = ["ubuntu-24.04"] -"browser-24.04" = ["browser-24.04"] +"ubuntu-24.04-headless" = ["ubuntu-24.04-headless"] diff --git a/src/agent_env/computers/artifacts/browser-24.04/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile similarity index 91% rename from src/agent_env/computers/artifacts/browser-24.04/Dockerfile rename to src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile index 5a1b01c6e..694f80d78 100644 --- a/src/agent_env/computers/artifacts/browser-24.04/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile @@ -1,4 +1,4 @@ -# The browser-24.04 base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), for one +# The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), for one # website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord runs # every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. Composing # a driver or an app onto it adds its services as programs in /etc/supervisor/conf.d. diff --git a/src/agent_env/computers/artifacts/browser-24.04/artifact.toml b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml similarity index 100% rename from src/agent_env/computers/artifacts/browser-24.04/artifact.toml rename to src/agent_env/computers/artifacts/ubuntu-24.04-headless/artifact.toml diff --git a/src/agent_env/computers/artifacts/browser-24.04/site-proxy.py b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py similarity index 100% rename from src/agent_env/computers/artifacts/browser-24.04/site-proxy.py rename to src/agent_env/computers/artifacts/ubuntu-24.04-headless/site-proxy.py diff --git a/src/agent_env/computers/artifacts/browser-24.04/start-computer b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer similarity index 100% rename from src/agent_env/computers/artifacts/browser-24.04/start-computer rename to src/agent_env/computers/artifacts/ubuntu-24.04-headless/start-computer diff --git a/src/agent_env/computers/artifacts/browser-24.04/supervisord.conf b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf similarity index 100% rename from src/agent_env/computers/artifacts/browser-24.04/supervisord.conf rename to src/agent_env/computers/artifacts/ubuntu-24.04-headless/supervisord.conf diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index c059cf8e5..0da83b5a8 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -7,7 +7,7 @@ pointer and keyboard groups, and `computer-ready`, `computer-display`, `computer `x11grab` through PyAV at up to 30 fps. Name it from another bundle as `reference`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/reference`. -`envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `browser-24.04`: +`envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `ubuntu-24.04-headless`: its screen is the computer's one website browser's active tab, its `screen` feature `page`. A screenshot is the page, and input reaches it as CDP input events, both through `browser/v1`'s `page/*` operations on the browser's own server; a tab that paints nothing shows white. The browser ending on its own fails the computer for good: its ready check is diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index 8443bb98a..58e6773b3 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -72,7 +72,8 @@ def ref(self) -> str: EXAMPLE_IMAGES = (PublishedImage("agentenv-mail-mcp", _EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile"),) # The base computers the computers bundle ships, so a run pulls a base rather than building its desktop. COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"), - PublishedImage("agentenv-browser-24.04", _COMPUTERS / "browser-24.04" / "Dockerfile")) + PublishedImage("agentenv-ubuntu-24.04-headless", + _COMPUTERS / "ubuntu-24.04-headless" / "Dockerfile")) # The shipped bundles' published images, whose requirements keep a protocol floor. BUNDLE_IMAGES = AGENT_IMAGES + EXAMPLE_IMAGES + COMPUTER_IMAGES # What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index e81bac6c7..efe205fff 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1,4 +1,4 @@ -"""The browser as a computer: agentenv's chromium deployed alone, on the browser-24.04 base with no display and the page +"""The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; deployed through the computer provider like any computer, its computer_* tools see and drive the page; nothing runs X and nothing listens but the driver and the site proxy; a killed app server or driver is started again by supervisord and the diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index 8166fefe7..fc33fb45a 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -71,7 +71,7 @@ def website_browser_desktop(tmp_path_factory): @pytest.fixture(scope="module") def browser_computer(tmp_path_factory): - """agentenv's chromium, which deploys alone as a browser computer: the browser-24.04 base, with no display, and the + """agentenv's chromium, which deploys alone as a browser computer: the ubuntu-24.04-headless base, with no display, and the page driver, whose screen is the browser's page; composed, through the browser-computer test bundle's task that deploys it, as ``desktop`` is.""" yield from _composed(tmp_path_factory.mktemp("browser-computer"), parse_bundle(BROWSER_COMPUTER), CHROMIUM_ALONE) diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index 7933685ff..da1246a5c 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -254,7 +254,7 @@ def test_verify_record_refuses_a_digest_the_registry_doesnt_serve_under_its_tag( @pytest.mark.parametrize("record, problem", [ - ({"images": []}, "records no agentenv-browser-24.04, agentenv-claude-code, agentenv-codex"), + ({"images": []}, "records no agentenv-claude-code, agentenv-codex, agentenv-db-mcp"), ({"images": [], "extra": 1}, "it holds ['extra', 'images']"), ({"images": [{"repository": "ghcr.io/someone/thing", "tag": _TAG, "digest": _DIGEST}]}, "'ghcr.io/someone/thing' isn't an image agent-env ships"), diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index f7d7f3838..f1b7be2ab 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -76,7 +76,7 @@ def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(sc assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", "agentenv-pydantic-ai", - "agentenv-mail-mcp", "agentenv-ubuntu-24.04", "agentenv-browser-24.04"] + "agentenv-mail-mcp", "agentenv-ubuntu-24.04", "agentenv-ubuntu-24.04-headless"] def test_plan_missing_lists_the_images_the_registry_lacks_or_refuses_to_show(script, monkeypatch, capsys):