From e9937854cb9075e2ada5b1036a5d2fd7fc8342c3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 17:45:34 -0800 Subject: [PATCH 1/3] feat(computer): a browser computer's page tools: navigate, history, tabs, dialogs and uploads, and every frame's page The protocol gains an optional page group, declared by computer-page/v1: computer_navigate, computer_history, computer_tabs, computer_dialog and computer_upload, served by a driver that implements navigate, history, tabs, dialog, upload and page_state. Every frame such a driver captures carries its page, the active tab's id, URL and title and the dialog or file chooser open on it, and its summary names them; while a dialog is open, the input it blocks is refused dialog_open. browser/v1 gains navigate, history, tabs/list, tabs/open, tabs/activate, tabs/close, dialog/answer, files/choose and page/state, and the page driver serves the group over them. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 32 ++- .../agentenv_protocol/computer/__init__.py | 21 +- .../src/agentenv_protocol/computer/card.py | 8 + .../src/agentenv_protocol/computer/errors.py | 4 + .../src/agentenv_protocol/computer/frames.py | 32 ++- .../agentenv_protocol/computer/operations.py | 52 ++++- .../src/agentenv_protocol/computer/results.py | 31 +++ .../src/agentenv_protocol/computer/runtime.py | 30 ++- .../src/agentenv_protocol/computer/tools.py | 92 +++++++- .../src/agentenv_protocol/website_browser.py | 214 +++++++++++++++++- .../tests/test_computer_page.py | 144 ++++++++++++ .../tests/test_website_browser.py | 160 ++++++++++++- .../ubuntu-24.04-headless/Dockerfile | 11 +- src/agent_env/drivers/README.md | 4 +- src/agent_env/drivers/envs/page/driver.py | 60 ++++- .../computer/browser_computer_test.py | 114 +++++++++- tst/unit/drivers/page_driver_test.py | 52 ++++- 17 files changed, 1012 insertions(+), 49 deletions(-) create mode 100644 packages/agentenv-protocol/tests/test_computer_page.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 33b09b2fe..3c4c68f22 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -473,6 +473,7 @@ pixels), at the display's size times its scale. Input operations may return an ` | `@set_display` | no, with `display_modes` | `computer-display/v1` `set` (`get` is always served) | | `@pointer_move`, `@pointer_down`, `@pointer_up`, `@pointer_position` | all or none | `computer-pointer/v1` and its five tools; click and scroll at the pointer | | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | +| `@navigate`, `@history`, `@tabs`, `@dialog`, `@upload`, `@page_state` | all or none, for a screen that is a website browser's page | `computer-page/v1` and its five tools; the `page` of every frame | | `@run_in_session` | no; a Linux default | `computer-exec/v1` | | `@write_files` | no; a Linux default | `data/add` | | `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` | @@ -497,13 +498,21 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope group tools. Every result is structured, with a one-line summary as its text block; input results carry `applied`, `delivery` and `others_since_frame`, and `screenshot: true` returns a fresh frame after acting. Click and scroll leave out `x` and `y` to act at the pointer on a computer with - `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. + `computer-pointer/v1`; elsewhere a call without them is refused `invalid_argument`. With the page + group, `computer_navigate` opens a URL in the active tab, `computer_history` goes `back`, `forward` + or `reload`s, `computer_tabs` lists, opens, switches and closes tabs by the browser's own ids and + answers with the tabs after it, `computer_dialog` answers the page's open dialog and + `computer_upload` gives its open file chooser files in `~/Uploads`. - **Frames and spaces.** Points are pixels of a named frame (your latest by default), or a 0..`norm_scale` grid over it with `space: "norm"`. A point outside the frame or the range is refused (`out_of_bounds`), never clamped. The last 64 frames are kept, and a frame goes stale - (`stale_frame`) when the display geometry it was captured at changes. + (`stale_frame`) when the display geometry it was captured at changes. With the page group, every + frame carries `page`: the active tab's `tab`, `url` and `title`, and the `dialog` (`type`, `message`, + `default_text`) or `file_chooser` (`multiple`) open on it, which its summary names too. - **The input queue.** Input calls run one at a time in arrival order; one that waits past - `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. + `queue_wait_s` is refused `input_busy`. Screenshots and waits never queue. While a dialog is open + on the page, input it blocks (all but `computer_tabs`, `computer_dialog` and `computer_upload`) is + refused `dialog_open`, with `applied: "no"`. - **The computer skill.** `computer/v1` declares the seven tools, its params the driver's features, among them `screen`: `desktop`, or `page` for a driver whose screen is a website browser's active tab, which a computer env pairs with exactly one website browser. The driver serves its skill through `skill/v1` (an `@agent_skill` handler on the base @@ -799,7 +808,8 @@ are the profile's template. The base class adds: each as of its page's last load, with `closed_at` and `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state - and, completed, its size and sha256; and `dialogs`, each with its type, message, answer and who gave it. + and, completed, its size and sha256; and `dialogs`, each with its type, message, default text, answer + and who gave it. `hidden_page()` gives agentenv a page of its own for reading state, in no window and never in the record. - **`browser/v1`** on its card: `chromium` (its `version`), `clock: "real"` and, alone, `window`; and `record` (GET), framework-only, which returns that record with `launches`, the app server's log of each @@ -808,8 +818,18 @@ are the profile's template. The base class adds: Nothing relaunches a browser that exits on its own: in a computer the agent can, and alone it's an infra failure, its ready check false. `page/viewport`, `page/capture` and `page/input`, framework-only, are its active tab's page as a page driver sees it: its size, which headless is its window's; a PNG of it, - or `blank` for a tab that paints nothing within 3 s (1 s with no URL yet); and CDP input calls - (`Input.dispatchMouseEvent`, `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. + or `blank` for a tab that paints nothing within 3 s (1 s with no URL yet), its last capture while + a dialog keeps it from painting; and CDP input calls (`Input.dispatchMouseEvent`, + `Input.dispatchKeyEvent`, `Input.insertText`), sent to it in order. Also framework-only, for page + drivers and agentenv's own steps alike: + - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab; + - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab + ids. The active tab is the one a dialog is open on, else the one whose page has focus, else the one + agentenv last brought to the front; + - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab; + - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so + alone, it intercepts; + - `page/state`, the active tab's id, URL and title and the dialog or file chooser open on it. - **Its skill**: sites by their real names, files to upload in `~/Uploads` and downloads in `~/Downloads`, folders it makes. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 586e962cc..1825f368c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -32,6 +32,7 @@ ComputerError, DeliveryFailed, DeviceUnavailable, + DialogOpen, InputBusy, Internal, InvalidArgument, @@ -47,14 +48,22 @@ Display, Effect, ExecResult, + FileChooser, Image, + PageDialog, + PageState, ReadyCheck, + Tab, capture, click, + dialog, display, drag, + history, key_down, key_up, + navigate, + page_state, pointer_down, pointer_move, pointer_position, @@ -64,7 +73,9 @@ run_in_session, scroll, set_display, + tabs, type_text, + upload, write_files, ) from .dataplane import part_bytes @@ -79,6 +90,7 @@ COMPUTER_DISPLAY, COMPUTER_EXEC, COMPUTER_KEYBOARD, + COMPUTER_PAGE, COMPUTER_POINTER, COMPUTER_READY, COMPUTER_RECORDING, @@ -91,7 +103,7 @@ _ON_USE = { "driver": ("ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration"), "card": ("COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", - "COMPUTER_KEYBOARD", "COMPUTER_EXEC", "COMPUTER_RECORDING"), + "COMPUTER_KEYBOARD", "COMPUTER_PAGE", "COMPUTER_EXEC", "COMPUTER_RECORDING"), "recording": ("Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread"), } @@ -110,14 +122,17 @@ def __getattr__(name: str) -> Any: "AppDeclaration", "AppOs", "APP_SETTINGS_ENV", "part_bytes", "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", + "Tab", "PageState", "PageDialog", "FileChooser", "PRINTABLE_KEYS", "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", + "navigate", "history", "tabs", "dialog", "upload", "page_state", "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread", "RecordingManifest", "ManifestSegment", "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", - "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "LimitExceeded", "Timeout", "DeliveryFailed", + "BackgroundUnavailable", "TargetUnavailable", "DeviceUnavailable", "DialogOpen", "LimitExceeded", "Timeout", + "DeliveryFailed", "Internal", "COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", "COMPUTER_KEYBOARD", - "COMPUTER_EXEC", "COMPUTER_RECORDING", + "COMPUTER_PAGE", "COMPUTER_EXEC", "COMPUTER_RECORDING", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py index 20e46fd21..f83647e2f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/card.py @@ -18,6 +18,7 @@ COMPUTER_SPACES = "urn:agentenv:computer-spaces/v1" COMPUTER_POINTER = "urn:agentenv:computer-pointer/v1" COMPUTER_KEYBOARD = "urn:agentenv:computer-keyboard/v1" +COMPUTER_PAGE = "urn:agentenv:computer-page/v1" COMPUTER_EXEC = "urn:agentenv:computer-exec/v1" COMPUTER_RECORDING = "urn:agentenv:computer-recording/v1" @@ -29,6 +30,7 @@ "pointer": ("computer_pointer_move", "computer_pointer_down", "computer_pointer_up", "computer_pointer_position", "computer_pointer_list"), "keyboard": ("computer_keyboard_down", "computer_keyboard_up"), + "page": ("computer_navigate", "computer_history", "computer_tabs", "computer_dialog", "computer_upload"), } #: The tools whose description names the default coordinate space. POINT_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_pointer_move", @@ -101,6 +103,12 @@ def declared_extensions(driver: ComputerDriverEnvironment) -> list[EnvironmentEx declared.append(EnvironmentExtension( uri=COMPUTER_POINTER, description="The computer_pointer_* tools, and click and scroll at the pointer.", params={"buttons": list(driver.buttons), "kinds": ["foreground"]})) + if driver.has("page"): + declared.append(EnvironmentExtension( + uri=COMPUTER_PAGE, description="The page tools, for a screen that is a website browser's active tab: " + "computer_navigate, computer_history, computer_tabs, computer_dialog and " + "computer_upload; each frame names the page it shows.", + params={"uploads": "~/Uploads"})) intake = IntakeDeclaration(get=[IntakeFormat(part="file", format="file-tree")]) if driver.has("write_files"): intake.add = [IntakeFormat(part="file", format="file-tree", load="additive")] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py index 854b992ec..315bf7ee6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/errors.py @@ -55,6 +55,10 @@ class DeviceUnavailable(ComputerError): CODE, STATUS, APPLIED, RETRYABLE = "device_unavailable", 503, "no", True +class DialogOpen(ComputerError): + CODE, STATUS, APPLIED = "dialog_open", 409, "no" + + class LimitExceeded(ComputerError): CODE, STATUS, APPLIED, RETRYABLE = "limit_exceeded", 429, "no", True diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py index fa004dd82..b611fe0c9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/frames.py @@ -11,8 +11,8 @@ from PIL import Image as PILImage from .errors import Internal, StaleFrame -from .operations import Display, Image -from .results import DisplayData, FrameData +from .operations import Display, Image, PageState +from .results import DialogData, DisplayData, FileChooserData, FrameData, PageData KEEP = 64 #: The JPEG quality a screenshot answers with when it names none. @@ -34,6 +34,8 @@ class Frame: parent_id: str | None captured_at: datetime inputs: tuple = field(repr=False) + #: The page it shows, on a computer whose screen is a website browser's active tab. + page: PageState | None = None @property def width(self) -> int: @@ -71,8 +73,21 @@ def result(self) -> FrameData: display=DisplayData(width=self.display.width, height=self.display.height, scale=self.display.scale), parent_frame_id=self.parent_id, captured_at=self.captured_at.isoformat().replace("+00:00", "Z"), + page=_page_data(self.page), ) + def note(self) -> str: + """What a summary adds for the page it shows: its title and URL, and an open dialog or file chooser.""" + if self.page is None: + return "" + page = self.page + note = f' on "{page.title}" ({page.url})' + if page.dialog is not None: + note += f'; a {page.dialog.type} dialog is open: "{page.dialog.message}"' + if page.file_chooser is not None: + note += f"; a file chooser is open, for {'files' if page.file_chooser.multiple else 'one file'}" + return note + def encode(self, fmt: str = "png", quality: int | None = None) -> tuple[str, str]: """The frame as base64 and its MIME type.""" if fmt != "jpeg": @@ -92,7 +107,7 @@ def __init__(self, keep: int = KEEP) -> None: self._latest: dict[str, str] = {} def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identity: str, - parent: Frame | None = None, origin: tuple[int, int] = (0, 0)) -> Frame: + parent: Frame | None = None, origin: tuple[int, int] = (0, 0), page: PageState | None = None) -> Frame: frame = Frame( id=f"f{self._next}", png=_png(image), @@ -103,6 +118,7 @@ def add(self, image: PILImage.Image, display: Display, inputs: tuple, *, identit parent_id=parent.id if parent is not None else None, captured_at=parent.captured_at if parent is not None else datetime.now(timezone.utc), inputs=parent.inputs if parent is not None else inputs, + page=parent.page if parent is not None else page, ) self._next += 1 self._frames[frame.id] = frame @@ -153,6 +169,16 @@ def to_pil(capture: object, display: Display) -> PILImage.Image: return image +def _page_data(page: PageState | None) -> PageData | None: + if page is None: + return None + dialog, chooser = page.dialog, page.file_chooser + return PageData(tab=page.tab, url=page.url, title=page.title, + dialog=DialogData(type=dialog.type, message=dialog.message, default_text=dialog.default_text) + if dialog is not None else None, + file_chooser=FileChooserData(multiple=chooser.multiple) if chooser is not None else None) + + def _png(image: PILImage.Image) -> bytes: buffer = io.BytesIO() image.save(buffer, format="PNG") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index b6ead0141..fea11661d 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -12,6 +12,13 @@ key_down(keys), key_up(keys) run_in_session(argv, cwd, env, timeout_s) -> ExecResult write_files({path: bytes}) record(hints) -> an async iterator of RecordedFrame or RecordedSegment, until it is closed + +The page group, for a driver whose screen is a website browser's active tab: + + navigate(url) history(direction), back, forward or reload + tabs(action, tab, url) -> [Tab, ...] list, open, switch or close; the tabs open after it + dialog(accept, text) upload(paths), relative to the session user's ~/Uploads + page_state() -> PageState """ from __future__ import annotations @@ -25,10 +32,13 @@ GROUPS = { "pointer": ("pointer_move", "pointer_down", "pointer_up", "pointer_position"), "keyboard": ("key_down", "key_up"), + "page": ("navigate", "history", "tabs", "dialog", "upload", "page_state"), } OPTIONAL = ("set_display", "run_in_session", "write_files", "record") INPUT = frozenset({"click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", - "key_down", "key_up"}) + "key_down", "key_up", "navigate", "history", "tabs", "dialog", "upload"}) +#: The input a page's open dialog blocks, which is refused while one is open there. +DIALOG_BLOCKS = INPUT - {"tabs", "dialog", "upload"} _OP_ATTR = "_aec_operation" @@ -60,6 +70,40 @@ class Effect: detail: str | None = None +@dataclass(frozen=True) +class Tab: + """A tab of a website browser: its id, the browser's own, and its page's URL and title.""" + id: str + url: str + title: str + active: bool = False + + +@dataclass(frozen=True) +class PageDialog: + """A JavaScript dialog open on a page: ``alert``, ``confirm``, ``prompt`` or ``beforeunload``.""" + type: str + message: str + default_text: str = "" + + +@dataclass(frozen=True) +class FileChooser: + """A file chooser a page opened, waiting for files: several, or one.""" + multiple: bool = False + + +@dataclass(frozen=True) +class PageState: + """The page a page driver's screen shows: the active tab's id, URL and title, the dialog open on it, and the file + chooser it opened that waits.""" + tab: str + url: str + title: str + dialog: PageDialog | None = None + file_chooser: FileChooser | None = None + + @dataclass(frozen=True) class ExecResult: exit_code: int | None @@ -115,6 +159,12 @@ def deco(f: Callable) -> Callable: pointer_position = _marker("pointer_position") key_down = _marker("key_down") key_up = _marker("key_up") +navigate = _marker("navigate") +history = _marker("history") +tabs = _marker("tabs") +dialog = _marker("dialog") +upload = _marker("upload") +page_state = _marker("page_state") run_in_session = _marker("run_in_session") write_files = _marker("write_files") record = _marker("record") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py index 5794b9f78..684bb2485 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/results.py @@ -16,6 +16,25 @@ class DisplayData(BaseModel): scale: float +class DialogData(BaseModel): + type: str + message: str + default_text: str = "" + + +class FileChooserData(BaseModel): + multiple: bool + + +class PageData(BaseModel): + """The page a frame shows, on a computer whose screen is a website browser's active tab.""" + tab: str + url: str + title: str + dialog: Optional[DialogData] = None + file_chooser: Optional[FileChooserData] = None + + class FrameData(BaseModel): frame_id: str width: int @@ -23,6 +42,7 @@ class FrameData(BaseModel): display: DisplayData parent_frame_id: Optional[str] = None captured_at: str + page: Optional[PageData] = None class ScreenshotResult(FrameData): @@ -53,6 +73,17 @@ class InputResult(BaseModel): frame: Optional[FrameData] = None +class TabData(BaseModel): + tab: str + url: str + title: str + active: bool + + +class TabsResult(InputResult): + tabs: list[TabData] + + class WaitResult(BaseModel): summary: str waited_ms: int diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 51888f16a..e220aeb5e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -14,9 +14,9 @@ from ..errors import AgentEnvError from ..tool_context import ToolContext from . import apps -from .errors import DeliveryFailed, DeviceUnavailable, InputBusy, Internal +from .errors import DeliveryFailed, DeviceUnavailable, DialogOpen, InputBusy, Internal from .frames import Frame, FrameStore, to_pil -from .operations import GROUPS, INPUT, Display, ReadyCheck, resolve +from .operations import DIALOG_BLOCKS, GROUPS, INPUT, Display, PageState, ReadyCheck, resolve from .queue import InputQueue from .recording import RecordHints, Recordings from .results import EffectData, InputResult, send @@ -81,7 +81,14 @@ async def capture(self, identity: str) -> Frame: inputs = self.queue.snapshot() display = await self.display_now() image = to_pil(await self.call("capture"), display) - return self.frames.add(image, display, inputs, identity=identity) + page = await self.page_now() if self.has("page") else None + return self.frames.add(image, display, inputs, identity=identity, page=page) + + async def page_now(self) -> PageState: + page = await self.call("page_state") + if not isinstance(page, PageState): + raise Internal(f"@page_state returned {page!r}, not a PageState") + return page async def pointer(self) -> tuple[int, int]: where = await self.call("pointer_position") @@ -93,15 +100,20 @@ async def pointer(self) -> tuple[int, int]: async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Frame | None, screenshot: bool, summary: str, mapped: bool = False, at_pointer: bool = False, repeat: int = 1, - hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None) -> CallToolResult: + hold: tuple | None = None, move_to: tuple | None = None, then: Callable | None = None, + wrap: Callable[[InputResult, Any], InputResult] | None = None) -> CallToolResult: """One input call: its turn in the queue, the operation (after a move, repeated or held), then the result with others' input since ``frame`` and, when asked, a fresh frame. Whatever the call's turn can change is read once it has the turn: a point ``mapped`` from ``frame`` is checked again, since the display may have changed - while it waited, and ``at_pointer`` reads the pointer and puts it before ``args``. ``then`` runs as soon as - the input is delivered.""" + while it waited, and ``at_pointer`` reads the pointer and puts it before ``args``; input a page's open dialog + blocks is refused. ``then`` runs as soon as the input is delivered, and ``wrap`` makes the result from the + operation's own.""" identity = self.driver.caller_identity(ctx) sent = False async with self.queue.turn(self.driver.settings.queue_wait_s): + if operation in DIALOG_BLOCKS and self.has("page") and (dialog := (await self.page_now()).dialog): + raise DialogOpen(f'a {dialog.type} dialog is open on the page: "{dialog.message}"; answer it with ' + "computer_dialog first") if mapped and frame is not None: self.frames.check_current(frame, await self.display_now()) if at_pointer: @@ -129,14 +141,16 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra then() result = InputResult(summary=summary, applied="yes", delivery="foreground", others_since_frame=others, effect=EffectData.of(effect)) + if wrap is not None: + result = wrap(result, effect) if not screenshot: return send(result) try: new = await self.capture(identity) except AgentEnvError as e: - result.summary = f"{summary}; the screenshot after it failed ({e.code}: {e.message})" + result.summary = f"{result.summary}; the screenshot after it failed ({e.code}: {e.message})" return send(result) - result.summary, result.frame = f"{summary}, then captured {new.id}", new.result() + result.summary, result.frame = f"{result.summary}, then captured {new.id}{new.note()}", new.result() return send(result, new.encode()) async def _hold_for(self, ms: int, keys: list[str]) -> None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py index 9b99c91b7..17ad4e31c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/tools.py @@ -21,6 +21,8 @@ PointerListResult, PointerPositionResult, ScreenshotResult, + TabData, + TabsResult, WaitResult, send, ) @@ -87,7 +89,8 @@ async def computer_screenshot( if frame_id is not None: raise InvalidArgument("frame_id names the frame a region crops; give a region too") frame = await computer.capture(identity) - summary = f"Captured {frame.id}: {frame.width}×{frame.height} of the {_display(frame)} display" + summary = (f"Captured {frame.id}: {frame.width}×{frame.height} of the {_display(frame)} display" + f"{frame.note()}") else: parent = computer.frames.get(frame_id, identity) computer.frames.check_current(parent, await computer.display_now()) @@ -236,7 +239,8 @@ async def computer_wait( if not screenshot: return send(WaitResult(summary=summary, waited_ms=ms)) frame = await self._computer.capture(self.caller_identity(ctx)) - return send(WaitResult(summary=f"{summary}, then captured {frame.id}", waited_ms=ms, frame=frame.result()), + return send(WaitResult(summary=f"{summary}, then captured {frame.id}{frame.note()}", waited_ms=ms, + frame=frame.result()), frame.encode()) @tool(annotations=_INPUT) @@ -359,6 +363,90 @@ async def computer_keyboard_up( screenshot=screenshot, summary=f"Released {'+'.join(names)}", then=lambda: computer.release(identity, names)) + @tool(annotations=_INPUT) + async def computer_navigate( + self, + ctx: ToolContext, + url: Annotated[str, Field(description="The URL to open, as you'd type it into the address bar")], + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Open a URL in the active tab. The screen is the page alone, with no address bar to type it into.""" + return await _page_action(self, ctx, "navigate", (url,), screenshot, f"Opened {url}") + + @tool(annotations=_INPUT) + async def computer_history( + self, + ctx: ToolContext, + direction: Annotated[Literal["back", "forward", "reload"], Field( + description="back or forward in the active tab's history, or reload its page")], + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Go back, go forward or reload in the active tab, as the browser's buttons do.""" + verb = {"back": "Went back", "forward": "Went forward", "reload": "Reloaded the page"}[direction] + return await _page_action(self, ctx, "history", (direction,), screenshot, verb) + + @tool(annotations=_INPUT) + async def computer_tabs( + self, + ctx: ToolContext, + action: Annotated[Literal["list", "open", "switch", "close"], Field( + description="list the tabs; open one, at url or blank, which becomes active; switch to tab; close tab")], + tab: Annotated[Optional[str], Field(description="The tab to switch to or close, by its id from list")] = None, + url: Annotated[Optional[str], Field(description="For open: the URL the new tab opens")] = None, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, TabsResult]: + """List, open, switch and close the browser's tabs: the screen is the active one.""" + if (tab is None) == (action in ("switch", "close")): + raise InvalidArgument("switch and close take tab, from list; list and open take none") + if url is not None and action != "open": + raise InvalidArgument("only open takes url") + verb = {"list": "Listed the tabs", "open": f"Opened a tab{f' at {url}' if url else ''}", + "switch": f"Switched to tab {tab}", "close": f"Closed tab {tab}"}[action] + + def listed(result: InputResult, tabs: Any) -> TabsResult: + rows = [TabData(tab=t.id, url=t.url, title=t.title, active=t.active) for t in tabs or []] + active = next((row for row in rows if row.active), None) + shown = f', the active one "{active.title}" ({active.url})' if active is not None else "" + return TabsResult(**{**result.model_dump(), "summary": f"{verb}; {len(rows)} open{shown}" + if rows else f"{verb}; none open"}, tabs=rows) + + return await _page_action(self, ctx, "tabs", (action, tab, url), screenshot, verb, wrap=listed) + + @tool(annotations=_INPUT) + async def computer_dialog( + self, + ctx: ToolContext, + accept: Annotated[bool, Field(description="true for OK, false for Cancel")], + text: Annotated[Optional[str], Field(description="For a prompt dialog: the text to answer with")] = None, + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Answer the JavaScript dialog open on the page, which a screenshot names; nothing else reaches the page + until it's answered.""" + return await _page_action(self, ctx, "dialog", (accept, text), screenshot, + "Accepted the dialog" if accept else "Dismissed the dialog") + + @tool(annotations=_INPUT) + async def computer_upload( + self, + ctx: ToolContext, + paths: Annotated[list[str], Field(description="Files in ~/Uploads, by their paths there")], + screenshot: Screenshot = False, + ) -> Annotated[CallToolResult, InputResult]: + """Choose files in the file chooser the page opened, which a screenshot names, as picking them in its dialog + does.""" + if not paths: + raise InvalidArgument("give at least one path") + return await _page_action(self, ctx, "upload", (list(paths),), screenshot, + f"Chose {', '.join(paths)} in the file chooser") + + +async def _page_action(driver: Tools, ctx: ToolContext, op: str, args: tuple, screenshot: bool, summary: str, + wrap: Callable | None = None) -> CallToolResult: + computer = driver._computer + await computer.ensure_ready() + return await computer.act(ctx, op, args, frame=computer.frames.latest(driver.caller_identity(ctx)), + screenshot=screenshot, summary=summary, wrap=wrap) + async def _button_action(driver: Tools, ctx: ToolContext, op: str, verb: str, button: str | None, x: float | None, y: float | None, frame_id: str | None, screenshot: bool, space: str | None, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index c24cdd2a9..a807a3b13 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -55,6 +55,8 @@ #: The CDP input a computer's page driver may send to the active tab, and how long its screenshot may take. PAGE_INPUT = frozenset({"Input.dispatchMouseEvent", "Input.dispatchKeyEvent", "Input.insertText"}) CAPTURE_TIMEOUT_S, UNNAVIGATED_CAPTURE_TIMEOUT_S = 3.0, 1.0 +#: Where files a page's file chooser may be given come from, under the session user's home. +UPLOADS = "Uploads" #: Alone, loopback goes through the proxy too. In a computer, the apps on loopback beside it stay reachable; ``<-loopback>`` #: removes Chromium's own rule, so ``*.localhost`` names still don't. ALONE_BYPASS = "<-loopback>" @@ -204,6 +206,14 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl self._closing: set[str] = set() self._open_dialogs: dict[str, dict] = {} self._answering: set[str] = set() + #: The file chooser each page's session opened that waits, headless, and the tab agentenv last brought to the + #: front. + self._choosers: dict[str, dict] = {} + self._front: Optional[str] = None + #: Each tab's latest capture, which shows while a dialog keeps its page from painting; and, by session, what + #: input sent there waits on to learn a dialog opened. + self._shots: dict[str, str] = {} + self._dialog_opened: dict[str, asyncio.Event] = {} self._tasks: set[asyncio.Future] = set() devtools.listen(self._event) @@ -225,12 +235,117 @@ def ended(self, how: str) -> None: self.tabs.clear() self._sessions.clear() self._open_dialogs.clear() + self._choosers.clear() + self._shots.clear() + self._dialog_opened.clear() async def close_tab(self, target: str) -> None: """Close the tab ``target``, which the record says agentenv closed.""" self._closing.add(target) await self.devtools.call("Target.closeTarget", {"targetId": target}) + async def tab_list(self) -> list[dict]: + """The open tabs in opened order, each with its URL and title now and whether it's active.""" + await self._refresh() + active = await self.active_tab() + return [{"id": tab["id"], "url": tab["url"], "title": tab["title"], "active": tab["id"] == active} + for tab in self.tabs.values()] + + async def open_tab(self, url: Optional[str] = None) -> str: + """A new tab at ``url``, else blank, brought to the front; its id.""" + target = (await self.devtools.call("Target.createTarget", {"url": url or "about:blank"}))["targetId"] + await self._bring_to_front(target) + return target + + async def activate(self, target: str) -> None: + """The open tab ``target`` brought to the front, the active one.""" + self._open(target) + await self._bring_to_front(target) + + async def close(self, target: str) -> None: + """The open tab ``target`` closed, which the record says agentenv closed.""" + self._open(target) + await self.close_tab(target) + + async def navigate(self, url: str) -> None: + """The active tab opened at ``url``; InvalidArgument, with the browser's reason, when it can't.""" + answer = await self.devtools.call("Page.navigate", {"url": url}, session_id=await self.active_session()) + if error := answer.get("errorText"): + raise InvalidArgument(f"the browser couldn't open {url}: {error}") + + async def history(self, direction: str) -> None: + """The active tab's page reloaded, or the one before or after it in its history opened.""" + session = await self.active_session() + if direction == "reload": + await self.devtools.call("Page.reload", {}, session_id=session) + return + if direction not in ("back", "forward"): + raise InvalidArgument(f"direction is back, forward or reload, not {direction!r}") + history = await self.devtools.call("Page.getNavigationHistory", session_id=session) + index = history["currentIndex"] + (-1 if direction == "back" else 1) + if not 0 <= index < len(history["entries"]): + raise InvalidArgument(f"the active tab has no page to go {direction} to") + await self.devtools.call("Page.navigateToHistoryEntry", {"entryId": history["entries"][index]["id"]}, + session_id=session) + + async def answer_dialog(self, accept: bool, text: Optional[str] = None) -> None: + """The dialog open on the active tab answered, as the agent's answer; InvalidArgument with none open.""" + active = await self.active_tab() + session = next((s for s, dialog in self._open_dialogs.items() if dialog["tab"] == active), None) + if session is None: + raise InvalidArgument("no dialog is open on the page") + await self.devtools.call("Page.handleJavaScriptDialog", { + "accept": accept, **({"promptText": text} if text is not None else {})}, session_id=session) + + async def choose_files(self, paths: list[str]) -> None: + """The file chooser open on the active tab given ``paths``, files in the session user's ``~/Uploads``; + InvalidArgument for one outside it or missing, for several where it takes one, or with no chooser open.""" + active = await self.active_tab() + session = next((s for s in self._choosers if self._sessions.get(s) == active), None) + if session is None: + raise InvalidArgument("no file chooser is open on the page") + uploads = (Path.home() / UPLOADS).resolve() + files = [] + for path in paths: + file = (uploads / path).resolve() + if not file.is_relative_to(uploads): + raise InvalidArgument(f"{path} isn't in ~/{UPLOADS}") + if not file.is_file(): + raise InvalidArgument(f"there's no file {path} in ~/{UPLOADS}") + files.append(str(file)) + chooser = self._choosers[session] + if len(files) > 1 and not chooser["multiple"]: + raise InvalidArgument("the file chooser takes one file") + await self.devtools.call("DOM.setFileInputFiles", {"files": files, "backendNodeId": chooser["node"]}, + session_id=session) + self._choosers.pop(session, None) + + async def state(self) -> dict: + """The active tab's id, URL and title now, and the dialog and file chooser open on it.""" + await self._refresh() + active = await self.active_tab() + if active is None: + raise DeviceUnavailable("the browser has no tab open") + tab = self.tabs[active] + state: dict[str, Any] = {"tab": active, "url": tab["url"], "title": tab["title"]} + if dialog := next((d for d in self._open_dialogs.values() if d["tab"] == active), None): + state["dialog"] = {"type": dialog["type"], "message": dialog["message"], + "default_text": dialog["default_text"]} + if chooser := next((c for s, c in self._choosers.items() if self._sessions.get(s) == active), None): + state["file_chooser"] = {"multiple": chooser["multiple"]} + return state + + def _open(self, target: str) -> None: + if target not in self.tabs: + raise InvalidArgument(f"no tab {target} is open; list the tabs for their ids") + + async def _bring_to_front(self, target: str) -> None: + await self.devtools.call("Target.activateTarget", {"targetId": target}) + if (session := next((s for s, t in self._sessions.items() if t == target), None)) is not None: + with contextlib.suppress(DevToolsError): + await self.devtools.call("Page.bringToFront", session_id=session) + self._front = target + @contextlib.asynccontextmanager async def hidden_page(self, url: str = "about:blank") -> AsyncIterator[str]: """A page of agentenv's own for reading state, in no window and never in the record; its session's id. It's @@ -246,7 +361,10 @@ async def hidden_page(self, url: str = "about:blank") -> AsyncIterator[str]: await self.devtools.call("Target.closeTarget", {"targetId": target}) async def active_tab(self) -> Optional[str]: - """The tab whose page has focus, else the first visible one; CDP has no event for a tab made active.""" + """The tab a dialog is open on, whose page it blocks from saying; else the tab whose page has focus, else the + one agentenv last brought to the front, else the first visible one: CDP has no event for a tab made active.""" + if dialogs := [dialog["tab"] for dialog in self._open_dialogs.values() if dialog["tab"] in self.tabs]: + return dialogs[-1] visible = None for session, target in list(self._sessions.items()): if target not in self.tabs: @@ -262,7 +380,7 @@ async def active_tab(self) -> Optional[str]: return target if state[0] == "visible" and visible is None: visible = target - return visible + return self._front if self._front in self.tabs else visible async def record(self) -> dict: """The record ``since`` it started: open ``tabs``, each with its URL and title now and whether it's ``active``, @@ -286,22 +404,41 @@ async def capture(self) -> Optional[str]: ``CAPTURE_TIMEOUT_S``, which a desktop browser shows blank, or sooner for a tab with no URL yet, such as a new tab whose navigation the browser blocked, which may never paint.""" session = await self.active_session() - navigated = self.tabs.get(self._sessions[session], {}).get("url") + target = self._sessions[session] + navigated = self.tabs.get(target, {}).get("url") try: shot = await self.devtools.call("Page.captureScreenshot", {"format": "png", "optimizeForSpeed": True}, session_id=session, timeout_s=CAPTURE_TIMEOUT_S if navigated else UNNAVIGATED_CAPTURE_TIMEOUT_S) except asyncio.TimeoutError: - return None + return self._shots.get(target) if session in self._open_dialogs else None + self._shots[target] = shot["data"] return shot["data"] async def dispatch(self, events: list[dict]) -> None: - """Each of ``events``, a CDP input call, sent to the active tab in order; InvalidArgument for any other call.""" + """Each of ``events``, a CDP input call, sent to the active tab in order, until one opens a dialog: the page + answers that one only once the dialog is answered, so it finishes then, and the rest aren't sent. + InvalidArgument for any other call.""" if refused := [event.get("method") for event in events if event.get("method") not in PAGE_INPUT]: raise InvalidArgument(f"page input takes {', '.join(sorted(PAGE_INPUT))}, not {refused}") session = await self.active_session() + opened = self._dialog_opened.setdefault(session, asyncio.Event()) for event in events: - await self.devtools.call(event["method"], event.get("params") or {}, session_id=session) + if session in self._open_dialogs: + return + opened.clear() + sent = asyncio.ensure_future(self.devtools.call(event["method"], event.get("params") or {}, + session_id=session)) + waiting = asyncio.ensure_future(opened.wait()) + try: + await asyncio.wait({sent, waiting}, return_when=asyncio.FIRST_COMPLETED) + finally: + waiting.cancel() + if not sent.done(): + self._tasks.add(sent) + sent.add_done_callback(self._finished) + return + sent.result() async def _refresh(self, target: Optional[str] = None) -> None: """The URL and title of the tab ``target``, or of every open tab, as the browser has them now: it sends no @@ -353,21 +490,34 @@ async def _let_run(self, session: str) -> None: width, height = self.viewport await self.devtools.call("Emulation.setDeviceMetricsOverride", { "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) + await self.devtools.call("Page.setInterceptFileChooserDialog", {"enabled": True}, session_id=session) finally: await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: - self._sessions.pop(params["sessionId"], None) + if (target := self._sessions.pop(params["sessionId"], None)) is not None: + self._shots.pop(target, None) + self._choosers.pop(params["sessionId"], None) + self._dialog_opened.pop(params["sessionId"], None) def _on_Page_loadEventFired(self, params: dict, session: Optional[str]) -> None: + self._choosers.pop(session, None) if (target := self._sessions.get(session)) is not None: self._spawn(self._refresh(target)) + def _on_Page_fileChooserOpened(self, params: dict, session: Optional[str]) -> None: + if session in self._sessions: + self._choosers[session] = {"node": params.get("backendNodeId"), + "multiple": params.get("mode") == "selectMultiple"} + def _on_Page_javascriptDialogOpening(self, params: dict, session: Optional[str]) -> None: dialog = {"tab": self._sessions.get(session), "type": params.get("type"), "message": params.get("message", ""), - "url": params.get("url", ""), "opened_at": _now(), "answer": None, "by": None} + "default_text": params.get("defaultPrompt", ""), "url": params.get("url", ""), "opened_at": _now(), + "answer": None, "by": None} self.dialog_log.append(dialog) self._open_dialogs[session] = dialog + if (opened := self._dialog_opened.get(session)) is not None: + opened.set() if self.dialogs != AGENT: self._answering.add(session) leave = params.get("type") == "beforeunload" @@ -618,7 +768,11 @@ async def devtools_answers(self) -> bool: "who closed them, its downloads with their size and sha256, its JavaScript dialogs and who answered each, and " "each launch, stop and exit of it since the computer booted, with its cause. page/viewport, page/capture and " "page/input are its active tab's page as a computer's page driver sees it: its size in pixels, which headless " - "is its window's, a PNG of it, and CDP input calls sent to it in order.")) + "is its window's, a PNG of it, and CDP input calls sent to it in order. navigate and history open a URL in " + "the active tab, or go back, forward or reload; tabs/list, tabs/open, tabs/activate and tabs/close manage " + "its tabs by their ids; dialog/answer answers the dialog open on the active tab; files/choose gives the file " + "chooser open on it files in ~/Uploads, headless; and page/state is the active tab's id, URL and title and " + "what is open on it.")) async def browser_record(self) -> dict: return {**await self.control.record(), "launches": self._computer_app.launches()} @@ -637,6 +791,48 @@ async def browser_page_input(self, events: list[dict]) -> dict: await self.control.dispatch(events) return {} + @extension(BROWSER, op="page/state", method="GET") + async def browser_page_state(self) -> dict: + return await self.control.state() + + @extension(BROWSER, op="navigate") + async def browser_navigate(self, url: str) -> dict: + await self.control.navigate(url) + return {} + + @extension(BROWSER, op="history") + async def browser_history(self, direction: str) -> dict: + await self.control.history(direction) + return {} + + @extension(BROWSER, op="tabs/list", method="GET") + async def browser_tabs_list(self) -> dict: + return {"tabs": await self.control.tab_list()} + + @extension(BROWSER, op="tabs/open") + async def browser_tabs_open(self, url: Optional[str] = None) -> dict: + return {"tab": await self.control.open_tab(url)} + + @extension(BROWSER, op="tabs/activate") + async def browser_tabs_activate(self, tab: str) -> dict: + await self.control.activate(tab) + return {} + + @extension(BROWSER, op="tabs/close") + async def browser_tabs_close(self, tab: str) -> dict: + await self.control.close(tab) + return {} + + @extension(BROWSER, op="dialog/answer") + async def browser_dialog_answer(self, accept: bool, text: Optional[str] = None) -> dict: + await self.control.answer_dialog(accept, text) + return {} + + @extension(BROWSER, op="files/choose") + async def browser_files_choose(self, paths: list[str]) -> dict: + await self.control.choose_files(paths) + return {} + def extension_params(self, uri: str) -> dict: if uri == BROWSER: params: dict[str, Any] = {"chromium": self.version, "clock": "real"} diff --git a/packages/agentenv-protocol/tests/test_computer_page.py b/packages/agentenv-protocol/tests/test_computer_page.py new file mode 100644 index 000000000..36e8dd8a1 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_page.py @@ -0,0 +1,144 @@ +"""The page group: a driver whose screen is a website browser's active tab serves computer-page/v1 and its tools, every +frame names the page it shows, computer_tabs answers with the tabs, and while a dialog is open on the page the input +it blocks is refused until computer_dialog answers it.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from computer_fake import FakeDriver, http_client, mcp_session # noqa: E402 + +from agentenv_protocol.computer import ( # noqa: E402 + FileChooser, + PageDialog, + PageState, + Tab, + dialog, + history, + navigate, + page_state, + tabs, + upload, +) + +PAGE_TOOLS = {"computer_navigate", "computer_history", "computer_tabs", "computer_dialog", "computer_upload"} + + +class PageFake(FakeDriver): + """A fake whose screen is a browser's page, with two tabs, the first active, and a log of the page calls.""" + + def __init__(self) -> None: + super().__init__() + self.tabs = [Tab("a", "https://mail.test/", "Inbox", True), Tab("b", "https://docs.test/", "Docs")] + self.open_dialog: PageDialog | None = None + self.chooser: FileChooser | None = None + self.calls: list = [] + + @navigate + async def go_to(self, url): + self.calls.append(("navigate", url)) + + @history + async def step(self, direction): + self.calls.append(("history", direction)) + + @tabs + async def tab_action(self, action, tab, url): + self.calls.append(("tabs", action, tab, url)) + if action == "switch": + self.tabs = [Tab(t.id, t.url, t.title, t.id == tab) for t in self.tabs] + return self.tabs + + @dialog + async def answer(self, accept, text): + self.calls.append(("dialog", accept, text)) + self.open_dialog = None + + @upload + async def choose(self, paths): + self.calls.append(("upload", paths)) + + @page_state + async def page_now(self): + active = next(t for t in self.tabs if t.active) + return PageState(active.id, active.url, active.title, self.open_dialog, self.chooser) + + +@pytest.mark.asyncio +async def test_a_page_driver_declares_computer_page_and_serves_its_tools_and_a_desktop_driver_neither(): + async with http_client(PageFake()) as http: + page = (await http.get("/.well-known/agent-env.json")).json() + async with http_client(FakeDriver()) as http: + desktop = (await http.get("/.well-known/agent-env.json")).json() + + def served(card): + capabilities = card["capabilities"] + return {e["uri"] for e in capabilities["extensions"]}, {t["name"] for t in capabilities["tools"]} + + page_uris, page_tools = served(page) + desktop_uris, desktop_tools = served(desktop) + assert "urn:agentenv:computer-page/v1" in page_uris and PAGE_TOOLS <= page_tools + assert "urn:agentenv:computer-page/v1" not in desktop_uris and not PAGE_TOOLS & desktop_tools + + +@pytest.mark.asyncio +async def test_every_frame_names_the_page_it_shows_and_what_is_open_on_it(): + driver = PageFake() + driver.chooser = FileChooser(multiple=True) + async with mcp_session(driver) as session: + shot = await session.call_tool("computer_screenshot", {}) + clicked = await session.call_tool("computer_click", {"x": 5, "y": 5, "screenshot": True}) + async with mcp_session(FakeDriver()) as session: + desktop = await session.call_tool("computer_screenshot", {}) + + page = {"tab": "a", "url": "https://mail.test/", "title": "Inbox", "file_chooser": {"multiple": True}} + assert shot.structuredContent["page"] == page and clicked.structuredContent["frame"]["page"] == page + assert shot.structuredContent["summary"].endswith(' on "Inbox" (https://mail.test/); a file chooser is open, ' + "for files") + assert ' on "Inbox" (https://mail.test/)' in clicked.structuredContent["summary"] + assert "page" not in desktop.structuredContent + + +@pytest.mark.asyncio +async def test_the_page_tools_reach_the_driver_and_computer_tabs_answers_with_the_tabs(): + driver = PageFake() + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + navigated = await session.call_tool("computer_navigate", {"url": "https://mail.test/sent"}) + await session.call_tool("computer_history", {"direction": "back"}) + switched = await session.call_tool("computer_tabs", {"action": "switch", "tab": "b"}) + refused = await session.call_tool("computer_tabs", {"action": "switch"}) + await session.call_tool("computer_upload", {"paths": ["cv.pdf"]}) + + assert navigated.structuredContent["applied"] == "yes" and navigated.structuredContent["summary"].startswith( + "Opened https://mail.test/sent") + assert switched.structuredContent["tabs"] == [ + {"tab": "a", "url": "https://mail.test/", "title": "Inbox", "active": False}, + {"tab": "b", "url": "https://docs.test/", "title": "Docs", "active": True}] + assert switched.structuredContent["summary"] == ( + 'Switched to tab b; 2 open, the active one "Docs" (https://docs.test/)') + assert refused.isError and refused.structuredContent["code"] == "invalid_argument" + assert driver.calls == [("navigate", "https://mail.test/sent"), ("history", "back"), ("tabs", "switch", "b", None), + ("upload", ["cv.pdf"])] + + +@pytest.mark.asyncio +async def test_while_a_dialog_is_open_input_it_blocks_is_refused_until_computer_dialog_answers_it(): + driver = PageFake() + driver.open_dialog = PageDialog("confirm", "Delete this message?") + async with mcp_session(driver) as session: + await session.call_tool("computer_screenshot", {}) + refused = await session.call_tool("computer_click", {"x": 5, "y": 5}) + navigated = await session.call_tool("computer_navigate", {"url": "https://mail.test/"}) + listed = await session.call_tool("computer_tabs", {"action": "list"}) + answered = await session.call_tool("computer_dialog", {"accept": False}) + clicked = await session.call_tool("computer_click", {"x": 5, "y": 5}) + + assert refused.structuredContent == { + "code": "dialog_open", "applied": "no", "retryable": False, + "message": 'a confirm dialog is open on the page: "Delete this message?"; answer it with computer_dialog first'} + assert navigated.structuredContent["code"] == "dialog_open" and not listed.isError + assert answered.structuredContent["summary"] == "Dismissed the dialog" and not clicked.isError + assert ("dialog", False, None) in driver.calls and not any(entry[0] == "click" for entry in driver.screen.log[:-1]) diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index eeadda2b3..11c59ed1f 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -514,7 +514,8 @@ async def test_alone_each_page_it_attaches_to_is_held_to_its_window_before_it_ru sent = [method for method, _, session in devtools.calls if session == "s-popup"] assert next(params for method, params, _ in devtools.calls if method == "Target.setAutoAttach")[ "waitForDebuggerOnStart"] is True - assert sent == ["Page.enable", "Emulation.setDeviceMetricsOverride", "Runtime.runIfWaitingForDebugger"] + assert sent == ["Page.enable", "Emulation.setDeviceMetricsOverride", "Page.setInterceptFileChooserDialog", + "Runtime.runIfWaitingForDebugger"] assert ("Emulation.setDeviceMetricsOverride", {"width": 1024, "height": 768, "deviceScaleFactor": 1, "mobile": False}, "s-popup") in devtools.calls @@ -590,6 +591,163 @@ async def test_its_page_input_goes_to_the_active_tab_in_order_and_takes_only_inp ("Input.insertText", {"text": "hi"}, "s-a")] +class Tabbed(Painting): + """A browser whose tabs paint and none has focus, with a history of two pages, the second current, and each call's + answer by method in ``answers``.""" + + def __init__(self, answers=None, blank=()): + super().__init__(blank) + self.answers.update({"Runtime.evaluate": lambda params, session: {"result": {"value": ["visible", False]}}, + "Page.getNavigationHistory": {"currentIndex": 1, "entries": [{"id": 7}, {"id": 8}]}, + "Target.createTarget": {"targetId": "b"}, "Page.navigate": {}, **(answers or {})}) + + +async def _tabs(devtools, *targets) -> BrowserControl: + control = BrowserControl(devtools, "agent", (1280, 800)) + await control.attach() + for target in targets: + devtools.send("Target.targetCreated", _page(target, f"https://mail.test/{target}", f"Mail {target}")) + devtools.send("Target.attachedToTarget", {"sessionId": f"s-{target}", **_page(target)}) + await asyncio.sleep(0) + return control + + +def _sent(devtools, method): + return [(params, session) for name, params, session in devtools.calls if name == method] + + +@pytest.mark.asyncio +async def test_it_navigates_the_active_tab_and_moves_in_its_history_or_says_why_it_cant(): + devtools = Tabbed({"Page.navigate": lambda params, session: {"errorText": "net::ERR_NAME_NOT_RESOLVED"} + if "nowhere" in params["url"] else {}}) + control = await _tabs(devtools, "a") + + await control.navigate("https://mail.test/inbox") + await control.history("back") + await control.history("reload") + with pytest.raises(InvalidArgument, match="net::ERR_NAME_NOT_RESOLVED"): + await control.navigate("https://nowhere.test/") + with pytest.raises(InvalidArgument, match="no page to go forward to"): + await control.history("forward") + + assert _sent(devtools, "Page.navigate")[0] == ({"url": "https://mail.test/inbox"}, "s-a") + assert _sent(devtools, "Page.navigateToHistoryEntry") == [({"entryId": 7}, "s-a")] + assert _sent(devtools, "Page.reload") == [({}, "s-a")] + + +@pytest.mark.asyncio +async def test_a_tab_it_opens_or_activates_is_brought_to_the_front_and_the_active_one_and_one_it_closes_its_own(): + devtools = Tabbed() + control = await _tabs(devtools, "a") + + opened = await control.open_tab("https://mail.test/b") + devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")}) + await asyncio.sleep(0) + after_open = await control.tab_list() + await control.activate("a") + after_switch = await control.tab_list() + with pytest.raises(InvalidArgument, match="no tab c is open"): + await control.activate("c") + await control.close("b") + devtools.send("Target.targetDestroyed", {"targetId": "b"}) + + assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "https://mail.test/b"}, None)] + assert [(tab["id"], tab["active"]) for tab in after_open] == [("a", False), ("b", True)] + assert [(tab["id"], tab["active"]) for tab in after_switch] == [("a", True), ("b", False)] + assert ({}, "s-a") in _sent(devtools, "Page.bringToFront") + assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs] == [("b", "agentenv")] + + +@pytest.mark.asyncio +async def test_a_dialog_makes_its_tab_the_active_one_its_state_names_it_and_the_agent_answers_it(): + devtools = Tabbed() + control = await _tabs(devtools, "a", "b") + with pytest.raises(InvalidArgument, match="no dialog is open"): + await control.answer_dialog(True) + devtools.send("Page.javascriptDialogOpening", {"type": "prompt", "message": "Name?", "defaultPrompt": "Ada", + "url": "https://mail.test/b"}, session="s-b") + + state = await control.state() + await control.answer_dialog(True, "Grace") + devtools.send("Page.javascriptDialogClosed", {"result": True, "userInput": "Grace"}, session="s-b") + + assert state == {"tab": "b", "url": "https://mail.test/b", "title": "Mail b", + "dialog": {"type": "prompt", "message": "Name?", "default_text": "Ada"}} + assert _sent(devtools, "Page.handleJavaScriptDialog") == [({"accept": True, "promptText": "Grace"}, "s-b")] + (dialog,) = control.dialog_log + assert (dialog["answer"], dialog["by"], dialog["input"]) == ("accepted", "agent", "Grace") + + +@pytest.mark.asyncio +async def test_input_that_opens_a_dialog_returns_once_it_opens_and_the_rest_waits_for_nothing(): + class Held(Tabbed): + """A page whose handler of a mouse release opens a dialog, so the release isn't answered until it's closed.""" + + def __init__(self): + super().__init__() + self.released = asyncio.get_running_loop().create_future() + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Input.dispatchMouseEvent" and params["type"] == "mouseReleased": + self.calls.append((method, params, session_id)) + self.send("Page.javascriptDialogOpening", {"type": "confirm", "message": "Delete?"}, + session=session_id) + return await self.released + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + devtools = Held() + control = await _tabs(devtools, "a") + events = [{"method": "Input.dispatchMouseEvent", "params": {"type": kind, "x": 1, "y": 1}} + for kind in ("mousePressed", "mouseReleased", "mouseMoved")] + + await asyncio.wait_for(control.dispatch(events), 5) + sent = [params["type"] for method, params, _ in devtools.calls if method == "Input.dispatchMouseEvent"] + devtools.released.set_result({}) + await asyncio.sleep(0) + + assert sent == ["mousePressed", "mouseReleased"] and (await control.state())["dialog"]["message"] == "Delete?" + + +@pytest.mark.asyncio +async def test_while_a_dialog_keeps_its_page_from_painting_its_capture_is_the_last_it_took(): + devtools = Tabbed() + control = await _tabs(devtools, "a") + before = await control.capture() + devtools.blank.add("s-a") + devtools.send("Page.javascriptDialogOpening", {"type": "alert", "message": "Saved"}, session="s-a") + + assert before == "png of s-a" and await control.capture() == "png of s-a" + + +@pytest.mark.asyncio +async def test_the_file_chooser_a_page_opens_takes_files_in_uploads_only_and_as_many_as_it_allows(tmp_path, + monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + (tmp_path / "Uploads" / "docs").mkdir(parents=True) + for name in ("cv.pdf", "docs/letter.pdf"): + (tmp_path / "Uploads" / name).write_text("x") + (tmp_path / "secret.txt").write_text("x") + devtools = Tabbed() + control = await _tabs(devtools, "a") + with pytest.raises(InvalidArgument, match="no file chooser is open"): + await control.choose_files(["cv.pdf"]) + devtools.send("Page.fileChooserOpened", {"frameId": "f", "mode": "selectSingle", "backendNodeId": 42}, + session="s-a") + + state = await control.state() + for paths, refused in ((["../secret.txt"], "isn't in ~/Uploads"), (["gone.pdf"], "no file gone.pdf"), + (["cv.pdf", "docs/letter.pdf"], "takes one file")): + with pytest.raises(InvalidArgument, match=refused): + await control.choose_files(paths) + await control.choose_files(["docs/letter.pdf"]) + + assert state["file_chooser"] == {"multiple": False} + assert _sent(devtools, "DOM.setFileInputFiles") == [ + ({"files": [str((tmp_path / "Uploads" / "docs" / "letter.pdf").resolve())], "backendNodeId": 42}, "s-a")] + assert "file_chooser" not in await control.state() + + @pytest.mark.asyncio async def test_with_no_tab_open_its_capture_and_input_are_unavailable(): control = BrowserControl(Painting(), "agent") diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile index 694f80d78..f859aed21 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile +++ b/src/agent_env/computers/artifacts/ubuntu-24.04-headless/Dockerfile @@ -1,7 +1,8 @@ -# The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), for one -# website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord runs -# every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. Composing -# a driver or an app onto it adds its services as programs in /etc/supervisor/conf.d. +# The ubuntu-24.04-headless base computer: Ubuntu 24.04 with no display server or desktop (XDG_SESSION_TYPE=none), +# for one website browser, which runs headless there, and a page driver whose screen is the browser's page. supervisord +# runs every program as the session user, and the browser goes out only through the site proxy on 127.0.0.1:3128. +# Files a page's file chooser may be given go in ~/Uploads. Composing a driver or an app onto it adds its services as +# programs in /etc/supervisor/conf.d. FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -9,7 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && userdel --remove ubuntu \ && useradd --create-home --uid 1000 --shell /bin/bash user \ - && runuser -u user -- mkdir -p /home/user/Desktop /home/user/Documents /home/user/Downloads \ + && runuser -u user -- mkdir -p /home/user/Desktop /home/user/Documents /home/user/Downloads /home/user/Uploads \ && mkdir -p /etc/supervisor/conf.d /var/log/agentenv /run/agentenv \ && chown user:user /var/log/agentenv /run/agentenv COPY supervisord.conf /etc/supervisor/supervisord.conf diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index 0da83b5a8..68558d37c 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -10,7 +10,9 @@ alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/drivers/ `envs/page/` is the page driver, for a computer with no display on Linux (`amd64` and `arm64`), such as `ubuntu-24.04-headless`: its screen is the computer's one website browser's active tab, its `screen` feature `page`. A screenshot is the page, and input reaches it as CDP input events, both through `browser/v1`'s `page/*` operations on the browser's own server; -a tab that paints nothing shows white. The browser ending on its own fails the computer for good: its ready check is +a tab that paints nothing shows white. It serves the page group too: `computer_navigate`, `computer_history`, +`computer_tabs`, `computer_dialog` and `computer_upload`, and every frame names the page, its tab and what is open on +it. The browser ending on its own fails the computer for good: its ready check is `final`, and its tools answer with why. Name it as `page`, or by its id, `@local/agentenv-framework/drivers/page`. A run that names it writes it: its files, what its class declares for each platform it supports, read without diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index e9dcdc60f..3baa751f4 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -19,24 +19,35 @@ ComputerDriverEnvironment, DeviceUnavailable, Display, + FileChooser, Image, + InvalidArgument, + PageDialog, + PageState, ReadyCheck, Recording, + Tab, Unsupported, capture, click, computer_driver, + dialog, display, drag, + history, key_down, key_up, + navigate, + page_state, pointer_down, pointer_move, pointer_position, pointer_up, press, scroll, + tabs, type_text, + upload, ) from agentenv_protocol.computer import apps from agentenv_protocol.computer.app import SOCKET @@ -80,7 +91,10 @@ class PageDriver(ComputerDriverEnvironment): recording = Recording(max_fps=10) screen_kind = "page" notes = ("This computer is one web page: the browser's active tab, with no address bar, tab strip or desktop " - "around it. Its screen is the page's viewport.") + "around it. Its screen is the page's viewport. Go to a URL with computer_navigate, back, forward or " + "reload with computer_history, and list, open, switch or close tabs with computer_tabs. Each screenshot " + "names the page, and an open dialog, which computer_dialog answers, or file chooser, which " + "computer_upload gives files in ~/Uploads.") def __init__(self) -> None: super().__init__() @@ -202,6 +216,39 @@ async def keys_up(self, keys) -> None: self._held = [key for key in self._held if key not in keys] await self._input([_key("keyUp", key, self._mask()) for key in keys]) + @navigate + async def go_to(self, url) -> None: + await self._call("navigate", {"url": url}) + + @history + async def move_in_history(self, direction) -> None: + await self._call("history", {"direction": direction}) + + @tabs + async def tab_action(self, action, tab, url) -> list[Tab]: + if action == "open": + await self._call("tabs/open", {"url": url} if url else {}) + elif action in ("switch", "close"): + await self._call("tabs/activate" if action == "switch" else "tabs/close", {"tab": tab}) + listed = (await self._call("tabs/list"))["tabs"] + return [Tab(t["id"], t["url"], t["title"], t["active"]) for t in listed] + + @dialog + async def answer_dialog(self, accept, text) -> None: + await self._call("dialog/answer", {"accept": accept, **({"text": text} if text is not None else {})}) + + @upload + async def choose_files(self, paths) -> None: + await self._call("files/choose", {"paths": paths}) + + @page_state + async def page_now(self) -> PageState: + state = await self._call("page/state") + open_dialog, chooser = state.get("dialog"), state.get("file_chooser") + return PageState(state["tab"], state["url"], state["title"], + PageDialog(**open_dialog) if open_dialog else None, + FileChooser(**chooser) if chooser else None) + def _mask(self) -> int: return _modifiers(self._held) @@ -225,6 +272,8 @@ async def _call(self, op: str, params: Optional[dict] = None) -> Any: raise DeviceUnavailable(_ended(folder) or f"the browser's server isn't answering: {type(e).__name__}: {e}" ) from None if answer.status_code >= 400: + if (refused := _refusal(answer)) is not None: + raise InvalidArgument(refused) raise DeviceUnavailable(_ended(folder) or f"the browser couldn't {op}: {answer.text[:300]}") return answer.json() @@ -241,6 +290,15 @@ async def _found(self) -> tuple[Path, dict]: return self._browser +def _refusal(answer: httpx.Response) -> Optional[str]: + """Why the browser's server refused a call it was given wrong, which the agent can put right.""" + try: + error = answer.json().get("error") or {} + except ValueError: + return None + return error.get("message") if error.get("code") == "invalid_argument" else None + + def _ended(folder: Path) -> Optional[str]: if (failed := apps.ended_on_its_own(folder)) is not None: return f"the browser in {folder.name} has failed: {failed}" diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index efe205fff..c80e31a07 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1,9 +1,10 @@ -"""The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display and the page -driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; deployed -through the computer provider like any computer, its computer_* tools see and drive the page; nothing runs X and nothing -listens but the driver and the site proxy; a killed app server or driver is started again by supervisord and the -computer is ready again; and chromium killed fails it for good, computer-ready/v1 saying why and the tools answering -with it. Needs Docker and the local registry on localhost:5000.""" +"""The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display +and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; +deployed through the computer provider like any computer, its computer_* tools see and drive the page, and its page +tools navigate, move in history, answer a dialog, use tabs and upload; nothing runs X and nothing listens but the driver +and the site proxy; a killed app server or driver is started again by supervisord and the computer is ready again; and +chromium killed fails it for good, computer-ready/v1 saying why and the tools answering with it. Needs Docker and the +local registry on localhost:5000.""" import asyncio import json @@ -53,6 +54,33 @@ show(); """ PAGE_URL = "data:text/html," + urllib.parse.quote(PAGE) +SITE = "http://127.0.0.1:8771/" +# Serves, on loopback for the seconds it's given, a home page with a button that asks to confirm and a file input, each +# saying in the title what came of it, and two more pages. +SERVE_SITE = """import http.server, sys, threading, time +PAGES = { + "/": (b"Home" + b""), + "/two": b"Two

Two

", + "/three": b"Three

Three

", +} +class Site(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = PAGES.get(self.path, b"Missing") + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass +server = http.server.ThreadingHTTPServer(("127.0.0.1", 8771), Site) +threading.Thread(target=server.serve_forever, daemon=True).start() +time.sleep(float(sys.argv[1])) +""" +SERVE_START_S = 3 # Every TCP port something in the computer listens on, and whether any process is an X server. LISTENING = """import glob ports = set() @@ -155,6 +183,80 @@ async def test_a_killed_app_server_or_driver_comes_back_and_a_killed_browser_fai assert not report.failed +@pytest.mark.asyncio +async def test_an_agent_navigates_moves_in_history_answers_a_dialog_uses_tabs_and_uploads_with_its_page_tools( + browser_computer): + with namespace_routing(): + deployed = await Env.get(browser_computer.id).deploy( + env_params={"app_settings": {"chromium": {"start_urls": [SITE]}}}) + context = TaskStepContext(deployed_envs=[deployed]) + try: + serving = asyncio.ensure_future(inside.run(deployed, ["python3", "-c", SERVE_SITE, "900"], timeout_s=960)) + await inside.run(deployed, ["sh", "-c", "printf cv > ~/Uploads/cv.txt"], timeout_s=30) + await asyncio.sleep(SERVE_START_S) + async with streamablehttp_client(deployed.mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + await _tool(session, "computer_history", {"direction": "reload"}) + home = await _shown(session, lambda page: page["title"] == "Home") + await _tool(session, "computer_navigate", {"url": SITE + "two"}) + two = await _shown(session, lambda page: page["title"] == "Two") + await _tool(session, "computer_history", {"direction": "back"}) + back = await _shown(session, lambda page: page["title"] == "Home") + await _tool(session, "computer_history", {"direction": "forward"}) + forward = await _shown(session, lambda page: page["title"] == "Two") + await _tool(session, "computer_history", {"direction": "back"}) + await _shown(session, lambda page: page["title"] == "Home") + asked = await session.call_tool("computer_click", {"x": 150, "y": 50, "screenshot": True}) + open_dialog = await _shown(session, lambda page: page.get("dialog") is not None) + blocked = await session.call_tool("computer_click", {"x": 150, "y": 50}) + await _tool(session, "computer_dialog", {"accept": True}) + confirmed = await _shown(session, lambda page: page["title"] == "confirmed true") + opened = await session.call_tool("computer_tabs", {"action": "open", "url": SITE + "three"}) + three = await _shown(session, lambda page: page["title"] == "Three") + first = next(tab["tab"] for tab in opened.structuredContent["tabs"] if not tab["active"]) + second = next(tab["tab"] for tab in opened.structuredContent["tabs"] if tab["active"]) + await _tool(session, "computer_tabs", {"action": "switch", "tab": first}) + switched = await _shown(session, lambda page: page["tab"] == first) + closed = await session.call_tool("computer_tabs", {"action": "close", "tab": second}) + await _tool(session, "computer_click", {"x": 150, "y": 250}) + chooser = await _shown(session, lambda page: page.get("file_chooser") is not None) + await _tool(session, "computer_upload", {"paths": ["cv.txt"]}) + chose = await _shown(session, lambda page: page["title"] == "chose cv.txt") + record = await _record(deployed) + serving.cancel() + finally: + report = await teardown_run(context) + + assert home["page"]["url"] == SITE and two["page"]["url"] == SITE + "two" + assert back["page"]["title"] == "Home" and forward["page"]["title"] == "Two" + assert not asked.isError and asked.structuredContent["applied"] == "yes" + assert open_dialog["page"]["dialog"] == {"type": "confirm", "message": "Delete this message?", "default_text": ""} + assert 'a confirm dialog is open: "Delete this message?"' in open_dialog["summary"] + assert blocked.isError and blocked.structuredContent["code"] == "dialog_open" + assert confirmed["page"]["title"] == "confirmed true" and "dialog" not in confirmed["page"] + assert [tab["url"] for tab in opened.structuredContent["tabs"]] == [SITE, SITE + "three"] + assert three["page"]["tab"] == second and switched["page"]["title"] == "confirmed true" + assert [tab["tab"] for tab in closed.structuredContent["tabs"]] == [first] + assert chooser["page"]["file_chooser"] == {"multiple": False} and chose["page"]["title"] == "chose cv.txt" + assert [(d["message"], d["answer"], d["by"]) for d in record["dialogs"]] == [ + ("Delete this message?", "accepted", "agent")] + assert any(tab["id"] == second and tab["closed_by"] == "agentenv" for tab in record["closed_tabs"]) + assert not report.failed + + +async def _shown(session: ClientSession, holds) -> dict: + """A screenshot's result once the page it shows is what ``holds`` waits for, within ``WAIT_S``.""" + last, deadline = None, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + shot = await session.call_tool("computer_screenshot", {"space": "frame"}) + last = shot.structuredContent + if not shot.isError and last.get("page") and holds(last["page"]): + return last + await asyncio.sleep(POLL_S) + raise AssertionError(f"the page didn't get there within {WAIT_S:g} s; it last showed {last!r}") + + async def _tool(session: ClientSession, name: str, args: dict) -> None: result = await session.call_tool(name, args) assert not result.isError, f"{name} {args}: {result.content[0].text}" diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py index e99c7f4ef..af41a0939 100644 --- a/tst/unit/drivers/page_driver_test.py +++ b/tst/unit/drivers/page_driver_test.py @@ -12,16 +12,19 @@ import pytest import agent_env -from agentenv_protocol.computer import DeviceUnavailable, Platform +from agentenv_protocol.computer import (DeviceUnavailable, FileChooser, InvalidArgument, PageDialog, PageState, + Platform, Tab) from agentenv_protocol.computer import driver as computer_driver from agentenv_protocol.computer.app_runtime import LAUNCHES from agentenv_protocol.website_browser import BROWSER DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "page" / "driver.py" NO_DISPLAY = Platform(os="linux", distribution="ubuntu", version="24.04", arch="amd64", display="none") -PAGE_OPS = ("page/viewport", "page/capture", "page/input") +PAGE_OPS = ("page/viewport", "page/capture", "page/input", "page/state", "navigate", "history", "tabs/list", + "tabs/open", "tabs/activate", "tabs/close", "dialog/answer", "files/choose") +GETS = ("page/viewport", "page/capture", "page/state", "tabs/list") CARD = {"name": "chromium", "capabilities": {"extensions": [{"uri": BROWSER, "params": {"methods": { - op: {"method": "POST" if op == "page/input" else "GET", "endpoint": f"/agentenv/ext/browser/{op}"} + op: {"method": "GET" if op in GETS else "POST", "endpoint": f"/agentenv/ext/browser/{op}"} for op in PAGE_OPS}}}]}} #: A 2×1 PNG's first 24 bytes: its signature, then its IHDR's length, type, width and height. PNG = b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + (2).to_bytes(4, "big") + (1).to_bytes(4, "big") @@ -163,3 +166,46 @@ async def test_once_the_browser_has_ended_its_calls_say_why_and_send_nothing_tho with pytest.raises(DeviceUnavailable, match="has failed"): await page.click_at(1, 2, "left", 1, []) assert len(page.sent) == sent + + +@pytest.mark.asyncio +async def test_its_page_operations_call_the_browsers_and_a_tab_action_answers_with_the_tabs_after_it(page): + page.served["tabs/list"] = {"tabs": [{"id": "A1", "url": "https://mail.test/", "title": "Inbox", "active": True}]} + await page.go_to("https://mail.test/") + await page.move_in_history("back") + opened = await page.tab_action("open", None, "https://mail.test/") + await page.tab_action("switch", "A1", None) + await page.tab_action("close", "A1", None) + await page.answer_dialog(False, None) + await page.answer_dialog(True, "Grace") + await page.choose_files(["cv.pdf"]) + + assert opened == [Tab("A1", "https://mail.test/", "Inbox", True)] + assert [entry for entry in page.sent if entry[0] != "tabs/list"] == [ + ("navigate", {"url": "https://mail.test/"}), ("history", {"direction": "back"}), + ("tabs/open", {"url": "https://mail.test/"}), ("tabs/activate", {"tab": "A1"}), ("tabs/close", {"tab": "A1"}), + ("dialog/answer", {"accept": False}), ("dialog/answer", {"accept": True, "text": "Grace"}), + ("files/choose", {"paths": ["cv.pdf"]})] + + +@pytest.mark.asyncio +async def test_its_page_state_is_the_browsers_with_what_is_open_on_it(page): + page.served["page/state"] = {"tab": "A1", "url": "https://mail.test/", "title": "Inbox", + "dialog": {"type": "confirm", "message": "Delete?", "default_text": ""}, + "file_chooser": {"multiple": True}} + open_on_it = await page.page_now() + page.served["page/state"] = {"tab": "A1", "url": "https://mail.test/", "title": "Inbox"} + + assert open_on_it == PageState("A1", "https://mail.test/", "Inbox", PageDialog("confirm", "Delete?"), + FileChooser(True)) + assert await page.page_now() == PageState("A1", "https://mail.test/", "Inbox") + + +@pytest.mark.asyncio +async def test_a_call_the_browser_refuses_as_given_wrong_is_an_invalid_argument_the_agent_can_put_right(page): + page.served["files/choose"] = httpx.Response(400, json={"ok": False, "error": { + "code": "invalid_argument", "message": "there's no file cv.pdf in ~/Uploads"}}) + + with pytest.raises(InvalidArgument, match="there's no file cv.pdf in ~/Uploads"): + await page.choose_files(["cv.pdf"]) + From 23d3eebf0105df3ebdc52a12a748379643dca20e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 17:59:24 -0800 Subject: [PATCH 2/3] fix(computer): input goes on past a dialog answered for the agent, and a frame names the tab its capture came from A dialog the deploy's setting answers at once no longer cuts an input batch short; only one left to the agent does. A page driver's capture now returns the page it shows with it, the captured tab's, which the frame carries rather than a state read after, when another caller may have switched tabs. Co-Authored-By: Claude Opus 5.5 --- .../agentenv_protocol/computer/operations.py | 6 +- .../src/agentenv_protocol/computer/runtime.py | 9 ++- .../src/agentenv_protocol/website_browser.py | 37 ++++++++---- .../tests/test_computer_page.py | 21 +++++++ .../tests/test_website_browser.py | 59 +++++++++++++++++++ src/agent_env/drivers/envs/page/driver.py | 19 +++--- tst/unit/drivers/page_driver_test.py | 6 +- 7 files changed, 133 insertions(+), 24 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index fea11661d..ad7f67b7c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -19,6 +19,9 @@ tabs(action, tab, url) -> [Tab, ...] list, open, switch or close; the tabs open after it dialog(accept, text) upload(paths), relative to the session user's ~/Uploads page_state() -> PageState + +A page driver's ``capture`` may return its ``Image`` with the ``page`` it shows, read with it, which every frame then +carries rather than one ``page_state`` reads after. """ from __future__ import annotations @@ -54,11 +57,12 @@ class Display: @dataclass(frozen=True) class Image: """A capture: encoded ``png`` or ``jpeg`` bytes, or raw pixels in a Pillow raw mode (``RGB``, ``RGBA``, ``BGRA``, - ``BGRX``).""" + ``BGRX``); and, from a page driver, the page it shows.""" width: int height: int data: bytes mode: str = "png" + page: PageState | None = None @dataclass(frozen=True) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index e220aeb5e..11d537c1e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -16,7 +16,7 @@ from . import apps from .errors import DeliveryFailed, DeviceUnavailable, DialogOpen, InputBusy, Internal from .frames import Frame, FrameStore, to_pil -from .operations import DIALOG_BLOCKS, GROUPS, INPUT, Display, PageState, ReadyCheck, resolve +from .operations import DIALOG_BLOCKS, GROUPS, INPUT, Display, Image, PageState, ReadyCheck, resolve from .queue import InputQueue from .recording import RecordHints, Recordings from .results import EffectData, InputResult, send @@ -80,8 +80,11 @@ async def capture(self, identity: str) -> Frame: counts as after the frame, which the frame may not show.""" inputs = self.queue.snapshot() display = await self.display_now() - image = to_pil(await self.call("capture"), display) - page = await self.page_now() if self.has("page") else None + captured = await self.call("capture") + image = to_pil(captured, display) + page = captured.page if isinstance(captured, Image) and isinstance(captured.page, PageState) else None + if page is None and self.has("page"): + page = await self.page_now() return self.frames.add(image, display, inputs, identity=identity, page=page) async def page_now(self) -> PageState: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index a807a3b13..8ea633257 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -326,12 +326,15 @@ async def state(self) -> dict: active = await self.active_tab() if active is None: raise DeviceUnavailable("the browser has no tab open") - tab = self.tabs[active] - state: dict[str, Any] = {"tab": active, "url": tab["url"], "title": tab["title"]} - if dialog := next((d for d in self._open_dialogs.values() if d["tab"] == active), None): + return self._state_of(active) + + def _state_of(self, target: str) -> dict: + tab = self.tabs.get(target) or {"url": "", "title": ""} + state: dict[str, Any] = {"tab": target, "url": tab["url"], "title": tab["title"]} + if dialog := next((d for d in self._open_dialogs.values() if d["tab"] == target), None): state["dialog"] = {"type": dialog["type"], "message": dialog["message"], "default_text": dialog["default_text"]} - if chooser := next((c for s, c in self._choosers.items() if self._sessions.get(s) == active), None): + if chooser := next((c for s, c in self._choosers.items() if self._sessions.get(s) == target), None): state["file_chooser"] = {"multiple": chooser["multiple"]} return state @@ -403,7 +406,18 @@ async def capture(self) -> Optional[str]: """The active tab's page as it shows now, a PNG, base64; None for a tab that paints nothing within ``CAPTURE_TIMEOUT_S``, which a desktop browser shows blank, or sooner for a tab with no URL yet, such as a new tab whose navigation the browser blocked, which may never paint.""" + return await self._capture(await self.active_session()) + + async def capture_page(self) -> dict: + """The active tab's capture, ``png``, else ``blank``, with its ``page`` state, of that tab whatever becomes + active meanwhile.""" session = await self.active_session() + target = self._sessions[session] + png = await self._capture(session) + await self._refresh(target) + return {**({"png": png} if png is not None else {"blank": True}), "page": self._state_of(target)} + + async def _capture(self, session: str) -> Optional[str]: target = self._sessions[session] navigated = self.tabs.get(target, {}).get("url") try: @@ -416,15 +430,16 @@ async def capture(self) -> Optional[str]: return shot["data"] async def dispatch(self, events: list[dict]) -> None: - """Each of ``events``, a CDP input call, sent to the active tab in order, until one opens a dialog: the page - answers that one only once the dialog is answered, so it finishes then, and the rest aren't sent. - InvalidArgument for any other call.""" + """Each of ``events``, a CDP input call, sent to the active tab in order, until one opens a dialog left to the + agent: the page answers that one only once the dialog is answered, so it finishes then, and the rest aren't + sent. A dialog answered for the agent is answered at once, and the rest go on. InvalidArgument for any other + call.""" if refused := [event.get("method") for event in events if event.get("method") not in PAGE_INPUT]: raise InvalidArgument(f"page input takes {', '.join(sorted(PAGE_INPUT))}, not {refused}") session = await self.active_session() opened = self._dialog_opened.setdefault(session, asyncio.Event()) for event in events: - if session in self._open_dialogs: + if self.dialogs == AGENT and session in self._open_dialogs: return opened.clear() sent = asyncio.ensure_future(self.devtools.call(event["method"], event.get("params") or {}, @@ -435,6 +450,9 @@ async def dispatch(self, events: list[dict]) -> None: finally: waiting.cancel() if not sent.done(): + if self.dialogs != AGENT: + await sent + continue self._tasks.add(sent) sent.add_done_callback(self._finished) return @@ -783,8 +801,7 @@ async def browser_page_viewport(self) -> dict: @extension(BROWSER, op="page/capture", method="GET") async def browser_page_capture(self) -> dict: - png = await self.control.capture() - return {"png": png} if png is not None else {"blank": True} + return await self.control.capture_page() @extension(BROWSER, op="page/input") async def browser_page_input(self, events: list[dict]) -> dict: diff --git a/packages/agentenv-protocol/tests/test_computer_page.py b/packages/agentenv-protocol/tests/test_computer_page.py index 36e8dd8a1..f417cbb30 100644 --- a/packages/agentenv-protocol/tests/test_computer_page.py +++ b/packages/agentenv-protocol/tests/test_computer_page.py @@ -12,9 +12,11 @@ from agentenv_protocol.computer import ( # noqa: E402 FileChooser, + Image, PageDialog, PageState, Tab, + capture, dialog, history, navigate, @@ -35,6 +37,14 @@ def __init__(self) -> None: self.open_dialog: PageDialog | None = None self.chooser: FileChooser | None = None self.calls: list = [] + self.captured: PageState | None = None + + @capture + async def grab(self): + image = await super().grab() + if self.captured is None: + return image + return Image(image.width, image.height, image.tobytes(), "RGB", self.captured) @navigate async def go_to(self, url): @@ -142,3 +152,14 @@ async def test_while_a_dialog_is_open_input_it_blocks_is_refused_until_computer_ assert navigated.structuredContent["code"] == "dialog_open" and not listed.isError assert answered.structuredContent["summary"] == "Dismissed the dialog" and not clicked.isError assert ("dialog", False, None) in driver.calls and not any(entry[0] == "click" for entry in driver.screen.log[:-1]) + + +@pytest.mark.asyncio +async def test_a_frame_names_the_page_its_capture_came_with_rather_than_one_read_after_it(): + driver = PageFake() + driver.captured = PageState("b", "https://docs.test/", "Docs") + async with mcp_session(driver) as session: + shot = await session.call_tool("computer_screenshot", {}) + + assert shot.structuredContent["page"] == {"tab": "b", "url": "https://docs.test/", "title": "Docs"} + diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 11c59ed1f..97f1905fa 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -709,6 +709,65 @@ async def call(self, method, params=None, *, session_id=None, timeout_s=None): assert sent == ["mousePressed", "mouseReleased"] and (await control.state())["dialog"]["message"] == "Delete?" +@pytest.mark.asyncio +async def test_input_that_opens_a_dialog_answered_for_the_agent_goes_on_once_it_is_answered(): + class Answered(Tabbed): + """A page whose handler of a mouse release opens a dialog, answered at once by the control's setting.""" + + def __init__(self): + super().__init__({"Page.handleJavaScriptDialog": self._close}) + self.released = asyncio.get_running_loop().create_future() + + def _close(self, params, session): + self.released.set_result({}) + return {} + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Input.dispatchMouseEvent" and params["type"] == "mouseReleased": + self.calls.append((method, params, session_id)) + self.send("Page.javascriptDialogOpening", {"type": "alert", "message": "Saved"}, session=session_id) + return await self.released + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + devtools = Answered() + control = BrowserControl(devtools, "accept", (1280, 800)) + await control.attach() + devtools.send("Target.targetCreated", _page("a", "https://mail.test/a", "Mail a")) + devtools.send("Target.attachedToTarget", {"sessionId": "s-a", **_page("a")}) + await asyncio.sleep(0) + events = [{"method": "Input.dispatchMouseEvent", "params": {"type": kind, "x": 1, "y": 1}} + for kind in ("mousePressed", "mouseReleased", "mouseMoved")] + + await asyncio.wait_for(control.dispatch(events), 5) + + assert [params["type"] for method, params, _ in devtools.calls if method == "Input.dispatchMouseEvent"] == [ + "mousePressed", "mouseReleased", "mouseMoved"] + + +@pytest.mark.asyncio +async def test_its_page_capture_names_the_tab_it_captured_though_another_becomes_active_meanwhile(): + class Switching(Tabbed): + """Tabs of which only ``focused``'s page has focus, which a capture of the first moves to the second.""" + + def __init__(self): + super().__init__({"Runtime.evaluate": lambda params, session: { + "result": {"value": ["visible", session == self.focused]}}}) + self.focused = "s-a" + + async def call(self, method, params=None, *, session_id=None, timeout_s=None): + if method == "Page.captureScreenshot": + self.focused = "s-b" + return await super().call(method, params, session_id=session_id, timeout_s=timeout_s) + + devtools = Switching() + control = await _tabs(devtools, "a", "b") + + shot = await control.capture_page() + + assert shot == {"png": "png of s-a", "page": {"tab": "a", "url": "https://mail.test/a", "title": "Mail a"}} + assert (await control.state())["tab"] == "b" + + @pytest.mark.asyncio async def test_while_a_dialog_keeps_its_page_from_painting_its_capture_is_the_last_it_took(): devtools = Tabbed() diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index 3baa751f4..a35efd442 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -111,13 +111,14 @@ async def viewport(self) -> Display: @capture async def page(self) -> Image: """The page, or a blank one where the browser shows nothing, as a desktop browser shows a tab with nothing - in it.""" + in it; with the page it shows, read with it.""" shot = await self._call("page/capture") + page = _page_state(shot["page"]) if shot.get("page") else None if shot.get("blank"): size = await self.viewport() - return Image(size.width, size.height, BLANK * (size.width * size.height), "RGB") + return Image(size.width, size.height, BLANK * (size.width * size.height), "RGB", page) data = base64.b64decode(shot["png"]) - return Image(int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big"), data, "png") + return Image(int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big"), data, "png", page) async def ready_checks(self) -> list[ReadyCheck]: try: @@ -243,11 +244,7 @@ async def choose_files(self, paths) -> None: @page_state async def page_now(self) -> PageState: - state = await self._call("page/state") - open_dialog, chooser = state.get("dialog"), state.get("file_chooser") - return PageState(state["tab"], state["url"], state["title"], - PageDialog(**open_dialog) if open_dialog else None, - FileChooser(**chooser) if chooser else None) + return _page_state(await self._call("page/state")) def _mask(self) -> int: return _modifiers(self._held) @@ -290,6 +287,12 @@ async def _found(self) -> tuple[Path, dict]: return self._browser +def _page_state(state: dict) -> PageState: + open_dialog, chooser = state.get("dialog"), state.get("file_chooser") + return PageState(state["tab"], state["url"], state["title"], PageDialog(**open_dialog) if open_dialog else None, + FileChooser(**chooser) if chooser else None) + + def _refusal(answer: httpx.Response) -> Optional[str]: """Why the browser's server refused a call it was given wrong, which the agent can put right.""" try: diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py index af41a0939..5eef37282 100644 --- a/tst/unit/drivers/page_driver_test.py +++ b/tst/unit/drivers/page_driver_test.py @@ -131,13 +131,15 @@ async def test_a_double_click_is_one_move_then_two_presses_and_releases_counted( @pytest.mark.asyncio async def test_its_capture_is_the_pages_png_and_a_tab_that_paints_nothing_is_white(page): - page.served["page/capture"] = {"png": base64.b64encode(PNG).decode()} + shown = {"tab": "A1", "url": "https://mail.test/", "title": "Inbox"} + page.served["page/capture"] = {"png": base64.b64encode(PNG).decode(), "page": shown} shot = await page.page() - page.served["page/capture"] = {"blank": True} + page.served["page/capture"] = {"blank": True, "page": shown} blank = await page.page() assert (shot.width, shot.height, shot.mode, shot.data) == (2, 1, "png", PNG) assert (blank.width, blank.height, blank.mode) == (1280, 800, "RGB") and set(blank.data) == {0xff} + assert shot.page == blank.page == PageState("A1", "https://mail.test/", "Inbox") @pytest.mark.asyncio From e7f89b23b3f2b7b78a8b456ce1bfd29cd4e090b8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 18:19:50 -0800 Subject: [PATCH 3/3] Expect the browser's new page operations in its methods Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/website_browser_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tst/integration/computer/website_browser_test.py b/tst/integration/computer/website_browser_test.py index a2e440229..b428b0f29 100644 --- a/tst/integration/computer/website_browser_test.py +++ b/tst/integration/computer/website_browser_test.py @@ -131,7 +131,9 @@ async def test_chromium_runs_with_agentenvs_flags_on_its_pipe_and_a_reset_relaun policies["stdout"]).items() params = protocol_v1.extension_params(card, BROWSER) assert (params["chromium"], params["clock"], sorted(params["methods"])) == ( - "156.0.8078.4", "real", ["page/capture", "page/input", "page/viewport", "record"]) + "156.0.8078.4", "real", ["dialog/answer", "files/choose", "history", "navigate", "page/capture", "page/input", + "page/state", "page/viewport", "record", "tabs/activate", "tabs/close", "tabs/list", + "tabs/open"]) assert readiness["ready"] is True, readiness assert relaunched["pid"] != browser["pid"] and relaunched["listening"] == 0 assert relaunched_readiness["ready"] is True, relaunched_readiness