From 66db8a41e5c94a38ab071eb8e105169e104f0e93 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 19:30:39 -0800 Subject: [PATCH 1/4] feat(computer): gate 3 checks a browser computer's input and page tools on an HTML Fixture Desk On a driver whose screen is a page, the probe stages an HTML desk with the Tk desk's widgets, serves it on loopback, opens it in a tab of its own and runs P3-P7, P9 and the full X checks on it, then checks the page tools against it. The page driver sends one wheel event per notch. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 12 +- .../computer/conformance/checks.py | 164 ++++++++++- .../computer/conformance/page_fixture.py | 256 ++++++++++++++++++ .../agentenv-protocol/tests/probe_fake.py | 135 ++++++++- .../tests/test_computer_conformance_page.py | 81 ++++++ src/agent_env/drivers/envs/page/driver.py | 7 +- .../computer/browser_computer_test.py | 17 +- tst/unit/drivers/page_driver_test.py | 9 + 8 files changed, 660 insertions(+), 21 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py create mode 100644 packages/agentenv-protocol/tests/test_computer_conformance_page.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 3c4c68f22..5f57e9ff1 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -625,12 +625,20 @@ driver booted with a control token is probed with it: `--token`, by default | Level | Checks | Needs on the computer | |---|---|---| | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | -| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the window's widgets report, reads back what reached them, and then removes the fixture. When the -fixture can't be staged, the `full` checks are unverified rather than failed. The probe leaves out +fixture can't be staged, the `full` checks are unverified rather than failed. + +On a driver whose card says `screen: "page"`, the desk is an HTML page with the same widgets at the +same places (`conformance/page_fixture.py`). The probe starts its server on loopback, which writes +what the page reports to the same files, and opens the desk with `computer_tabs` in a tab of its +own. "Typing leaves the clipboard as it was" fails there when typing arrives as a paste. After the +input checks it checks the page tools against the desk; a browser that answers dialogs itself makes +the dialog check not applicable. It then closes its tab and removes the desk, and the file it put in +`~/Uploads`. The tab that was open before keeps its page and history. The probe leaves out the checks for an extension the card doesn't declare. P8 is not applicable to a driver without leasing, and unverified for one with it until the probe has lease checks. The probe calls as the default role; the tool-access check hides a tool from a role of its own, `agentenv-probe-`, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 074767c40..a7b500b29 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -28,6 +28,7 @@ COMPUTER_DISPLAY, COMPUTER_EXEC, COMPUTER_KEYBOARD, + COMPUTER_PAGE, COMPUTER_POINTER, COMPUTER_READY, COMPUTER_RECORDING, @@ -39,6 +40,7 @@ from ..recording import RECORDINGS_DIR from .client import Driver, image from .mp4 import read_mp4 +from .page_fixture import QUESTION, TITLE, TWO, TWO_TITLE from .report import CheckResult MINIMUM_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", @@ -46,6 +48,10 @@ DATA_PLANE = (INTAKE_EXTENSION_URI, DATA_OBJECTS_EXTENSION_URI) #: Where the fixture is staged and keeps its state, relative to the session user's home. STAGE = ".agentenv-probe" +#: Where a page driver's computer_upload takes files from, relative to the session user's home. +UPLOADS = "Uploads" +#: What the probe answers the page fixture's prompt with. +WORD = "tangerine" #: What the tool-access check's role, new each run, and the probe's request ids start with. PROBE_ROLE = "agentenv-probe" #: How long the probe waits for an effect to reach the fixture, and how often it looks. @@ -72,6 +78,10 @@ Test = Callable[[], Awaitable[str | None]] +class NotApplicable(Exception): + """Raised by a test whose check doesn't apply to this driver, with why.""" + + class Probe: """One probe run: the driver, an MCP session on its tools, and the results so far. A test returns what went wrong, or None when its check passes; a test that raises fails with what it raised.""" @@ -84,6 +94,10 @@ def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: floa self.features = driver.params(COMPUTER) self.layout: dict = {} self.fixture_pid: str | None = None + self.page = self.features.get("screen") == "page" + self.desk = "" + self.tab: str | None = None + self.upload: str | None = None async def basic(self) -> None: for check, section in (("P0", self._ready), ("P1", self._card), ("P2", self._frame), ("P5", self._refusals), @@ -95,11 +109,14 @@ async def basic(self) -> None: self._grants() async def full(self) -> None: - missing = [uri for uri in (COMPUTER_EXEC,) if not self.driver.declares(uri)] + needs = (COMPUTER_EXEC, COMPUTER_PAGE) if self.page else (COMPUTER_EXEC,) + missing = [uri for uri in needs if not self.driver.declares(uri)] if not (protocol.intake_declaration(self.driver.card) or {}).get("add"): missing.append("data/add") if missing: - return self._skip_full(f"the fixture is staged with data/add and computer-exec; the driver lacks {missing}") + how = "data/add and computer-exec, and opened with computer-page/v1" if self.page else \ + "data/add and computer-exec" + return self._skip_full(f"the fixture is staged with {how}; the driver lacks {missing}") try: try: problem = await self._stage() @@ -109,7 +126,8 @@ async def full(self) -> None: return self._skip_full(f"the fixture didn't start: {problem}") for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), - ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording)): + ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording), + ("X", self._page_tools)): await self._guarded(check, section) finally: await self._unstage() @@ -119,6 +137,9 @@ async def full(self) -> None: async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool: try: problem = await test() + except NotApplicable as e: + self._record(check, name, "not_applicable", str(e), extensions) + return True except Exception as e: problem = f"{type(e).__name__}: {e}" self._record(check, name, "failed" if problem else "passed", problem or "", extensions) @@ -181,6 +202,14 @@ async def settled(self, holds: Callable[[dict], bool], show: Callable[[dict], ob ok, state = await self.until(holds) return None if ok else f"the fixture shows {show(state)!r}" + async def page_now(self) -> dict: + """The page a new frame names, or {} for a frame that names none.""" + return (await self.shot())[0].structuredContent.get("page") or {} + + async def showing(self, holds: Callable[[dict], bool]) -> str | None: + ok, page = await self.until(holds, self.page_now) + return None if ok else f"the frame's page is {page!r}" + @staticmethod def refused(result: CallToolResult, code: str) -> str | None: data = result.structuredContent or {} @@ -507,16 +536,21 @@ def _grants(self) -> None: # full: the fixture async def _stage(self) -> str | None: - source = resources.files(__package__).joinpath("fixture.py").read_bytes() - added = await self.driver.add_file(f"{STAGE}/fixture.py", source) + """The Fixture Desk started, or what stopped it: the Tk desk, or on a page, the HTML desk's server, with the + desk opened in a tab of the probe's own.""" + fixture = "page_fixture.py" if self.page else "fixture.py" + source = resources.files(__package__).joinpath(fixture).read_bytes() + added = await self.driver.add_file(f"{STAGE}/{fixture}", source) if "error" in added: return f"data/add refused it: {added['error'].get('message')}" await self.driver.run("sh", "-c", f"rm -rf {STAGE}/state") started = await self.driver.run( - "sh", "-c", f"python3 {STAGE}/fixture.py {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") + "sh", "-c", f"python3 {STAGE}/{fixture} {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") self.fixture_pid = started["stdout"].strip() or None deadline = time.monotonic() + FIXTURE_START_S while time.monotonic() < deadline: + if self.page and self.tab is None and (problem := await self._open_desk()): + return problem layout = await self.driver.run("cat", f"{STAGE}/state/layout.json") if layout.get("exit_code") == 0 and layout["stdout"].strip(): self.layout = json.loads(layout["stdout"]) @@ -528,10 +562,29 @@ async def _stage(self) -> str | None: log = await self.driver.run("tail", "-n", "5", f"{STAGE}/fixture.log") return f"no layout after {FIXTURE_START_S:g} s: {log['stdout'].strip() or log['stderr'].strip()}" + async def _open_desk(self) -> str | None: + """Once the page fixture's server has its port, the desk opened in a new tab, which is the probe's.""" + port = await self.driver.run("cat", f"{STAGE}/state/port") + if port.get("exit_code") != 0 or not port["stdout"].strip(): + return None + self.desk = f"http://127.0.0.1:{port['stdout'].strip()}/" + opened = await self.tool("computer_tabs", {"action": "open", "url": self.desk}) + if opened.isError: + return f"computer_tabs couldn't open it: {opened.content[0].text}" + self.tab = next((t["tab"] for t in opened.structuredContent["tabs"] if t["active"]), None) + return None if self.tab else f"the tab computer_tabs opened isn't active: {opened.structuredContent['tabs']}" + async def _unstage(self) -> None: + if self.tab: + with contextlib.suppress(Exception): + listed = await self.tool("computer_tabs", {"action": "list"}) + if len(listed.structuredContent["tabs"]) > 1: + await self.tool("computer_tabs", {"action": "close", "tab": self.tab}) if self.fixture_pid: await self.driver.run("kill", self.fixture_pid) await self.driver.run("rm", "-rf", STAGE) + if self.upload: + await self.driver.run("rm", "-f", f"{UPLOADS}/{self.upload}") async def _base(self) -> tuple[str, float, object]: """A fresh frame of the fixture: its id, the display's scale, and its image.""" @@ -708,7 +761,11 @@ async def whitespace() -> str | None: async def clipboard() -> str | None: state = await self.state() - return None if state["clipboard"] == self.layout["clipboard"] else f"the clipboard is {state['clipboard']!r}" + if pasted := state.get("pasted"): + return f"typing arrived as a paste of {pasted!r}" + if state["clipboard"] != self.layout["clipboard"]: + return f"the clipboard is {state['clipboard']!r}" + return None async def held() -> str | None: await clear() @@ -789,6 +846,99 @@ async def picked_up() -> str | None: await self.check("X", "a file data/add writes is there for an app to read", picked_up, *DATA_PLANE) + async def _page_tools(self) -> None: + """The page tools against the HTML desk, each confirmed on the desk or in the page a new frame names; those + that leave the desk run last.""" + if not self.page: + return + frame, scale, _ = await self._base() + two = self.desk + TWO.lstrip("/") + + async def dialog() -> str | None: + before = (await self.state())["saved_count"] + await self.act("computer_click", {"x": self._at("ask", scale)[0], "y": self._at("ask", scale)[1], + "frame_id": frame}) + + async def seen() -> tuple[dict, dict]: + return await self.page_now(), await self.state() + + ok, (page, state) = await self.until(lambda s: bool(s[0].get("dialog")) or s[1]["prompts"] > 0, seen) + if not page.get("dialog"): + if state["prompts"]: + raise NotApplicable(f"the browser answers dialogs itself: the prompt returned {state['prompted']!r}") + return f"no dialog shows in the frame's page: {page!r}" + if page["dialog"] != {"type": "prompt", "message": QUESTION, "default_text": ""}: + return f"the frame's page shows the dialog as {page['dialog']!r}" + x, y = self._at("save", scale) + latest = (await self.shot())[0].structuredContent["frame_id"] + if problem := self.refused(await self.tool("computer_click", {"x": x, "y": y, "frame_id": latest}), + "dialog_open"): + return f"a click while it's open: {problem}" + await self.act("computer_dialog", {"accept": True, "text": WORD}) + return await self.settled(lambda s: s["prompted"] == WORD and s["saved_count"] == before, + lambda s: {"prompted": s["prompted"], "saved_count": s["saved_count"]}) + + async def upload() -> str | None: + self.upload = f"{PROBE_ROLE}-{uuid.uuid4().hex[:8]}.txt" + text = f"uploaded by the probe {time.time_ns()}" + added = await self.driver.add_file(f"{UPLOADS}/{self.upload}", text.encode()) + if "error" in added: + return f"data/add refused it: {added['error'].get('message')}" + x, y = self._at("file", scale, dx=40) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) + if problem := await self.showing(lambda p: p.get("file_chooser") is not None): + return f"no file chooser shows: {problem}" + outside = await self.tool("computer_upload", {"paths": [f"../{STAGE}/page_fixture.py"]}) + if problem := self.refused(outside, "invalid_argument"): + return f"a file outside ~/{UPLOADS}: {problem}" + await self.act("computer_upload", {"paths": [self.upload]}) + return await self.settled(lambda s: s["uploaded"] == {"name": self.upload, "content": text}, + lambda s: s["uploaded"]) + + async def reload() -> str | None: + loads = (await self.state())["loads"] + await self.act("computer_history", {"direction": "reload"}) + return await self.settled(lambda s: s["loads"] == loads + 1, lambda s: f"{s['loads'] - loads} loads") + + async def navigate() -> str | None: + await self.act("computer_navigate", {"url": two}) + return await self.showing(lambda p: p.get("url") == two and p.get("title") == TWO_TITLE) + + async def back_and_forth() -> str | None: + for direction, url, title in (("back", self.desk, TITLE), ("forward", two, TWO_TITLE)): + await self.act("computer_history", {"direction": direction}) + if problem := await self.showing(lambda p: p.get("url") == url and p.get("title") == title): + return f"after {direction}: {problem}" + return None + + async def tabs() -> str | None: + opened = await self.tool("computer_tabs", {"action": "open", "url": self.desk}) + if opened.isError: + return f"open: {opened.content[0].text}" + active = [t["tab"] for t in opened.structuredContent["tabs"] if t["active"]] + if len(active) != 1 or active[0] == self.tab: + return f"the opened tab isn't the one active: {opened.structuredContent['tabs']}" + other = active[0] + if problem := await self.showing(lambda p: p.get("tab") == other and p.get("url") == self.desk): + return f"after open: {problem}" + await self.act("computer_tabs", {"action": "switch", "tab": self.tab}) + if problem := await self.showing(lambda p: p.get("tab") == self.tab): + return f"after switching back: {problem}" + closed = await self.tool("computer_tabs", {"action": "close", "tab": other}) + if closed.isError or any(t["tab"] == other for t in closed.structuredContent["tabs"]): + return f"close: {closed.content[0].text if closed.isError else closed.structuredContent['tabs']}" + return await self.showing(lambda p: p.get("tab") == self.tab) + + for name, test in ( + ("computer_dialog answers a prompt the frame's page shows, and input waits for it", dialog), + (f"computer_upload gives the page's file chooser a file from ~/{UPLOADS}, and refuses one outside it", + upload), + ("computer_history reload loads the page again", reload), + ("computer_navigate opens a page, and the frame's page names it", navigate), + ("computer_history goes back and forward through the tab's pages", back_and_forth), + ("computer_tabs opens, switches to and closes a tab, and the frame shows the active one", tabs)): + await self.check("X", name, test, COMPUTER_PAGE) + async def _recording(self) -> None: if not self.driver.declares(COMPUTER_RECORDING): return diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py new file mode 100644 index 000000000..aa8093e18 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py @@ -0,0 +1,256 @@ +"""The page fixture the conformance probe stages with data/add and starts with computer-exec on a computer whose screen +is a page: a loopback server for the HTML Fixture Desk, the Tk desk's widgets at the same places, which reports their +boxes and every effect that reaches it here, written to /layout.json and /state.json as the Tk +desk writes them, and reads /inbox.txt back to it. Its port goes to /port. Run as +``python3 page_fixture.py ``.""" +from __future__ import annotations + +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +TITLE = "Fixture Desk" +TWO = "/two" +TWO_TITLE = "Fixture Desk two" +QUESTION = "The Fixture Desk asks for a word" + +DESK = """ +Fixture Desk + + + + + + + +
+ + +
+ + + + + +""".replace("%QUESTION%", QUESTION) + +PAGE_TWO = f""" +{TWO_TITLE} + +

{TWO_TITLE}

+""" + + +class Desk(BaseHTTPRequestHandler): + state_dir: Path + + def do_GET(self) -> None: + if self.path == "/": + return self._send(200, DESK.encode(), "text/html; charset=utf-8") + if self.path == TWO: + return self._send(200, PAGE_TWO.encode(), "text/html; charset=utf-8") + if self.path == "/state": + kept = self.state_dir / "state.json" + return self._send(200, kept.read_bytes() if kept.exists() else b"null", "application/json") + message = self.state_dir / "inbox.txt" + if self.path == "/inbox" and message.exists(): + return self._send(200, message.read_bytes(), "text/plain; charset=utf-8") + self._send(404, b"", "text/plain") + + def do_POST(self) -> None: + name = {"/state": "state.json", "/layout": "layout.json"}.get(self.path) + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + if name is None: + return self._send(404, b"", "text/plain") + json.loads(body) + part = self.state_dir / f".{name}.part" + part.write_bytes(body) + os.replace(part, self.state_dir / name) + self._send(204, b"", "text/plain") + + def _send(self, status: int, body: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + pass + + +def main(state_dir: Path) -> None: + state_dir.mkdir(parents=True, exist_ok=True) + Desk.state_dir = state_dir + server = ThreadingHTTPServer(("127.0.0.1", 0), Desk) + part = state_dir / ".port.part" + part.write_text(str(server.server_address[1])) + os.replace(part, state_dir / "port") + server.serve_forever() + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index a6a6e85a3..42071d4ed 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -19,16 +19,28 @@ from agentenv_protocol.computer import ( ComputerDriverEnvironment, ExecResult, + FileChooser, + InvalidArgument, + PageDialog, + PageState, + Tab, click, + dialog, drag, + history, + navigate, + page_state, pointer_up, press, run_in_session, scroll, + tabs, type_text, + upload, ) from agentenv_protocol.computer import _linux as linux from agentenv_protocol.computer.conformance.checks import STAGE +from agentenv_protocol.computer.conformance.page_fixture import QUESTION, TITLE, TWO, TWO_TITLE CLIPBOARD = "agentenv-probe clipboard" FIELD = (1080, 500) @@ -39,11 +51,16 @@ "pad": [840, 160, 400, 300], "wheel": [840, 500, 200, 120], "field": [*FIELD, 48, 48], "list_row_height": 24, "markers": [[FIELD[0] + x, FIELD[1] + y, c] for x, y, c in MARKERS], "cell": 50, "clipboard": CLIPBOARD} KEYSYMS = {" ": "space", ",": "comma", "!": "exclam", "\t": "Tab", "\n": "Return"} +PAGE_LAYOUT = {**LAYOUT, "ask": [40, 600, 120, 44], "file": [40, 670, 300, 34]} +DESK_URL = "http://127.0.0.1:4343/" +TITLES = {DESK_URL: TITLE, DESK_URL + TWO.lstrip("/"): TWO_TITLE} class FakeDesk: """What the Fixture Desk would record of the input the fake screen gets.""" + layout = LAYOUT + def __init__(self, driver: FakeDriver) -> None: self.driver = driver self.started = False @@ -69,7 +86,7 @@ def read(self) -> dict: return self.state def _in(self, box: str, x: float, y: float) -> tuple[float, float] | None: - x0, y0, w, h = LAYOUT[box] + x0, y0, w, h = self.layout[box] return (x - x0, y - y0) if x0 <= x < x0 + w and y0 <= y < y0 + h else None def click(self, x: float, y: float, button: str, count: int) -> None: @@ -192,6 +209,122 @@ def _fixture(self, argv: list[str]) -> ExecResult | None: return None +class FakePageDesk(FakeDesk): + """What the HTML desk would record: the Tk desk's record, with a prompt behind its Ask button, a file chooser behind + its file input, and the fields only a page has.""" + + layout = PAGE_LAYOUT + + def __init__(self, driver: PageFixtureDriver) -> None: + super().__init__(driver) + self.state.update(pasted=[], prompts=0, prompted=None, uploaded=None, loads=0) + + def click(self, x: float, y: float, button: str, count: int) -> None: + super().click(x, y, button, count) + if not self.started: + return + if self._in("ask", x, y) is not None: + if self.driver.answers_dialogs: + self.state.update(prompts=self.state["prompts"] + 1, prompted="") + else: + self.driver.open_dialog = PageDialog("prompt", QUESTION) + elif self._in("file", x, y) is not None: + self.driver.chooser = FileChooser() + + +class PageFixtureDriver(FixtureDriver): + """The fixture driver as a page driver: its screen a browser's active tab, its page tools over fake tabs and their + history, and the HTML desk's server among its commands.""" + + screen_kind = "page" + + def __init__(self, answers_dialogs: bool = False, **kwargs) -> None: + super().__init__(**kwargs) + self.desk = FakePageDesk(self) + self.answers_dialogs = answers_dialogs + self.open_dialog: PageDialog | None = None + self.chooser: FileChooser | None = None + self.opened = 0 + self.tabs = [{"id": "start", "urls": ["about:blank"], "at": 0, "active": True}] + + def _active(self) -> dict: + return next(tab for tab in self.tabs if tab["active"]) + + def _visit(self, url: str) -> None: + if url == DESK_URL: + self.desk.state["loads"] += 1 + + def _listed(self) -> list[Tab]: + return [Tab(t["id"], t["urls"][t["at"]], TITLES.get(t["urls"][t["at"]], ""), t["active"]) for t in self.tabs] + + @navigate + async def go_to(self, url): + tab = self._active() + tab["urls"], tab["at"] = tab["urls"][:tab["at"] + 1] + [url], tab["at"] + 1 + self._visit(url) + + @history + async def move_in_history(self, direction): + tab = self._active() + tab["at"] = max(0, min(len(tab["urls"]) - 1, tab["at"] + {"back": -1, "forward": 1}.get(direction, 0))) + self._visit(tab["urls"][tab["at"]]) + + @tabs + async def tab_action(self, action, tab, url): + known = next((t for t in self.tabs if t["id"] == tab), None) + if action in ("switch", "close") and known is None: + raise InvalidArgument(f"no tab {tab}") + if action == "open": + self.opened += 1 + for t in self.tabs: + t["active"] = False + self.tabs.append({"id": f"t{self.opened}", "urls": [url or "about:blank"], "at": 0, "active": True}) + self._visit(url) + elif action == "switch": + for t in self.tabs: + t["active"] = t is known + elif action == "close": + self.tabs.remove(known) + if known["active"]: + self.tabs[-1]["active"] = True + return self._listed() + + @dialog + async def answer_dialog(self, accept, text): + if self.open_dialog is None: + raise InvalidArgument("no dialog is open") + self.open_dialog = None + self.desk.state.update(prompts=self.desk.state["prompts"] + 1, prompted=(text or "") if accept else None) + + @upload + async def choose_files(self, paths): + uploads = (Path.home() / "Uploads").resolve() + files = [(uploads / path).resolve() for path in paths] + if any(uploads not in file.parents for file in files): + raise InvalidArgument("a file outside ~/Uploads") + self.chooser = None + self.desk.state["uploaded"] = {"name": files[0].name, "content": files[0].read_text()} + + @page_state + async def page_now(self): + tab = self._active() + url = tab["urls"][tab["at"]] + return PageState(tab["id"], url, TITLES.get(url, ""), self.open_dialog, self.chooser) + + def _fixture(self, argv: list[str]) -> ExecResult | None: + text = " ".join(argv) + if f"python3 {STAGE}/page_fixture.py" in text: + self.desk.start() + return ExecResult(0, b"4343\n") + if argv == ["cat", f"{STAGE}/state/port"]: + return ExecResult(0, b"4343") if self.desk.started else ExecResult(1) + if argv[:1] == ["cat"] and argv[1].endswith("layout.json") and self.desk.started: + return ExecResult(0, json.dumps(PAGE_LAYOUT).encode()) + if argv == ["stat", "-c", "%u", f"{STAGE}/page_fixture.py"]: + return ExecResult(0, b"1000\n") + return super()._fixture(argv) + + @contextlib.contextmanager def served(driver: ComputerDriverEnvironment) -> Iterator[str]: """``driver`` on loopback in a thread of its own: the URL its card is served at.""" diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_page.py b/packages/agentenv-protocol/tests/test_computer_conformance_page.py new file mode 100644 index 000000000..d9a95d6ab --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance_page.py @@ -0,0 +1,81 @@ +"""The probe on a driver whose screen is a page: it stages the HTML desk in a tab of its own, verifies the input checks +and the page tools on it and leaves the tab it was given as it was; without the page tools the input checks are +unverified; a browser that answers dialogs itself makes the dialog check not applicable; typing that pastes fails.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from probe_fake import FixtureDriver, PageFixtureDriver, served # noqa: E402 + +from agentenv_protocol.computer import COMPUTER_PAGE, type_text # noqa: E402 +from agentenv_protocol.computer.conformance import ProbeReport, run_probe # noqa: E402 +from agentenv_protocol.computer.conformance import checks # noqa: E402 + + +@pytest.fixture(autouse=True) +def home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(checks, "RECORD_S", 0.8) + return tmp_path + + +async def _probe(driver) -> ProbeReport: + with served(driver) as url: + return await run_probe(url, "full") + + +def _status(report: ProbeReport, start: str) -> tuple[str, str]: + return next((c.status, c.detail) for c in report.checks if c.name.startswith(start)) + + +@pytest.mark.asyncio +async def test_a_page_driver_passes_every_check_on_the_html_desk_and_the_probe_closes_its_own_tab(home): + driver = PageFixtureDriver() + report = await _probe(driver) + + failed = [f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed"] + page = [c for c in report.checks if COMPUTER_PAGE in c.extensions] + assert report.passed, failed + assert report.counts()["unverified"] == 0 and report.extensions[COMPUTER_PAGE] == "verified" + assert len(page) == 6 and {c.status for c in page} == {"passed"} + assert [(t["id"], t["urls"]) for t in driver.tabs] == [("start", ["about:blank"])] + assert not (home / ".agentenv-probe").exists() and not list((home / "Uploads").iterdir()) + assert ["kill", "4343"] in driver.commands and driver.desk.state["prompted"] == checks.WORD + + +@pytest.mark.asyncio +async def test_a_page_driver_without_the_page_tools_leaves_the_input_checks_unverified_and_says_why(): + class NoPageTools(FixtureDriver): + screen_kind = "page" + + report = await _probe(NoPageTools()) + + unverified = {c.detail for c in report.checks if c.status == "unverified" and c.check != "P8"} + assert report.passed and len(unverified) == 1 + assert "opened with computer-page/v1; the driver lacks ['urn:agentenv:computer-page/v1']" in unverified.pop() + + +@pytest.mark.asyncio +async def test_a_browser_that_answers_dialogs_itself_makes_the_dialog_check_not_applicable(): + report = await _probe(PageFixtureDriver(answers_dialogs=True)) + + assert report.passed + assert _status(report, "computer_dialog") == ("not_applicable", + "the browser answers dialogs itself: the prompt returned ''") + + +@pytest.mark.asyncio +async def test_typing_that_arrives_as_a_paste_fails_the_clipboard_check_on_a_page(): + class Pastes(PageFixtureDriver): + @type_text + async def type_in(self, text): + await super().type_in(text) + self.desk.state["pasted"].append(text) + + report = await _probe(Pastes()) + + status, detail = _status(report, "typing leaves the clipboard as it was") + assert status == "failed" and detail.startswith("typing arrived as a paste of ['Hello, Wörld! 東京 AbC'") diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index a35efd442..8503ab774 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -156,12 +156,13 @@ async def drag_along(self, path, button, duration_ms, modifiers) -> None: @scroll async def scroll_at(self, x, y, direction, amount, modifiers) -> None: - dx = {"left": -1, "right": 1}.get(direction, 0) * amount * WHEEL_PX - dy = {"up": -1, "down": 1}.get(direction, 0) * amount * WHEEL_PX + """One wheel event a notch, as a mouse wheel sends them.""" + dx = {"left": -1, "right": 1}.get(direction, 0) * WHEEL_PX + dy = {"up": -1, "down": 1}.get(direction, 0) * WHEEL_PX self._pointer = (x, y) await self._input([{"method": "Input.dispatchMouseEvent", "params": { "type": "mouseWheel", "x": x, "y": y, "deltaX": dx, "deltaY": dy, - "modifiers": _modifiers(modifiers) | self._mask()}}]) + "modifiers": _modifiers(modifiers) | self._mask()}}] * amount) @pointer_move async def move(self, x, y) -> None: diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index c80e31a07..6d20b9689 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1,10 +1,10 @@ """The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display -and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; -deployed through the computer provider like any computer, its computer_* tools see and drive the page, and its page -tools navigate, move in history, answer a dialog, use tabs and upload; nothing runs X and nothing listens but the driver -and the site proxy; a killed app server or driver is started again by supervisord and the computer is ready again; and -chromium killed fails it for good, computer-ready/v1 saying why and the tools answering with it. Needs Docker and the -local registry on localhost:5000.""" +and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it, +its input checks and page tools verified on the HTML desk; deployed through the computer provider like any computer, +its computer_* tools see and drive the page, and its page tools navigate, move in history, answer a dialog, use tabs +and upload; nothing runs X and nothing listens but the driver and the site proxy; a killed app server or driver is +started again by supervisord and the computer is ready again; and chromium killed fails it for good, computer-ready/v1 +saying why and the tools answering with it. Needs Docker and the local registry on localhost:5000.""" import asyncio import json @@ -20,7 +20,7 @@ from PIL import Image from agentenv_protocol import client as protocol_v1 -from agentenv_protocol.computer import COMPUTER_READY +from agentenv_protocol.computer import COMPUTER_PAGE, COMPUTER_READY from agentenv_protocol.computer.conformance.client import image from agentenv_protocol.website_browser import BROWSER @@ -94,7 +94,7 @@ def log_message(self, *args): """ -def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver(browser_computer): +def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver_its_input_and_page_tools_verified(browser_computer): results = browser_computer.metadata["composed"]["container"] probe = results[ARCH]["probe"] @@ -104,6 +104,7 @@ def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver(browser_comp "not passed": [(c["check"], c["name"], c["status"], c["detail"][:160]) for c in report["checks"] if c["status"] != "passed"]}, indent=1)) assert results[ARCH]["status"] == "ready" and probe["passed"], results[ARCH] + assert probe["checks"]["unverified"] == 0 and probe["extensions"][COMPUTER_PAGE] == "verified", probe assert [app.environment_name for app in browser_computer.composed_apps()] == ["chromium"] diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py index 5eef37282..f6ee8f1e4 100644 --- a/tst/unit/drivers/page_driver_test.py +++ b/tst/unit/drivers/page_driver_test.py @@ -110,6 +110,15 @@ async def test_with_shift_held_typed_text_is_shifted_and_with_control_held_a_cli assert events[-1]["params"]["type"] == "mouseWheel" and events[-1]["params"]["modifiers"] == 3 +@pytest.mark.asyncio +async def test_a_scroll_is_one_wheel_event_a_notch_sent_together(page): + await page.scroll_at(10, 20, "left", 3, []) + + assert [op for op, _ in page.sent] == ["page/input"] + assert [(e["params"]["type"], e["params"]["deltaX"], e["params"]["deltaY"]) for e in _events(page)] == [ + ("mouseWheel", -100, 0)] * 3 + + @pytest.mark.asyncio async def test_typed_text_is_keystrokes_with_shift_where_it_takes_one_and_inserted_text_for_the_rest(page): await page.type_in("Hi!\né") From 223f13aeda30174db0e3a50fb1e32a11fb4aa998 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 19:37:01 -0800 Subject: [PATCH 2/4] fix(computer): P9 stats the desk the probe staged, and the probe closes every tab it opened Co-Authored-By: Claude Opus 5.5 --- .../computer/conformance/checks.py | 22 +++++++++++------- .../agentenv-protocol/tests/probe_fake.py | 2 ++ .../tests/test_computer_conformance_page.py | 23 ++++++++++++++++--- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index a7b500b29..d7ddda155 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -95,8 +95,10 @@ def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: floa self.layout: dict = {} self.fixture_pid: str | None = None self.page = self.features.get("screen") == "page" + self.fixture = "page_fixture.py" if self.page else "fixture.py" self.desk = "" self.tab: str | None = None + self.opened: list[str] = [] self.upload: str | None = None async def basic(self) -> None: @@ -538,14 +540,13 @@ def _grants(self) -> None: async def _stage(self) -> str | None: """The Fixture Desk started, or what stopped it: the Tk desk, or on a page, the HTML desk's server, with the desk opened in a tab of the probe's own.""" - fixture = "page_fixture.py" if self.page else "fixture.py" - source = resources.files(__package__).joinpath(fixture).read_bytes() - added = await self.driver.add_file(f"{STAGE}/{fixture}", source) + source = resources.files(__package__).joinpath(self.fixture).read_bytes() + added = await self.driver.add_file(f"{STAGE}/{self.fixture}", source) if "error" in added: return f"data/add refused it: {added['error'].get('message')}" await self.driver.run("sh", "-c", f"rm -rf {STAGE}/state") started = await self.driver.run( - "sh", "-c", f"python3 {STAGE}/{fixture} {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") + "sh", "-c", f"python3 {STAGE}/{self.fixture} {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") self.fixture_pid = started["stdout"].strip() or None deadline = time.monotonic() + FIXTURE_START_S while time.monotonic() < deadline: @@ -572,14 +573,18 @@ async def _open_desk(self) -> str | None: if opened.isError: return f"computer_tabs couldn't open it: {opened.content[0].text}" self.tab = next((t["tab"] for t in opened.structuredContent["tabs"] if t["active"]), None) + self.opened += [self.tab] if self.tab else [] return None if self.tab else f"the tab computer_tabs opened isn't active: {opened.structuredContent['tabs']}" async def _unstage(self) -> None: - if self.tab: + if self.opened: with contextlib.suppress(Exception): listed = await self.tool("computer_tabs", {"action": "list"}) - if len(listed.structuredContent["tabs"]) > 1: - await self.tool("computer_tabs", {"action": "close", "tab": self.tab}) + left = [t["tab"] for t in listed.structuredContent["tabs"]] + for tab in [t for t in left if t in self.opened]: + if len(left) > 1: + await self.tool("computer_tabs", {"action": "close", "tab": tab}) + left.remove(tab) if self.fixture_pid: await self.driver.run("kill", self.fixture_pid) await self.driver.run("rm", "-rf", STAGE) @@ -600,7 +605,7 @@ def _at(self, box: str, scale: float, dx: float | None = None, dy: float | None async def _security(self) -> None: async def one_user() -> str | None: uid = (await self.driver.run("id", "-u"))["stdout"].strip() - owner = (await self.driver.run("stat", "-c", "%u", f"{STAGE}/fixture.py"))["stdout"].strip() + owner = (await self.driver.run("stat", "-c", "%u", f"{STAGE}/{self.fixture}"))["stdout"].strip() if uid == "0": return "commands run as root" return None if owner == uid else f"data/add wrote as uid {owner}, commands run as uid {uid}" @@ -919,6 +924,7 @@ async def tabs() -> str | None: if len(active) != 1 or active[0] == self.tab: return f"the opened tab isn't the one active: {opened.structuredContent['tabs']}" other = active[0] + self.opened.append(other) if problem := await self.showing(lambda p: p.get("tab") == other and p.get("url") == self.desk): return f"after open: {problem}" await self.act("computer_tabs", {"action": "switch", "tab": self.tab}) diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index 42071d4ed..e780a593a 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -322,6 +322,8 @@ def _fixture(self, argv: list[str]) -> ExecResult | None: return ExecResult(0, json.dumps(PAGE_LAYOUT).encode()) if argv == ["stat", "-c", "%u", f"{STAGE}/page_fixture.py"]: return ExecResult(0, b"1000\n") + if argv == ["stat", "-c", "%u", f"{STAGE}/fixture.py"]: + return ExecResult(1, b"", b"stat: cannot statx: No such file or directory") return super()._fixture(argv) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_page.py b/packages/agentenv-protocol/tests/test_computer_conformance_page.py index d9a95d6ab..e78217c01 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance_page.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance_page.py @@ -1,6 +1,7 @@ """The probe on a driver whose screen is a page: it stages the HTML desk in a tab of its own, verifies the input checks -and the page tools on it and leaves the tab it was given as it was; without the page tools the input checks are -unverified; a browser that answers dialogs itself makes the dialog check not applicable; typing that pastes fails.""" +and the page tools on it and leaves the tab it was given as it was, closing the tabs it opened even when a check +fails; without the page tools the input checks are unverified; a browser that answers dialogs itself makes the dialog +check not applicable; typing that pastes fails.""" from __future__ import annotations import pytest @@ -10,7 +11,7 @@ from probe_fake import FixtureDriver, PageFixtureDriver, served # noqa: E402 -from agentenv_protocol.computer import COMPUTER_PAGE, type_text # noqa: E402 +from agentenv_protocol.computer import COMPUTER_PAGE, InvalidArgument, tabs, type_text # noqa: E402 from agentenv_protocol.computer.conformance import ProbeReport, run_probe # noqa: E402 from agentenv_protocol.computer.conformance import checks # noqa: E402 @@ -46,6 +47,22 @@ async def test_a_page_driver_passes_every_check_on_the_html_desk_and_the_probe_c assert ["kill", "4343"] in driver.commands and driver.desk.state["prompted"] == checks.WORD +@pytest.mark.asyncio +async def test_a_tab_the_probe_opened_is_closed_though_the_check_that_opened_it_failed(): + class NoSwitch(PageFixtureDriver): + @tabs + async def tab_action(self, action, tab, url): + if action == "switch": + raise InvalidArgument("this browser can't switch tabs") + return await super().tab_action(action, tab, url) + + driver = NoSwitch() + report = await _probe(driver) + + assert _status(report, "computer_tabs")[0] == "failed" + assert [t["id"] for t in driver.tabs] == ["start"] + + @pytest.mark.asyncio async def test_a_page_driver_without_the_page_tools_leaves_the_input_checks_unverified_and_says_why(): class NoPageTools(FixtureDriver): From 6862f4f0222eb9fcbecdca4a0a650c2ba63a0ded Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 20:01:37 -0800 Subject: [PATCH 3/4] fix(computer): a tab opened headless loads its page at the window's size Target.createTarget with a URL starts loading before the tab is attached and held to the window, so its page loaded at 1280x713 and then resized. Headless, the tab now opens blank and navigates once held. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/website_browser.py | 23 ++++++++++++++++--- .../tests/test_website_browser.py | 12 ++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 8ea633257..0e707260e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -45,6 +45,8 @@ AGENTENV, UNKNOWN, EXIT, STOP = "agentenv", "unknown", "exit", "stop" #: How long the active tab's check waits on each page. ACTIVE_CHECK_TIMEOUT_S = 2.0 +#: How long a tab opened headless may take to be attached and held to its window's size. +PIN_TIMEOUT_S = 10.0 #: How much of a download each read hashes. HASH_CHUNK_BYTES = 1 << 20 @@ -214,6 +216,7 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl #: input sent there waits on to learn a dialog opened. self._shots: dict[str, str] = {} self._dialog_opened: dict[str, asyncio.Event] = {} + self._pinned: dict[str, asyncio.Event] = {} self._tasks: set[asyncio.Future] = set() devtools.listen(self._event) @@ -238,6 +241,7 @@ def ended(self, how: str) -> None: self._choosers.clear() self._shots.clear() self._dialog_opened.clear() + self._pinned.clear() async def close_tab(self, target: str) -> None: """Close the tab ``target``, which the record says agentenv closed.""" @@ -252,8 +256,15 @@ async def tab_list(self) -> list[dict]: for tab in self.tabs.values()] async def open_tab(self, url: Optional[str] = None) -> str: - """A new tab at ``url``, else blank, brought to the front; its id.""" - target = (await self.devtools.call("Target.createTarget", {"url": url or "about:blank"}))["targetId"] + """A new tab at ``url``, else blank, brought to the front; its id. Headless, it opens blank and goes to ``url`` + once it's held to the window's size, so its page loads at that size.""" + held = self.viewport is not None and url + start = "about:blank" if held else url or "about:blank" + target = (await self.devtools.call("Target.createTarget", {"url": start}))["targetId"] + if held: + await asyncio.wait_for(self._pinned.setdefault(target, asyncio.Event()).wait(), PIN_TIMEOUT_S) + session = next(s for s, t in self._sessions.items() if t == target) + await self.devtools.call("Page.navigate", {"url": url}, session_id=session) await self._bring_to_front(target) return target @@ -502,6 +513,7 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N async def _let_run(self, session: str) -> None: """A page it attached to, enabled and, headless, held to its window's size, then told to run: a page attached as it opens, a popup say, or any new page headless, waits for that before it runs.""" + target = self._sessions.get(session) try: await self.devtools.call("Page.enable", session_id=session) if self.viewport is not None: @@ -510,11 +522,16 @@ async def _let_run(self, session: str) -> None: "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) await self.devtools.call("Page.setInterceptFileChooserDialog", {"enabled": True}, session_id=session) finally: - await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) + try: + await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) + finally: + if target is not None: + self._pinned.setdefault(target, asyncio.Event()).set() def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: if (target := self._sessions.pop(params["sessionId"], None)) is not None: self._shots.pop(target, None) + self._pinned.pop(target, None) self._choosers.pop(params["sessionId"], None) self._dialog_opened.pop(params["sessionId"], None) diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 97f1905fa..0e9c5a088 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -636,14 +636,15 @@ async def test_it_navigates_the_active_tab_and_moves_in_its_history_or_says_why_ @pytest.mark.asyncio -async def test_a_tab_it_opens_or_activates_is_brought_to_the_front_and_the_active_one_and_one_it_closes_its_own(): +async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_and_one_it_closes_is_its_own(): devtools = Tabbed() control = await _tabs(devtools, "a") - opened = await control.open_tab("https://mail.test/b") + opening = asyncio.ensure_future(control.open_tab("https://mail.test/b")) + await asyncio.sleep(0) devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b")) devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")}) - await asyncio.sleep(0) + opened = await opening after_open = await control.tab_list() await control.activate("a") after_switch = await control.tab_list() @@ -652,7 +653,10 @@ async def test_a_tab_it_opens_or_activates_is_brought_to_the_front_and_the_activ await control.close("b") devtools.send("Target.targetDestroyed", {"targetId": "b"}) - assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "https://mail.test/b"}, None)] + on_b = [name for name, _, session in devtools.calls if session == "s-b"] + assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank"}, None)] + assert ({"url": "https://mail.test/b"}, "s-b") in _sent(devtools, "Page.navigate") + assert on_b.index("Emulation.setDeviceMetricsOverride") < on_b.index("Page.navigate") assert [(tab["id"], tab["active"]) for tab in after_open] == [("a", False), ("b", True)] assert [(tab["id"], tab["active"]) for tab in after_switch] == [("a", True), ("b", False)] assert ({}, "s-a") in _sent(devtools, "Page.bringToFront") From 1a94a58a60139b4e18f3709d1fa6c90ee2d475ec Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 3 Nov 2026 20:31:29 -0800 Subject: [PATCH 4/4] fix(computer): the active tab is the visible one with focus A hidden tab whose page had input still says document.hasFocus(), so after an agent typed in one tab and opened another, the hidden first tab stayed the active one and got its input. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 8 +++++--- .../src/agentenv_protocol/website_browser.py | 7 ++++--- packages/agentenv-protocol/tests/test_website_browser.py | 9 +++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 5f57e9ff1..16afa7c85 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -812,7 +812,8 @@ are the profile's template. The base class adds: - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches to every page with flattened sessions, telling each page it attached to as it opened, a popup say, to run, and keeps a record, `since` its server started: open tabs in opened order, each with its URL and title as the browser has them - when the record is read (Chrome sends no event for a title), the one whose page has focus `active`; `closed_tabs`, + when the record is read (Chrome sends no event for a title), the visible one whose page has focus `active`; + `closed_tabs`, each as of its page's last load, with `closed_at` and `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state @@ -832,8 +833,9 @@ are the profile's template. The base class adds: drivers and agentenv's own steps alike: - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab; - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab - ids. The active tab is the one a dialog is open on, else the one whose page has focus, else the one - agentenv last brought to the front; + ids. The active tab is the one a dialog is open on, else the visible one whose page has focus (a hidden + tab's page that had input still says it has focus), else the one agentenv last brought to the front. + Headless, a tab opened at a URL opens blank and goes there once it's held to the window's size; - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab; - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so alone, it intercepts; diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 0e707260e..d88120714 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -375,8 +375,9 @@ async def hidden_page(self, url: str = "about:blank") -> AsyncIterator[str]: await self.devtools.call("Target.closeTarget", {"targetId": target}) async def active_tab(self) -> Optional[str]: - """The tab a dialog is open on, whose page it blocks from saying; else the tab whose page has focus, else the - one agentenv last brought to the front, else the first visible one: CDP has no event for a tab made active.""" + """The tab a dialog is open on, whose page it blocks from saying; else the visible tab whose page has focus, as + a hidden tab's page that had input still says it has; else the one agentenv last brought to the front, else the + first visible one: CDP has no event for a tab made active.""" if dialogs := [dialog["tab"] for dialog in self._open_dialogs.values() if dialog["tab"] in self.tabs]: return dialogs[-1] visible = None @@ -390,7 +391,7 @@ async def active_tab(self) -> Optional[str]: timeout_s=ACTIVE_CHECK_TIMEOUT_S))["result"]["value"] except (ConnectionError, DevToolsError, asyncio.TimeoutError, KeyError): continue - if state[1]: + if state[0] == "visible" and state[1]: return target if state[0] == "visible" and visible is None: visible = target diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 0e9c5a088..6ae0861fd 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -663,6 +663,15 @@ async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_a assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs] == [("b", "agentenv")] +@pytest.mark.asyncio +async def test_a_hidden_tab_whose_page_still_says_it_has_focus_isnt_the_active_one(): + states = {"s-a": ["hidden", True], "s-b": ["visible", True]} + devtools = Tabbed({"Runtime.evaluate": lambda params, session: {"result": {"value": states[session]}}}) + control = await _tabs(devtools, "a", "b") + + assert await control.active_tab() == "b" + + @pytest.mark.asyncio async def test_a_dialog_makes_its_tab_the_active_one_its_state_names_it_and_the_agent_answers_it(): devtools = Tabbed()