diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 3c4c68f22..16afa7c85 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -625,12 +625,20 @@ driver booted with a control token is probed with it: `--token`, by default | Level | Checks | Needs on the computer | |---|---|---| | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | -| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held | Linux, `python3` with Tk, a 1280x800 screen or larger, `computer-exec` and `data/add` | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the window's widgets report, reads back what reached them, and then removes the fixture. When the -fixture can't be staged, the `full` checks are unverified rather than failed. The probe leaves out +fixture can't be staged, the `full` checks are unverified rather than failed. + +On a driver whose card says `screen: "page"`, the desk is an HTML page with the same widgets at the +same places (`conformance/page_fixture.py`). The probe starts its server on loopback, which writes +what the page reports to the same files, and opens the desk with `computer_tabs` in a tab of its +own. "Typing leaves the clipboard as it was" fails there when typing arrives as a paste. After the +input checks it checks the page tools against the desk; a browser that answers dialogs itself makes +the dialog check not applicable. It then closes its tab and removes the desk, and the file it put in +`~/Uploads`. The tab that was open before keeps its page and history. The probe leaves out the checks for an extension the card doesn't declare. P8 is not applicable to a driver without leasing, and unverified for one with it until the probe has lease checks. The probe calls as the default role; the tool-access check hides a tool from a role of its own, `agentenv-probe-`, @@ -804,7 +812,8 @@ are the profile's template. The base class adds: - **Its control connection**, `self.control`, a `BrowserControl` over the pipe. At each launch it attaches to every page with flattened sessions, telling each page it attached to as it opened, a popup say, to run, and keeps a record, `since` its server started: open tabs in opened order, each with its URL and title as the browser has them - when the record is read (Chrome sends no event for a title), the one whose page has focus `active`; `closed_tabs`, + when the record is read (Chrome sends no event for a title), the visible one whose page has focus `active`; + `closed_tabs`, each as of its page's last load, with `closed_at` and `closed_by` (`agentenv`, over this connection; `unknown`, anyone else; `exit` or `stop`, when the browser ended on its own or its server stopped it); `downloads`, into `~/Downloads`, each with its state @@ -824,8 +833,9 @@ are the profile's template. The base class adds: drivers and agentenv's own steps alike: - `navigate {url}` and `history {direction}` (`back`, `forward` or `reload`), in the active tab; - `tabs/list`, `tabs/open {url?}`, `tabs/activate {tab}` and `tabs/close {tab}`, by the browser's own tab - ids. The active tab is the one a dialog is open on, else the one whose page has focus, else the one - agentenv last brought to the front; + ids. The active tab is the one a dialog is open on, else the visible one whose page has focus (a hidden + tab's page that had input still says it has focus), else the one agentenv last brought to the front. + Headless, a tab opened at a URL opens blank and goes there once it's held to the window's size; - `dialog/answer {accept, text?}`, the agent's answer to the dialog open on the active tab; - `files/choose {paths}`, files in `~/Uploads` for the file chooser open on it, which headless, and so alone, it intercepts; diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index 074767c40..d7ddda155 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -28,6 +28,7 @@ COMPUTER_DISPLAY, COMPUTER_EXEC, COMPUTER_KEYBOARD, + COMPUTER_PAGE, COMPUTER_POINTER, COMPUTER_READY, COMPUTER_RECORDING, @@ -39,6 +40,7 @@ from ..recording import RECORDINGS_DIR from .client import Driver, image from .mp4 import read_mp4 +from .page_fixture import QUESTION, TITLE, TWO, TWO_TITLE from .report import CheckResult MINIMUM_TOOLS = ("computer_screenshot", "computer_click", "computer_drag", "computer_scroll", "computer_type", @@ -46,6 +48,10 @@ DATA_PLANE = (INTAKE_EXTENSION_URI, DATA_OBJECTS_EXTENSION_URI) #: Where the fixture is staged and keeps its state, relative to the session user's home. STAGE = ".agentenv-probe" +#: Where a page driver's computer_upload takes files from, relative to the session user's home. +UPLOADS = "Uploads" +#: What the probe answers the page fixture's prompt with. +WORD = "tangerine" #: What the tool-access check's role, new each run, and the probe's request ids start with. PROBE_ROLE = "agentenv-probe" #: How long the probe waits for an effect to reach the fixture, and how often it looks. @@ -72,6 +78,10 @@ Test = Callable[[], Awaitable[str | None]] +class NotApplicable(Exception): + """Raised by a test whose check doesn't apply to this driver, with why.""" + + class Probe: """One probe run: the driver, an MCP session on its tools, and the results so far. A test returns what went wrong, or None when its check passes; a test that raises fails with what it raised.""" @@ -84,6 +94,12 @@ def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: floa self.features = driver.params(COMPUTER) self.layout: dict = {} self.fixture_pid: str | None = None + self.page = self.features.get("screen") == "page" + self.fixture = "page_fixture.py" if self.page else "fixture.py" + self.desk = "" + self.tab: str | None = None + self.opened: list[str] = [] + self.upload: str | None = None async def basic(self) -> None: for check, section in (("P0", self._ready), ("P1", self._card), ("P2", self._frame), ("P5", self._refusals), @@ -95,11 +111,14 @@ async def basic(self) -> None: self._grants() async def full(self) -> None: - missing = [uri for uri in (COMPUTER_EXEC,) if not self.driver.declares(uri)] + needs = (COMPUTER_EXEC, COMPUTER_PAGE) if self.page else (COMPUTER_EXEC,) + missing = [uri for uri in needs if not self.driver.declares(uri)] if not (protocol.intake_declaration(self.driver.card) or {}).get("add"): missing.append("data/add") if missing: - return self._skip_full(f"the fixture is staged with data/add and computer-exec; the driver lacks {missing}") + how = "data/add and computer-exec, and opened with computer-page/v1" if self.page else \ + "data/add and computer-exec" + return self._skip_full(f"the fixture is staged with {how}; the driver lacks {missing}") try: try: problem = await self._stage() @@ -109,7 +128,8 @@ async def full(self) -> None: return self._skip_full(f"the fixture didn't start: {problem}") for check, section in (("P9", self._security), ("P3", self._geometry), ("P4", self._effects), ("P5", self._unchanged), ("P6", self._typing), ("P7", self._scroll_and_drag), - ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording)): + ("X", self._pointer_input), ("X", self._data_add), ("X", self._recording), + ("X", self._page_tools)): await self._guarded(check, section) finally: await self._unstage() @@ -119,6 +139,9 @@ async def full(self) -> None: async def check(self, check: str, name: str, test: Test, *extensions: str) -> bool: try: problem = await test() + except NotApplicable as e: + self._record(check, name, "not_applicable", str(e), extensions) + return True except Exception as e: problem = f"{type(e).__name__}: {e}" self._record(check, name, "failed" if problem else "passed", problem or "", extensions) @@ -181,6 +204,14 @@ async def settled(self, holds: Callable[[dict], bool], show: Callable[[dict], ob ok, state = await self.until(holds) return None if ok else f"the fixture shows {show(state)!r}" + async def page_now(self) -> dict: + """The page a new frame names, or {} for a frame that names none.""" + return (await self.shot())[0].structuredContent.get("page") or {} + + async def showing(self, holds: Callable[[dict], bool]) -> str | None: + ok, page = await self.until(holds, self.page_now) + return None if ok else f"the frame's page is {page!r}" + @staticmethod def refused(result: CallToolResult, code: str) -> str | None: data = result.structuredContent or {} @@ -507,16 +538,20 @@ def _grants(self) -> None: # full: the fixture async def _stage(self) -> str | None: - source = resources.files(__package__).joinpath("fixture.py").read_bytes() - added = await self.driver.add_file(f"{STAGE}/fixture.py", source) + """The Fixture Desk started, or what stopped it: the Tk desk, or on a page, the HTML desk's server, with the + desk opened in a tab of the probe's own.""" + source = resources.files(__package__).joinpath(self.fixture).read_bytes() + added = await self.driver.add_file(f"{STAGE}/{self.fixture}", source) if "error" in added: return f"data/add refused it: {added['error'].get('message')}" await self.driver.run("sh", "-c", f"rm -rf {STAGE}/state") started = await self.driver.run( - "sh", "-c", f"python3 {STAGE}/fixture.py {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") + "sh", "-c", f"python3 {STAGE}/{self.fixture} {STAGE}/state > {STAGE}/fixture.log 2>&1 & echo $!") self.fixture_pid = started["stdout"].strip() or None deadline = time.monotonic() + FIXTURE_START_S while time.monotonic() < deadline: + if self.page and self.tab is None and (problem := await self._open_desk()): + return problem layout = await self.driver.run("cat", f"{STAGE}/state/layout.json") if layout.get("exit_code") == 0 and layout["stdout"].strip(): self.layout = json.loads(layout["stdout"]) @@ -528,10 +563,33 @@ async def _stage(self) -> str | None: log = await self.driver.run("tail", "-n", "5", f"{STAGE}/fixture.log") return f"no layout after {FIXTURE_START_S:g} s: {log['stdout'].strip() or log['stderr'].strip()}" + async def _open_desk(self) -> str | None: + """Once the page fixture's server has its port, the desk opened in a new tab, which is the probe's.""" + port = await self.driver.run("cat", f"{STAGE}/state/port") + if port.get("exit_code") != 0 or not port["stdout"].strip(): + return None + self.desk = f"http://127.0.0.1:{port['stdout'].strip()}/" + opened = await self.tool("computer_tabs", {"action": "open", "url": self.desk}) + if opened.isError: + return f"computer_tabs couldn't open it: {opened.content[0].text}" + self.tab = next((t["tab"] for t in opened.structuredContent["tabs"] if t["active"]), None) + self.opened += [self.tab] if self.tab else [] + return None if self.tab else f"the tab computer_tabs opened isn't active: {opened.structuredContent['tabs']}" + async def _unstage(self) -> None: + if self.opened: + with contextlib.suppress(Exception): + listed = await self.tool("computer_tabs", {"action": "list"}) + left = [t["tab"] for t in listed.structuredContent["tabs"]] + for tab in [t for t in left if t in self.opened]: + if len(left) > 1: + await self.tool("computer_tabs", {"action": "close", "tab": tab}) + left.remove(tab) if self.fixture_pid: await self.driver.run("kill", self.fixture_pid) await self.driver.run("rm", "-rf", STAGE) + if self.upload: + await self.driver.run("rm", "-f", f"{UPLOADS}/{self.upload}") async def _base(self) -> tuple[str, float, object]: """A fresh frame of the fixture: its id, the display's scale, and its image.""" @@ -547,7 +605,7 @@ def _at(self, box: str, scale: float, dx: float | None = None, dy: float | None async def _security(self) -> None: async def one_user() -> str | None: uid = (await self.driver.run("id", "-u"))["stdout"].strip() - owner = (await self.driver.run("stat", "-c", "%u", f"{STAGE}/fixture.py"))["stdout"].strip() + owner = (await self.driver.run("stat", "-c", "%u", f"{STAGE}/{self.fixture}"))["stdout"].strip() if uid == "0": return "commands run as root" return None if owner == uid else f"data/add wrote as uid {owner}, commands run as uid {uid}" @@ -708,7 +766,11 @@ async def whitespace() -> str | None: async def clipboard() -> str | None: state = await self.state() - return None if state["clipboard"] == self.layout["clipboard"] else f"the clipboard is {state['clipboard']!r}" + if pasted := state.get("pasted"): + return f"typing arrived as a paste of {pasted!r}" + if state["clipboard"] != self.layout["clipboard"]: + return f"the clipboard is {state['clipboard']!r}" + return None async def held() -> str | None: await clear() @@ -789,6 +851,100 @@ async def picked_up() -> str | None: await self.check("X", "a file data/add writes is there for an app to read", picked_up, *DATA_PLANE) + async def _page_tools(self) -> None: + """The page tools against the HTML desk, each confirmed on the desk or in the page a new frame names; those + that leave the desk run last.""" + if not self.page: + return + frame, scale, _ = await self._base() + two = self.desk + TWO.lstrip("/") + + async def dialog() -> str | None: + before = (await self.state())["saved_count"] + await self.act("computer_click", {"x": self._at("ask", scale)[0], "y": self._at("ask", scale)[1], + "frame_id": frame}) + + async def seen() -> tuple[dict, dict]: + return await self.page_now(), await self.state() + + ok, (page, state) = await self.until(lambda s: bool(s[0].get("dialog")) or s[1]["prompts"] > 0, seen) + if not page.get("dialog"): + if state["prompts"]: + raise NotApplicable(f"the browser answers dialogs itself: the prompt returned {state['prompted']!r}") + return f"no dialog shows in the frame's page: {page!r}" + if page["dialog"] != {"type": "prompt", "message": QUESTION, "default_text": ""}: + return f"the frame's page shows the dialog as {page['dialog']!r}" + x, y = self._at("save", scale) + latest = (await self.shot())[0].structuredContent["frame_id"] + if problem := self.refused(await self.tool("computer_click", {"x": x, "y": y, "frame_id": latest}), + "dialog_open"): + return f"a click while it's open: {problem}" + await self.act("computer_dialog", {"accept": True, "text": WORD}) + return await self.settled(lambda s: s["prompted"] == WORD and s["saved_count"] == before, + lambda s: {"prompted": s["prompted"], "saved_count": s["saved_count"]}) + + async def upload() -> str | None: + self.upload = f"{PROBE_ROLE}-{uuid.uuid4().hex[:8]}.txt" + text = f"uploaded by the probe {time.time_ns()}" + added = await self.driver.add_file(f"{UPLOADS}/{self.upload}", text.encode()) + if "error" in added: + return f"data/add refused it: {added['error'].get('message')}" + x, y = self._at("file", scale, dx=40) + await self.act("computer_click", {"x": x, "y": y, "frame_id": frame}) + if problem := await self.showing(lambda p: p.get("file_chooser") is not None): + return f"no file chooser shows: {problem}" + outside = await self.tool("computer_upload", {"paths": [f"../{STAGE}/page_fixture.py"]}) + if problem := self.refused(outside, "invalid_argument"): + return f"a file outside ~/{UPLOADS}: {problem}" + await self.act("computer_upload", {"paths": [self.upload]}) + return await self.settled(lambda s: s["uploaded"] == {"name": self.upload, "content": text}, + lambda s: s["uploaded"]) + + async def reload() -> str | None: + loads = (await self.state())["loads"] + await self.act("computer_history", {"direction": "reload"}) + return await self.settled(lambda s: s["loads"] == loads + 1, lambda s: f"{s['loads'] - loads} loads") + + async def navigate() -> str | None: + await self.act("computer_navigate", {"url": two}) + return await self.showing(lambda p: p.get("url") == two and p.get("title") == TWO_TITLE) + + async def back_and_forth() -> str | None: + for direction, url, title in (("back", self.desk, TITLE), ("forward", two, TWO_TITLE)): + await self.act("computer_history", {"direction": direction}) + if problem := await self.showing(lambda p: p.get("url") == url and p.get("title") == title): + return f"after {direction}: {problem}" + return None + + async def tabs() -> str | None: + opened = await self.tool("computer_tabs", {"action": "open", "url": self.desk}) + if opened.isError: + return f"open: {opened.content[0].text}" + active = [t["tab"] for t in opened.structuredContent["tabs"] if t["active"]] + if len(active) != 1 or active[0] == self.tab: + return f"the opened tab isn't the one active: {opened.structuredContent['tabs']}" + other = active[0] + self.opened.append(other) + if problem := await self.showing(lambda p: p.get("tab") == other and p.get("url") == self.desk): + return f"after open: {problem}" + await self.act("computer_tabs", {"action": "switch", "tab": self.tab}) + if problem := await self.showing(lambda p: p.get("tab") == self.tab): + return f"after switching back: {problem}" + closed = await self.tool("computer_tabs", {"action": "close", "tab": other}) + if closed.isError or any(t["tab"] == other for t in closed.structuredContent["tabs"]): + return f"close: {closed.content[0].text if closed.isError else closed.structuredContent['tabs']}" + return await self.showing(lambda p: p.get("tab") == self.tab) + + for name, test in ( + ("computer_dialog answers a prompt the frame's page shows, and input waits for it", dialog), + (f"computer_upload gives the page's file chooser a file from ~/{UPLOADS}, and refuses one outside it", + upload), + ("computer_history reload loads the page again", reload), + ("computer_navigate opens a page, and the frame's page names it", navigate), + ("computer_history goes back and forward through the tab's pages", back_and_forth), + ("computer_tabs opens, switches to and closes a tab, and the frame shows the active one", tabs)): + await self.check("X", name, test, COMPUTER_PAGE) + async def _recording(self) -> None: if not self.driver.declares(COMPUTER_RECORDING): return diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py new file mode 100644 index 000000000..aa8093e18 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/page_fixture.py @@ -0,0 +1,256 @@ +"""The page fixture the conformance probe stages with data/add and starts with computer-exec on a computer whose screen +is a page: a loopback server for the HTML Fixture Desk, the Tk desk's widgets at the same places, which reports their +boxes and every effect that reaches it here, written to /layout.json and /state.json as the Tk +desk writes them, and reads /inbox.txt back to it. Its port goes to /port. Run as +``python3 page_fixture.py ``.""" +from __future__ import annotations + +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +TITLE = "Fixture Desk" +TWO = "/two" +TWO_TITLE = "Fixture Desk two" +QUESTION = "The Fixture Desk asks for a word" + +DESK = """ +Fixture Desk + + + + + + + +
+ + +
+ + + + + +""".replace("%QUESTION%", QUESTION) + +PAGE_TWO = f""" +{TWO_TITLE} + +

{TWO_TITLE}

+""" + + +class Desk(BaseHTTPRequestHandler): + state_dir: Path + + def do_GET(self) -> None: + if self.path == "/": + return self._send(200, DESK.encode(), "text/html; charset=utf-8") + if self.path == TWO: + return self._send(200, PAGE_TWO.encode(), "text/html; charset=utf-8") + if self.path == "/state": + kept = self.state_dir / "state.json" + return self._send(200, kept.read_bytes() if kept.exists() else b"null", "application/json") + message = self.state_dir / "inbox.txt" + if self.path == "/inbox" and message.exists(): + return self._send(200, message.read_bytes(), "text/plain; charset=utf-8") + self._send(404, b"", "text/plain") + + def do_POST(self) -> None: + name = {"/state": "state.json", "/layout": "layout.json"}.get(self.path) + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + if name is None: + return self._send(404, b"", "text/plain") + json.loads(body) + part = self.state_dir / f".{name}.part" + part.write_bytes(body) + os.replace(part, self.state_dir / name) + self._send(204, b"", "text/plain") + + def _send(self, status: int, body: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + pass + + +def main(state_dir: Path) -> None: + state_dir.mkdir(parents=True, exist_ok=True) + Desk.state_dir = state_dir + server = ThreadingHTTPServer(("127.0.0.1", 0), Desk) + part = state_dir / ".port.part" + part.write_text(str(server.server_address[1])) + os.replace(part, state_dir / "port") + server.serve_forever() + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py index 8ea633257..d88120714 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser.py @@ -45,6 +45,8 @@ AGENTENV, UNKNOWN, EXIT, STOP = "agentenv", "unknown", "exit", "stop" #: How long the active tab's check waits on each page. ACTIVE_CHECK_TIMEOUT_S = 2.0 +#: How long a tab opened headless may take to be attached and held to its window's size. +PIN_TIMEOUT_S = 10.0 #: How much of a download each read hashes. HASH_CHUNK_BYTES = 1 << 20 @@ -214,6 +216,7 @@ def __init__(self, devtools: DevToolsPipe, dialogs: str, viewport: Optional[tupl #: input sent there waits on to learn a dialog opened. self._shots: dict[str, str] = {} self._dialog_opened: dict[str, asyncio.Event] = {} + self._pinned: dict[str, asyncio.Event] = {} self._tasks: set[asyncio.Future] = set() devtools.listen(self._event) @@ -238,6 +241,7 @@ def ended(self, how: str) -> None: self._choosers.clear() self._shots.clear() self._dialog_opened.clear() + self._pinned.clear() async def close_tab(self, target: str) -> None: """Close the tab ``target``, which the record says agentenv closed.""" @@ -252,8 +256,15 @@ async def tab_list(self) -> list[dict]: for tab in self.tabs.values()] async def open_tab(self, url: Optional[str] = None) -> str: - """A new tab at ``url``, else blank, brought to the front; its id.""" - target = (await self.devtools.call("Target.createTarget", {"url": url or "about:blank"}))["targetId"] + """A new tab at ``url``, else blank, brought to the front; its id. Headless, it opens blank and goes to ``url`` + once it's held to the window's size, so its page loads at that size.""" + held = self.viewport is not None and url + start = "about:blank" if held else url or "about:blank" + target = (await self.devtools.call("Target.createTarget", {"url": start}))["targetId"] + if held: + await asyncio.wait_for(self._pinned.setdefault(target, asyncio.Event()).wait(), PIN_TIMEOUT_S) + session = next(s for s, t in self._sessions.items() if t == target) + await self.devtools.call("Page.navigate", {"url": url}, session_id=session) await self._bring_to_front(target) return target @@ -364,8 +375,9 @@ async def hidden_page(self, url: str = "about:blank") -> AsyncIterator[str]: await self.devtools.call("Target.closeTarget", {"targetId": target}) async def active_tab(self) -> Optional[str]: - """The tab a dialog is open on, whose page it blocks from saying; else the tab whose page has focus, else the - one agentenv last brought to the front, else the first visible one: CDP has no event for a tab made active.""" + """The tab a dialog is open on, whose page it blocks from saying; else the visible tab whose page has focus, as + a hidden tab's page that had input still says it has; else the one agentenv last brought to the front, else the + first visible one: CDP has no event for a tab made active.""" if dialogs := [dialog["tab"] for dialog in self._open_dialogs.values() if dialog["tab"] in self.tabs]: return dialogs[-1] visible = None @@ -379,7 +391,7 @@ async def active_tab(self) -> Optional[str]: timeout_s=ACTIVE_CHECK_TIMEOUT_S))["result"]["value"] except (ConnectionError, DevToolsError, asyncio.TimeoutError, KeyError): continue - if state[1]: + if state[0] == "visible" and state[1]: return target if state[0] == "visible" and visible is None: visible = target @@ -502,6 +514,7 @@ def _on_Target_attachedToTarget(self, params: dict, session: Optional[str]) -> N async def _let_run(self, session: str) -> None: """A page it attached to, enabled and, headless, held to its window's size, then told to run: a page attached as it opens, a popup say, or any new page headless, waits for that before it runs.""" + target = self._sessions.get(session) try: await self.devtools.call("Page.enable", session_id=session) if self.viewport is not None: @@ -510,11 +523,16 @@ async def _let_run(self, session: str) -> None: "width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, session_id=session) await self.devtools.call("Page.setInterceptFileChooserDialog", {"enabled": True}, session_id=session) finally: - await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) + try: + await self.devtools.call("Runtime.runIfWaitingForDebugger", session_id=session) + finally: + if target is not None: + self._pinned.setdefault(target, asyncio.Event()).set() def _on_Target_detachedFromTarget(self, params: dict, session: Optional[str]) -> None: if (target := self._sessions.pop(params["sessionId"], None)) is not None: self._shots.pop(target, None) + self._pinned.pop(target, None) self._choosers.pop(params["sessionId"], None) self._dialog_opened.pop(params["sessionId"], None) diff --git a/packages/agentenv-protocol/tests/probe_fake.py b/packages/agentenv-protocol/tests/probe_fake.py index a6a6e85a3..e780a593a 100644 --- a/packages/agentenv-protocol/tests/probe_fake.py +++ b/packages/agentenv-protocol/tests/probe_fake.py @@ -19,16 +19,28 @@ from agentenv_protocol.computer import ( ComputerDriverEnvironment, ExecResult, + FileChooser, + InvalidArgument, + PageDialog, + PageState, + Tab, click, + dialog, drag, + history, + navigate, + page_state, pointer_up, press, run_in_session, scroll, + tabs, type_text, + upload, ) from agentenv_protocol.computer import _linux as linux from agentenv_protocol.computer.conformance.checks import STAGE +from agentenv_protocol.computer.conformance.page_fixture import QUESTION, TITLE, TWO, TWO_TITLE CLIPBOARD = "agentenv-probe clipboard" FIELD = (1080, 500) @@ -39,11 +51,16 @@ "pad": [840, 160, 400, 300], "wheel": [840, 500, 200, 120], "field": [*FIELD, 48, 48], "list_row_height": 24, "markers": [[FIELD[0] + x, FIELD[1] + y, c] for x, y, c in MARKERS], "cell": 50, "clipboard": CLIPBOARD} KEYSYMS = {" ": "space", ",": "comma", "!": "exclam", "\t": "Tab", "\n": "Return"} +PAGE_LAYOUT = {**LAYOUT, "ask": [40, 600, 120, 44], "file": [40, 670, 300, 34]} +DESK_URL = "http://127.0.0.1:4343/" +TITLES = {DESK_URL: TITLE, DESK_URL + TWO.lstrip("/"): TWO_TITLE} class FakeDesk: """What the Fixture Desk would record of the input the fake screen gets.""" + layout = LAYOUT + def __init__(self, driver: FakeDriver) -> None: self.driver = driver self.started = False @@ -69,7 +86,7 @@ def read(self) -> dict: return self.state def _in(self, box: str, x: float, y: float) -> tuple[float, float] | None: - x0, y0, w, h = LAYOUT[box] + x0, y0, w, h = self.layout[box] return (x - x0, y - y0) if x0 <= x < x0 + w and y0 <= y < y0 + h else None def click(self, x: float, y: float, button: str, count: int) -> None: @@ -192,6 +209,124 @@ def _fixture(self, argv: list[str]) -> ExecResult | None: return None +class FakePageDesk(FakeDesk): + """What the HTML desk would record: the Tk desk's record, with a prompt behind its Ask button, a file chooser behind + its file input, and the fields only a page has.""" + + layout = PAGE_LAYOUT + + def __init__(self, driver: PageFixtureDriver) -> None: + super().__init__(driver) + self.state.update(pasted=[], prompts=0, prompted=None, uploaded=None, loads=0) + + def click(self, x: float, y: float, button: str, count: int) -> None: + super().click(x, y, button, count) + if not self.started: + return + if self._in("ask", x, y) is not None: + if self.driver.answers_dialogs: + self.state.update(prompts=self.state["prompts"] + 1, prompted="") + else: + self.driver.open_dialog = PageDialog("prompt", QUESTION) + elif self._in("file", x, y) is not None: + self.driver.chooser = FileChooser() + + +class PageFixtureDriver(FixtureDriver): + """The fixture driver as a page driver: its screen a browser's active tab, its page tools over fake tabs and their + history, and the HTML desk's server among its commands.""" + + screen_kind = "page" + + def __init__(self, answers_dialogs: bool = False, **kwargs) -> None: + super().__init__(**kwargs) + self.desk = FakePageDesk(self) + self.answers_dialogs = answers_dialogs + self.open_dialog: PageDialog | None = None + self.chooser: FileChooser | None = None + self.opened = 0 + self.tabs = [{"id": "start", "urls": ["about:blank"], "at": 0, "active": True}] + + def _active(self) -> dict: + return next(tab for tab in self.tabs if tab["active"]) + + def _visit(self, url: str) -> None: + if url == DESK_URL: + self.desk.state["loads"] += 1 + + def _listed(self) -> list[Tab]: + return [Tab(t["id"], t["urls"][t["at"]], TITLES.get(t["urls"][t["at"]], ""), t["active"]) for t in self.tabs] + + @navigate + async def go_to(self, url): + tab = self._active() + tab["urls"], tab["at"] = tab["urls"][:tab["at"] + 1] + [url], tab["at"] + 1 + self._visit(url) + + @history + async def move_in_history(self, direction): + tab = self._active() + tab["at"] = max(0, min(len(tab["urls"]) - 1, tab["at"] + {"back": -1, "forward": 1}.get(direction, 0))) + self._visit(tab["urls"][tab["at"]]) + + @tabs + async def tab_action(self, action, tab, url): + known = next((t for t in self.tabs if t["id"] == tab), None) + if action in ("switch", "close") and known is None: + raise InvalidArgument(f"no tab {tab}") + if action == "open": + self.opened += 1 + for t in self.tabs: + t["active"] = False + self.tabs.append({"id": f"t{self.opened}", "urls": [url or "about:blank"], "at": 0, "active": True}) + self._visit(url) + elif action == "switch": + for t in self.tabs: + t["active"] = t is known + elif action == "close": + self.tabs.remove(known) + if known["active"]: + self.tabs[-1]["active"] = True + return self._listed() + + @dialog + async def answer_dialog(self, accept, text): + if self.open_dialog is None: + raise InvalidArgument("no dialog is open") + self.open_dialog = None + self.desk.state.update(prompts=self.desk.state["prompts"] + 1, prompted=(text or "") if accept else None) + + @upload + async def choose_files(self, paths): + uploads = (Path.home() / "Uploads").resolve() + files = [(uploads / path).resolve() for path in paths] + if any(uploads not in file.parents for file in files): + raise InvalidArgument("a file outside ~/Uploads") + self.chooser = None + self.desk.state["uploaded"] = {"name": files[0].name, "content": files[0].read_text()} + + @page_state + async def page_now(self): + tab = self._active() + url = tab["urls"][tab["at"]] + return PageState(tab["id"], url, TITLES.get(url, ""), self.open_dialog, self.chooser) + + def _fixture(self, argv: list[str]) -> ExecResult | None: + text = " ".join(argv) + if f"python3 {STAGE}/page_fixture.py" in text: + self.desk.start() + return ExecResult(0, b"4343\n") + if argv == ["cat", f"{STAGE}/state/port"]: + return ExecResult(0, b"4343") if self.desk.started else ExecResult(1) + if argv[:1] == ["cat"] and argv[1].endswith("layout.json") and self.desk.started: + return ExecResult(0, json.dumps(PAGE_LAYOUT).encode()) + if argv == ["stat", "-c", "%u", f"{STAGE}/page_fixture.py"]: + return ExecResult(0, b"1000\n") + if argv == ["stat", "-c", "%u", f"{STAGE}/fixture.py"]: + return ExecResult(1, b"", b"stat: cannot statx: No such file or directory") + return super()._fixture(argv) + + @contextlib.contextmanager def served(driver: ComputerDriverEnvironment) -> Iterator[str]: """``driver`` on loopback in a thread of its own: the URL its card is served at.""" diff --git a/packages/agentenv-protocol/tests/test_computer_conformance_page.py b/packages/agentenv-protocol/tests/test_computer_conformance_page.py new file mode 100644 index 000000000..e78217c01 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_conformance_page.py @@ -0,0 +1,98 @@ +"""The probe on a driver whose screen is a page: it stages the HTML desk in a tab of its own, verifies the input checks +and the page tools on it and leaves the tab it was given as it was, closing the tabs it opened even when a check +fails; without the page tools the input checks are unverified; a browser that answers dialogs itself makes the dialog +check not applicable; typing that pastes fails.""" +from __future__ import annotations + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +pytest.importorskip("PIL") + +from probe_fake import FixtureDriver, PageFixtureDriver, served # noqa: E402 + +from agentenv_protocol.computer import COMPUTER_PAGE, InvalidArgument, tabs, type_text # noqa: E402 +from agentenv_protocol.computer.conformance import ProbeReport, run_probe # noqa: E402 +from agentenv_protocol.computer.conformance import checks # noqa: E402 + + +@pytest.fixture(autouse=True) +def home(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(checks, "RECORD_S", 0.8) + return tmp_path + + +async def _probe(driver) -> ProbeReport: + with served(driver) as url: + return await run_probe(url, "full") + + +def _status(report: ProbeReport, start: str) -> tuple[str, str]: + return next((c.status, c.detail) for c in report.checks if c.name.startswith(start)) + + +@pytest.mark.asyncio +async def test_a_page_driver_passes_every_check_on_the_html_desk_and_the_probe_closes_its_own_tab(home): + driver = PageFixtureDriver() + report = await _probe(driver) + + failed = [f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed"] + page = [c for c in report.checks if COMPUTER_PAGE in c.extensions] + assert report.passed, failed + assert report.counts()["unverified"] == 0 and report.extensions[COMPUTER_PAGE] == "verified" + assert len(page) == 6 and {c.status for c in page} == {"passed"} + assert [(t["id"], t["urls"]) for t in driver.tabs] == [("start", ["about:blank"])] + assert not (home / ".agentenv-probe").exists() and not list((home / "Uploads").iterdir()) + assert ["kill", "4343"] in driver.commands and driver.desk.state["prompted"] == checks.WORD + + +@pytest.mark.asyncio +async def test_a_tab_the_probe_opened_is_closed_though_the_check_that_opened_it_failed(): + class NoSwitch(PageFixtureDriver): + @tabs + async def tab_action(self, action, tab, url): + if action == "switch": + raise InvalidArgument("this browser can't switch tabs") + return await super().tab_action(action, tab, url) + + driver = NoSwitch() + report = await _probe(driver) + + assert _status(report, "computer_tabs")[0] == "failed" + assert [t["id"] for t in driver.tabs] == ["start"] + + +@pytest.mark.asyncio +async def test_a_page_driver_without_the_page_tools_leaves_the_input_checks_unverified_and_says_why(): + class NoPageTools(FixtureDriver): + screen_kind = "page" + + report = await _probe(NoPageTools()) + + unverified = {c.detail for c in report.checks if c.status == "unverified" and c.check != "P8"} + assert report.passed and len(unverified) == 1 + assert "opened with computer-page/v1; the driver lacks ['urn:agentenv:computer-page/v1']" in unverified.pop() + + +@pytest.mark.asyncio +async def test_a_browser_that_answers_dialogs_itself_makes_the_dialog_check_not_applicable(): + report = await _probe(PageFixtureDriver(answers_dialogs=True)) + + assert report.passed + assert _status(report, "computer_dialog") == ("not_applicable", + "the browser answers dialogs itself: the prompt returned ''") + + +@pytest.mark.asyncio +async def test_typing_that_arrives_as_a_paste_fails_the_clipboard_check_on_a_page(): + class Pastes(PageFixtureDriver): + @type_text + async def type_in(self, text): + await super().type_in(text) + self.desk.state["pasted"].append(text) + + report = await _probe(Pastes()) + + status, detail = _status(report, "typing leaves the clipboard as it was") + assert status == "failed" and detail.startswith("typing arrived as a paste of ['Hello, Wörld! 東京 AbC'") diff --git a/packages/agentenv-protocol/tests/test_website_browser.py b/packages/agentenv-protocol/tests/test_website_browser.py index 97f1905fa..6ae0861fd 100644 --- a/packages/agentenv-protocol/tests/test_website_browser.py +++ b/packages/agentenv-protocol/tests/test_website_browser.py @@ -636,14 +636,15 @@ async def test_it_navigates_the_active_tab_and_moves_in_its_history_or_says_why_ @pytest.mark.asyncio -async def test_a_tab_it_opens_or_activates_is_brought_to_the_front_and_the_active_one_and_one_it_closes_its_own(): +async def test_a_tab_it_opens_loads_once_held_to_the_window_comes_to_the_front_and_one_it_closes_is_its_own(): devtools = Tabbed() control = await _tabs(devtools, "a") - opened = await control.open_tab("https://mail.test/b") + opening = asyncio.ensure_future(control.open_tab("https://mail.test/b")) + await asyncio.sleep(0) devtools.send("Target.targetCreated", _page("b", "https://mail.test/b", "Mail b")) devtools.send("Target.attachedToTarget", {"sessionId": "s-b", **_page("b")}) - await asyncio.sleep(0) + opened = await opening after_open = await control.tab_list() await control.activate("a") after_switch = await control.tab_list() @@ -652,13 +653,25 @@ async def test_a_tab_it_opens_or_activates_is_brought_to_the_front_and_the_activ await control.close("b") devtools.send("Target.targetDestroyed", {"targetId": "b"}) - assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "https://mail.test/b"}, None)] + on_b = [name for name, _, session in devtools.calls if session == "s-b"] + assert opened == "b" and _sent(devtools, "Target.createTarget") == [({"url": "about:blank"}, None)] + assert ({"url": "https://mail.test/b"}, "s-b") in _sent(devtools, "Page.navigate") + assert on_b.index("Emulation.setDeviceMetricsOverride") < on_b.index("Page.navigate") assert [(tab["id"], tab["active"]) for tab in after_open] == [("a", False), ("b", True)] assert [(tab["id"], tab["active"]) for tab in after_switch] == [("a", True), ("b", False)] assert ({}, "s-a") in _sent(devtools, "Page.bringToFront") assert [(tab["id"], tab["closed_by"]) for tab in control.closed_tabs] == [("b", "agentenv")] +@pytest.mark.asyncio +async def test_a_hidden_tab_whose_page_still_says_it_has_focus_isnt_the_active_one(): + states = {"s-a": ["hidden", True], "s-b": ["visible", True]} + devtools = Tabbed({"Runtime.evaluate": lambda params, session: {"result": {"value": states[session]}}}) + control = await _tabs(devtools, "a", "b") + + assert await control.active_tab() == "b" + + @pytest.mark.asyncio async def test_a_dialog_makes_its_tab_the_active_one_its_state_names_it_and_the_agent_answers_it(): devtools = Tabbed() diff --git a/src/agent_env/drivers/envs/page/driver.py b/src/agent_env/drivers/envs/page/driver.py index a35efd442..8503ab774 100644 --- a/src/agent_env/drivers/envs/page/driver.py +++ b/src/agent_env/drivers/envs/page/driver.py @@ -156,12 +156,13 @@ async def drag_along(self, path, button, duration_ms, modifiers) -> None: @scroll async def scroll_at(self, x, y, direction, amount, modifiers) -> None: - dx = {"left": -1, "right": 1}.get(direction, 0) * amount * WHEEL_PX - dy = {"up": -1, "down": 1}.get(direction, 0) * amount * WHEEL_PX + """One wheel event a notch, as a mouse wheel sends them.""" + dx = {"left": -1, "right": 1}.get(direction, 0) * WHEEL_PX + dy = {"up": -1, "down": 1}.get(direction, 0) * WHEEL_PX self._pointer = (x, y) await self._input([{"method": "Input.dispatchMouseEvent", "params": { "type": "mouseWheel", "x": x, "y": y, "deltaX": dx, "deltaY": dy, - "modifiers": _modifiers(modifiers) | self._mask()}}]) + "modifiers": _modifiers(modifiers) | self._mask()}}] * amount) @pointer_move async def move(self, x, y) -> None: diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index c80e31a07..6d20b9689 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -1,10 +1,10 @@ """The browser as a computer: agentenv's chromium deployed alone, on the ubuntu-24.04-headless base with no display -and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it; -deployed through the computer provider like any computer, its computer_* tools see and drive the page, and its page -tools navigate, move in history, answer a dialog, use tabs and upload; nothing runs X and nothing listens but the driver -and the site proxy; a killed app server or driver is started again by supervisord and the computer is ready again; and -chromium killed fails it for good, computer-ready/v1 saying why and the tools answering with it. Needs Docker and the -local registry on localhost:5000.""" +and the page driver, whose screen is chromium's active tab, headless. Gate 3 makes this machine's arch ready with it, +its input checks and page tools verified on the HTML desk; deployed through the computer provider like any computer, +its computer_* tools see and drive the page, and its page tools navigate, move in history, answer a dialog, use tabs +and upload; nothing runs X and nothing listens but the driver and the site proxy; a killed app server or driver is +started again by supervisord and the computer is ready again; and chromium killed fails it for good, computer-ready/v1 +saying why and the tools answering with it. Needs Docker and the local registry on localhost:5000.""" import asyncio import json @@ -20,7 +20,7 @@ from PIL import Image from agentenv_protocol import client as protocol_v1 -from agentenv_protocol.computer import COMPUTER_READY +from agentenv_protocol.computer import COMPUTER_PAGE, COMPUTER_READY from agentenv_protocol.computer.conformance.client import image from agentenv_protocol.website_browser import BROWSER @@ -94,7 +94,7 @@ def log_message(self, *args): """ -def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver(browser_computer): +def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver_its_input_and_page_tools_verified(browser_computer): results = browser_computer.metadata["composed"]["container"] probe = results[ARCH]["probe"] @@ -104,6 +104,7 @@ def test_gate_3_makes_this_machines_arch_ready_with_the_page_driver(browser_comp "not passed": [(c["check"], c["name"], c["status"], c["detail"][:160]) for c in report["checks"] if c["status"] != "passed"]}, indent=1)) assert results[ARCH]["status"] == "ready" and probe["passed"], results[ARCH] + assert probe["checks"]["unverified"] == 0 and probe["extensions"][COMPUTER_PAGE] == "verified", probe assert [app.environment_name for app in browser_computer.composed_apps()] == ["chromium"] diff --git a/tst/unit/drivers/page_driver_test.py b/tst/unit/drivers/page_driver_test.py index 5eef37282..f6ee8f1e4 100644 --- a/tst/unit/drivers/page_driver_test.py +++ b/tst/unit/drivers/page_driver_test.py @@ -110,6 +110,15 @@ async def test_with_shift_held_typed_text_is_shifted_and_with_control_held_a_cli assert events[-1]["params"]["type"] == "mouseWheel" and events[-1]["params"]["modifiers"] == 3 +@pytest.mark.asyncio +async def test_a_scroll_is_one_wheel_event_a_notch_sent_together(page): + await page.scroll_at(10, 20, "left", 3, []) + + assert [op for op, _ in page.sent] == ["page/input"] + assert [(e["params"]["type"], e["params"]["deltaX"], e["params"]["deltaY"]) for e in _events(page)] == [ + ("mouseWheel", -100, 0)] * 3 + + @pytest.mark.asyncio async def test_typed_text_is_keystrokes_with_shift_where_it_takes_one_and_inserted_text_for_the_rest(page): await page.type_in("Hi!\né")