Add Evervault proxy arms and retail-cart-geo (per-country proxies)

.github/workflows/_run.yml

@@ -71,6 +71,8 @@ jobs:
       STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }}
       HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }}
       ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }}
+      EV_APP_UUID: ${{ secrets.EV_APP_UUID }}
+      EV_API_KEY: ${{ secrets.EV_API_KEY }}
       BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
       BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }}
     steps:

README.md

@@ -92,6 +92,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary:
 | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. |
 | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. |
 | `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `STEEL_API_KEY`, `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"`); checked before a run that includes them |
+| `EV_APP_UUID`, `EV_API_KEY` | `codex-pw-kernel-evervault`, which mints an Evervault Browser Tokens session per trial for its proxy. The key needs `browserToken:create`. |
 | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results |
 
 ## Running in GitHub Actions
@@ -147,6 +148,7 @@ The `retail-cart` tasks on each browser provider's browser. Every arm uses the s
 | Arm | Browser |
 |---|---|
 | `codex-pw-kernel` | Kernel, `stealth: true` |
+| `codex-pw-kernel-evervault` | Kernel, `stealth: true`, with the whole session sent through Evervault's browser proxy: a new Evervault session and Kernel custom proxy with Evervault's CA per trial |
 | `codex-pw-browserbase` | Browserbase, Advanced Stealth (Scale plan), CAPTCHA solving, its residential proxies |
 | `codex-pw-steel` | Steel, residential proxy and CAPTCHA solving |
 | `codex-pw-hyperbrowser` | Hyperbrowser, stealth, proxy and CAPTCHA solving |
@@ -160,6 +162,19 @@ uv run bench run retail-cart-providers --attempts 10 --concurrency 4
 
 All arms' task containers run on one machine, so keep `--concurrency` low on a GitHub runner.
 
+### `retail-cart-geo`
+
+The `retail-cart` tasks reached from each site's own market, for sites outside the US. Tasks need a `country`, so pass a task file of the same shape as `suites/retail-cart/tasks.toml` with `--suite-arg tasks=<path>`.
+
+| Arm | Browser |
+|---|---|
+| `codex-pw-kernel-geo` | Kernel, `stealth: true`, with a new residential proxy in the task's country per trial (the `geo` configurator) |
+| `codex-pw-kernel-evervault-eu` | Kernel, `stealth: true`, with the whole session sent through Evervault's EU (Ireland) relay |
+
+```bash
+uv run bench run retail-cart-geo --suite-arg tasks=path/to/tasks.toml --attempts 5
+```
+
 ## Extending
 
 ### Add an arm

arms/codex-pw-kernel-evervault-eu.toml

@@ -0,0 +1,23 @@
+# Same as codex-pw-kernel-evervault, but through Evervault's EU (Ireland)
+# relay, for sites that serve European markets.
+agent = "codex"
+model = "gpt-6-luna"
+agent_version = "0.163.0"
+toolset = "playwright-mcp"
+environment = "remote-browser"
+
+[agent_kwargs]
+disable_web_search = true
+
+[browser.config]
+provider = "kernel-evervault"
+
+[browser.config.params]
+stealth = true
+timeout_seconds = 1800
+
+# See codex-pw-kernel-evervault for the TTL.
+[browser.config.params.evervault]
+ttl = 900
+hosts = ["checkout-test.adyen.com"]
+relay_host = "eu-ie.browser.relay.evervault.app"

arms/codex-pw-kernel-evervault.toml

@@ -0,0 +1,26 @@
+# Same as codex-pw-kernel, but the whole browser session goes through
+# Evervault's browser proxy, so sites see Evervault's egress instead of
+# Kernel's stealth proxy pool.
+agent = "codex"
+model = "gpt-6-luna"
+agent_version = "0.163.0"
+toolset = "playwright-mcp"
+environment = "remote-browser"
+
+[agent_kwargs]
+disable_web_search = true
+
+[browser.config]
+provider = "kernel-evervault"
+
+[browser.config.params]
+stealth = true
+timeout_seconds = 1800
+
+# The proxy closes its tunnels when the mint's TTL ends. Sandbox apps cap the
+# TTL at 900 seconds, the same as the agent timeout, so a trial that uses all
+# of it can lose the proxy before the verifier runs. A production app allows
+# up to 4 hours. No card is minted, so the host only has to be valid.
+[browser.config.params.evervault]
+ttl = 900
+hosts = ["checkout-test.adyen.com"]

arms/codex-pw-kernel-geo.toml

@@ -0,0 +1,18 @@
+# Same as codex-pw-kernel, but each trial gets a new residential proxy in the
+# country of the task's site (the task's `country`).
+agent = "codex"
+model = "gpt-6-luna"
+agent_version = "0.163.0"
+toolset = "playwright-mcp"
+environment = "remote-browser"
+
+[agent_kwargs]
+disable_web_search = true
+
+[browser]
+configurator = "geo"
+configurator_args = { proxy = "residential" }
+
+[browser.config]
+provider = "kernel"
+params = { stealth = true, timeout_seconds = 1800 }

benchmarks/retail-cart-geo.toml

@@ -0,0 +1,6 @@
+# The retail-cart tasks reached from each site's own market: a Kernel browser
+# with a residential proxy in the site's country, and a Kernel browser sent
+# through Evervault's EU relay. Tasks need a `country`; pass a task file with
+# `--suite-arg tasks=<path>`.
+suite = "retail-cart"
+arms = ["codex-pw-kernel-geo", "codex-pw-kernel-evervault-eu"]

benchmarks/retail-cart-providers.toml

@@ -3,6 +3,7 @@
 suite = "retail-cart"
 arms = [
   "codex-pw-kernel",
+  "codex-pw-kernel-evervault",
   "codex-pw-browserbase",
   "codex-pw-steel",
   "codex-pw-hyperbrowser",

configurators/geo/configure

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Give each trial a proxy in the country of the task's site.
+
+The country comes from `metadata.country` in the task's task.toml, so a site
+is reached from the market it serves. The proxy type is the `proxy` arg
+(default `residential`). The proxy is created per trial and deleted after it.
+
+stdin:  {"task": {"config": str, "instruction": str}, "args": {"proxy": str}}
+stdout: {"browser": {...}, "steering": null, "provenance": {...}}
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import tomllib
+
+
+def main() -> int:
+    payload = json.load(sys.stdin)
+    metadata = tomllib.loads(payload["task"]["config"]).get("metadata", {})
+    country = metadata.get("country")
+    if not country:
+        raise SystemExit(f"Task for {metadata.get('site')!r} has no metadata.country")
+    proxy = (payload.get("args") or {}).get("proxy", "residential")
+    json.dump(
+        {
+            "browser": {"proxy_create": {"type": proxy, "config": {"country": country}}},
+            "steering": None,
+            "provenance": {"site": metadata.get("site"), "proxy": proxy, "country": country},
+        },
+        sys.stdout,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

harness/environment.py

@@ -24,7 +24,9 @@
 `RemoteBrowserEnvironment` instead runs the task container on local Docker
 and drives a browser from any provider in `harness.providers` over CDP, so
 providers are compared with the same agent, tools and container. Its config
-file names the `provider` and that provider's session create `params`.
+file names the `provider` and that provider's session create `params`. A
+`proxy_create` entry, as a configurator writes it, is passed on in `params`;
+only the Kernel providers accept it.
 """
 
 from __future__ import annotations
@@ -232,7 +234,9 @@ def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None:
         config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text())
         self._provider_name: str = config["provider"]
         self._provider = providers.get(self._provider_name)
-        self._create_params: dict[str, Any] = config.get("params") or {}
+        self._create_params: dict[str, Any] = dict(config.get("params") or {})
+        if PROXY_CREATE in config:
+            self._create_params[PROXY_CREATE] = config[PROXY_CREATE]
         self._browser_session_id: str | None = None
         super().__init__(environment_dir, *args, **kwargs)
 

harness/providers.py

@@ -10,6 +10,7 @@
 from __future__ import annotations
 
 import os
+import uuid
 from dataclasses import dataclass
 from typing import Any
 
@@ -34,16 +35,109 @@ async def release(self, session_id: str) -> None:
 
 
 class KernelProvider(Provider):
+    """A Kernel browser. A `proxy_create` param, a POST /proxies body, creates a
+    proxy for the session and deletes it after the browser, as in
+    harness.environment."""
+
     env = ("KERNEL_API_KEY",)
 
-    async def create(self, params: dict[str, Any]) -> Session:
-        async with AsyncKernel() as client:
+    def __init__(self) -> None:
+        self._proxies: dict[str, str] = {}
+
+    async def _create_browser(
+        self, client: AsyncKernel, params: dict[str, Any], proxy: dict[str, Any] | None
+    ) -> Session:
+        from harness.environment import RUN_ID_ENV, proxy_prefix
+
+        if proxy is None:
             browser = await client.browsers.create(**params)
+            return Session(browser.session_id, browser.cdp_ws_url)
+        run_id = os.environ.get(RUN_ID_ENV) or "local"
+        name = f"{proxy_prefix(run_id)}{uuid.uuid4().hex[:12]}"
+        kernel_proxy = await client.proxies.create(**proxy, name=name)
+        try:
+            browser = await client.browsers.create(**params, proxy_id=kernel_proxy.id)
+        except Exception:
+            await client.proxies.delete(kernel_proxy.id)
+            raise
+        self._proxies[browser.session_id] = kernel_proxy.id
         return Session(browser.session_id, browser.cdp_ws_url)
 
+    async def create(self, params: dict[str, Any]) -> Session:
+        params = dict(params)
+        proxy = params.pop("proxy_create", None)
+        async with AsyncKernel() as client:
+            return await self._create_browser(client, params, proxy)
+
     async def release(self, session_id: str) -> None:
+        # The browser goes first: without its proxy it would reach sites directly.
         async with AsyncKernel() as client:
             await client.browsers.delete_by_id(session_id)
+            proxy_id = self._proxies.pop(session_id, None)
+            if proxy_id is not None:
+                await client.proxies.delete(proxy_id)
+
+
+class KernelEvervaultProvider(KernelProvider):
+    """A Kernel browser whose whole session goes through Evervault's browser proxy.
+
+    Each session mints its own Evervault Browser Tokens session, which carries
+    the proxy credentials, and creates a Kernel custom proxy from them with
+    Evervault's CA bundle, since the proxy terminates TLS. The mint holds one
+    placeholder credential: the session is only used for its proxy, and no
+    real value is ever revealed. `params` holds the Kernel browser create
+    params, plus an `evervault` table with the mint's `ttl`, the `hosts` it
+    may reveal to, optionally `relay_host` to use another region's relay than
+    the one the mint returns (the credentials work on any), and optionally
+    `proxy_settings` for Evervault's egress.
+    """
+
+    env = ("KERNEL_API_KEY", "EV_APP_UUID", "EV_API_KEY")
+    mint_url = "https://api.evervault.com/browser-tokens"
+    ca_url = "https://ca.evervault.com"
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._ca_bundle: str | None = None
+
+    async def _mint(self, settings: dict[str, Any]) -> dict[str, Any]:
+        body: dict[str, Any] = {
+            "ttl": settings.get("ttl", 900),
+            "data": [{"id": "bench", "category": "credential.password", "value": "bench"}],
+            "permissions": [{"action": "decrypt", "target": host} for host in settings["hosts"]],
+        }
+        if "proxy_settings" in settings:
+            body["proxySettings"] = settings["proxy_settings"]
+        async with httpx.AsyncClient(timeout=30) as client:
+            if self._ca_bundle is None:
+                response = await client.get(self.ca_url)
+                response.raise_for_status()
+                self._ca_bundle = response.text
+            response = await client.post(
+                self.mint_url,
+                json=body,
+                auth=(os.environ["EV_APP_UUID"], os.environ["EV_API_KEY"]),
+            )
+        # The response body carries the proxy password, so it is never logged.
+        if response.is_error:
+            raise RuntimeError(f"Evervault mint failed with HTTP {response.status_code}")
+        return response.json()["proxyConfiguration"]
+
+    async def create(self, params: dict[str, Any]) -> Session:
+        browser_params = dict(params)
+        settings = browser_params.pop("evervault")
+        proxy = await self._mint(settings)
+        config = {
+            "host": settings.get("relay_host") or proxy["hostname"],
+            "port": proxy["port"],
+            "username": proxy["username"],
+            "password": proxy["password"],
+            "ca_bundle": self._ca_bundle,
+        }
+        async with AsyncKernel() as client:
+            return await self._create_browser(
+                client, browser_params, {"type": "custom", "protocol": "https", "config": config}
+            )
 
 
 class HttpProvider(Provider):
@@ -131,6 +225,7 @@ async def release(self, session_id: str) -> None:
 
 PROVIDERS: dict[str, Provider] = {
     "kernel": KernelProvider(),
+    "kernel-evervault": KernelEvervaultProvider(),
     "browserbase": BrowserbaseProvider(),
     "steel": SteelProvider(),
     "hyperbrowser": HyperbrowserProvider(),

suites/retail-cart/suite.py

@@ -13,6 +13,10 @@
 itself (`cart_url`, or `cart_api` fetched from the page) and records
 `cart_verified` when it lists the item (`cart_match`, a case-insensitive
 regular expression).
+
+Tasks come from tasks.toml, or from another file of the same shape given as
+the `tasks` suite arg. A task's optional `country` is recorded in its
+task.toml metadata, where the `geo` configurator reads it.
 """
 
 from __future__ import annotations
@@ -52,19 +56,20 @@ class Task:
     cart_match: str
     cart_url: str | None = None
     cart_api: str | None = None
+    country: str | None = None
     timeout_sec: int = 900
 
 
 def preflight() -> None:
     """Nothing to check: tasks need only the Kernel browser Harbor creates."""
 
 
-def load_tasks() -> list[Task]:
-    return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]]
+def load_tasks(path: Path = HERE / "tasks.toml") -> list[Task]:
+    return [Task(**entry) for entry in tomllib.loads(path.read_text())["tasks"]]
 
 
-def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]:
-    tasks = load_tasks()
+def generate(task_ids: list[str] | None, out_dir: Path, tasks: str | None = None) -> list[Path]:
+    tasks = load_tasks(Path(tasks)) if tasks else load_tasks()
     if task_ids:
         known = {task.id: task for task in tasks}
         missing = [task_id for task_id in task_ids if task_id not in known]
@@ -108,6 +113,8 @@ def _write(task: Task, task_dir: Path) -> Path:
     config["source"] = "retail-cart"
     config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]}
     config["metadata"] = {"site": task.site, "start_url": task.start_url}
+    if task.country:
+        config["metadata"]["country"] = task.country
     config["environment"] = {
         "build_timeout_sec": 1200.0,
         "network_mode": "public",

tests/test_geo.py

@@ -0,0 +1,30 @@
+import json
+import subprocess
+
+from harness.definitions import ROOT
+
+CONFIGURE = ROOT / "configurators/geo/configure"
+
+
+def _run(config, args=None):
+    payload = {"task": {"config": config, "instruction": ""}, "args": args or {}}
+    return subprocess.run(
+        [CONFIGURE], input=json.dumps(payload), capture_output=True, text=True, check=False
+    )
+
+
+def test_the_proxy_is_in_the_task_country():
+    result = _run('[metadata]\nsite = "shop.example"\ncountry = "PL"\n', {"proxy": "isp"})
+    output = json.loads(result.stdout)
+    assert output["browser"] == {"proxy_create": {"type": "isp", "config": {"country": "PL"}}}
+
+
+def test_a_task_without_a_country_is_refused():
+    result = _run('[metadata]\nsite = "shop.example"\n')
+    assert result.returncode != 0
+    assert "country" in result.stderr
+
+
+def test_the_default_proxy_is_residential():
+    output = json.loads(_run('[metadata]\ncountry = "GB"\n').stdout)
+    assert output["browser"]["proxy_create"]["type"] == "residential"