From 07186de5a7a37715a94859280c2884f9094af818 Mon Sep 17 00:00:00 2001 From: "druks-operator[bot]" <284423593+druks-operator[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 06:17:05 +0000 Subject: [PATCH 1/2] Name exe transport error types and raise create_vm read-timeout floor Wrap httpx.RequestError with {exc!r} so bare timeouts name their type instead of stringifying blank, and apply a per-request 90s read-timeout floor to create_vm without mutating the cached client or shortening a higher configured timeout. Co-Authored-By: Claude Opus 4.8 --- docs/deploy.md | 2 +- src/providers/exe/api.py | 49 +++++++++++++++++++--- src/providers/exe/tests/test_api.py | 65 +++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index f6b0bba..d433990 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -190,7 +190,7 @@ exe.dev provider: | `EXE_API_TOKEN` | — (required) | Bearer token for the exe.dev exec API. | | `EXE_DEFAULT_IMAGE` | — (required) | Image used when the caller omits `image`. | | `EXE_API_URL` | `https://exe.dev` | API base URL. | -| `EXE_API_TIMEOUT` | `30.0` | Timeout for exe.dev API calls. | +| `EXE_API_TIMEOUT` | `30.0` | General timeout for exe.dev API calls. VM creation is legitimately slow, so it applies an internal per-request read-timeout floor (currently 90s) when this general timeout is lower; a configured value above the floor is used as-is. Connect, write, and pool budgets always follow this setting. No separate creation setting exists. | | `EXE_BOOTSTRAP_SSH_TIMEOUT_SECONDS` | `30.0` | ssh-keyscan retry budget for a fresh exe.dev sandbox. | | `EXE_SSH_USERNAME` | `exedev` | In-VM user callers SSH as. | diff --git a/src/providers/exe/api.py b/src/providers/exe/api.py index 65eb001..5350e5c 100644 --- a/src/providers/exe/api.py +++ b/src/providers/exe/api.py @@ -16,6 +16,14 @@ ) from .settings import ExeSettings +# VM creation is legitimately slow: a healthy create_host measured ~21s through +# the full stack (ENG-821 probe, 2026-08-06). The default 30s general budget sits +# too close to that ceiling, so minor exe.dev degradation trips the read timeout. +# Give VM creation a read-timeout floor with real headroom while leaving connect, +# write, and pool budgets — and every other exe.dev command — on the configured +# general timeout. +_CREATE_VM_READ_TIMEOUT_FLOOR = 90.0 + def _encode_setup_script(script: str) -> str: """Encode a multi-line script as a double-quoted exe.dev argument value. @@ -91,7 +99,26 @@ async def create_vm( if env: for key, value in env.items(): command_parts.extend(["--env", shlex.quote(f"{key}={value}")]) - return await self._exec_dict(" ".join(command_parts)) + return await self._exec_dict(" ".join(command_parts), timeout=self._creation_timeout()) + + def _creation_timeout(self) -> httpx.Timeout: + """Read-timeout floor for VM creation, derived from the general timeout. + + Raise only the read budget when it falls below the creation floor; never + shorten a configured read timeout already at or above it, and preserve the + connect, write, and pool budgets. Returns a per-request timeout — the + cached client is never mutated, so ordinary commands keep the general + timeout. + """ + read = self.timeout.read + if read is None or read >= _CREATE_VM_READ_TIMEOUT_FLOOR: + return self.timeout + return httpx.Timeout( + connect=self.timeout.connect, + read=_CREATE_VM_READ_TIMEOUT_FLOOR, + write=self.timeout.write, + pool=self.timeout.pool, + ) async def list_vms( self, @@ -211,19 +238,29 @@ async def aclose(self) -> None: await self._client.aclose() self._client = None - async def _exec_dict(self, command: str) -> dict[str, Any]: - response = await self._request(command) + async def _exec_dict( + self, command: str, *, timeout: httpx.Timeout | None = None + ) -> dict[str, Any]: + response = await self._request(command, timeout=timeout) payload = self._parse_json_response(response.text) if not isinstance(payload, dict): raise ExeResponseError("exe.dev API returned non-object JSON output") return payload - async def _request(self, command: str) -> httpx.Response: + async def _request( + self, command: str, *, timeout: httpx.Timeout | None = None + ) -> httpx.Response: + client = self._get_client() try: - response = await self._get_client().post("/exec", content=command) + if timeout is None: + response = await client.post("/exec", content=command) + else: + response = await client.post("/exec", content=command, timeout=timeout) except httpx.RequestError as exc: - raise ExeResponseError(f"exe.dev API transport failed: {exc}") from exc + # str(exc) is empty for bare transport errors like httpx.ReadTimeout(); + # {exc!r} always names the exception type so the failure is diagnosable. + raise ExeResponseError(f"exe.dev API transport failed: {exc!r}") from exc if response.status_code == 401: raise ExeAuthError("exe.dev API authentication failed") diff --git a/src/providers/exe/tests/test_api.py b/src/providers/exe/tests/test_api.py index 4148d6b..c521c3f 100644 --- a/src/providers/exe/tests/test_api.py +++ b/src/providers/exe/tests/test_api.py @@ -162,6 +162,71 @@ async def test_request_sends_authorization_and_text_body(respx_mock): assert call.request.content == b"whoami --json" +@pytest.mark.asyncio +@respx.mock(base_url="https://exe.dev") +async def test_request_names_exception_type_for_blank_transport_error(respx_mock): + # str(httpx.ReadTimeout("")) is an empty string; the wrapped ExeResponseError + # must still name the exception type and keep the original chained. + read_timeout = httpx.ReadTimeout("") + respx_mock.post("/exec").mock(side_effect=read_timeout) + + with pytest.raises(ExeResponseError, match="ReadTimeout") as exc_info: + await _api()._request("new --json") + + assert exc_info.value.__cause__ is read_timeout + + +@pytest.mark.asyncio +@respx.mock(base_url="https://exe.dev") +async def test_create_vm_raises_read_timeout_floor_when_general_timeout_below(respx_mock): + route = respx_mock.post("/exec").mock( + return_value=httpx.Response(200, content=b'{"vm_name": "sb-1", "ssh_port": 22}'), + ) + + # Default general timeout is 30s, below the creation floor. + await ExeAPI(base_url="https://exe.dev", token="token").create_vm( + name="sb-1", image="ubuntu:22.04" + ) + + timeout = route.calls.last.request.extensions["timeout"] + assert timeout["read"] == 90.0 + # Connect, write, and pool budgets stay on the configured general timeout. + assert timeout["connect"] == 5.0 + assert timeout["write"] == 30.0 + assert timeout["pool"] == 30.0 + + +@pytest.mark.asyncio +@respx.mock(base_url="https://exe.dev") +async def test_create_vm_keeps_configured_timeout_when_above_floor(respx_mock): + route = respx_mock.post("/exec").mock( + return_value=httpx.Response(200, content=b'{"vm_name": "sb-1", "ssh_port": 22}'), + ) + + # A configured general read timeout above the floor must not be shortened. + await ExeAPI(base_url="https://exe.dev", token="token", timeout=120.0).create_vm( + name="sb-1", image="ubuntu:22.04" + ) + + assert route.calls.last.request.extensions["timeout"]["read"] == 120.0 + + +@pytest.mark.asyncio +@respx.mock(base_url="https://exe.dev") +async def test_ordinary_command_after_create_vm_uses_general_timeout(respx_mock): + route = respx_mock.post("/exec").mock( + return_value=httpx.Response(200, content=b'{"vm_name": "sb-1", "ssh_port": 22}'), + ) + + # Creation must not mutate the cached client: a later ordinary command on the + # same instance still uses the configured general timeout. + api = ExeAPI(base_url="https://exe.dev", token="token") + await api.create_vm(name="sb-1", image="ubuntu:22.04") + await api.whoami() + + assert route.calls.last.request.extensions["timeout"]["read"] == 30.0 + + @pytest.mark.asyncio @respx.mock(base_url="https://exe.dev") async def test_delete_vm_uses_rm_json_command(respx_mock): From 527a3b655853e9b77e14338ac1d01d59ab46109d Mon Sep 17 00:00:00 2001 From: Paulo Date: Wed, 19 Aug 2026 06:57:37 +0200 Subject: [PATCH 2/2] Fold the create_vm timeout floor into construction The read floor is a fixed function of the configured timeout, so build it once in __init__ instead of re-deriving it per call through a one-caller helper guarding fields we set ourselves. _request always passes an explicit timeout, dropping the None branch. Trim the docs cell and comments to the why, and drop the internal ticket reference from a public repo. Co-Authored-By: Claude Fable 5 --- docs/deploy.md | 2 +- src/providers/exe/api.py | 45 ++++++++--------------------- src/providers/exe/tests/test_api.py | 9 ++---- 3 files changed, 15 insertions(+), 41 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index d433990..f6b0bba 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -190,7 +190,7 @@ exe.dev provider: | `EXE_API_TOKEN` | — (required) | Bearer token for the exe.dev exec API. | | `EXE_DEFAULT_IMAGE` | — (required) | Image used when the caller omits `image`. | | `EXE_API_URL` | `https://exe.dev` | API base URL. | -| `EXE_API_TIMEOUT` | `30.0` | General timeout for exe.dev API calls. VM creation is legitimately slow, so it applies an internal per-request read-timeout floor (currently 90s) when this general timeout is lower; a configured value above the floor is used as-is. Connect, write, and pool budgets always follow this setting. No separate creation setting exists. | +| `EXE_API_TIMEOUT` | `30.0` | Timeout for exe.dev API calls. | | `EXE_BOOTSTRAP_SSH_TIMEOUT_SECONDS` | `30.0` | ssh-keyscan retry budget for a fresh exe.dev sandbox. | | `EXE_SSH_USERNAME` | `exedev` | In-VM user callers SSH as. | diff --git a/src/providers/exe/api.py b/src/providers/exe/api.py index 5350e5c..3acba77 100644 --- a/src/providers/exe/api.py +++ b/src/providers/exe/api.py @@ -16,12 +16,10 @@ ) from .settings import ExeSettings -# VM creation is legitimately slow: a healthy create_host measured ~21s through -# the full stack (ENG-821 probe, 2026-08-06). The default 30s general budget sits -# too close to that ceiling, so minor exe.dev degradation trips the read timeout. -# Give VM creation a read-timeout floor with real headroom while leaving connect, -# write, and pool budgets — and every other exe.dev command — on the configured -# general timeout. +# Creating the VM itself is fast, but `new` waits on the setup script, and the +# tailscale join inside it is slow: ~21s healthy, uncomfortably close to the +# default 30s budget. Creation alone gets this read floor; a configured timeout +# above it wins. _CREATE_VM_READ_TIMEOUT_FLOOR = 90.0 @@ -51,6 +49,9 @@ def __init__( self.token = token self.default_image = default_image self.timeout = httpx.Timeout(timeout, connect=connect_timeout) + self.create_vm_timeout = httpx.Timeout( + timeout, connect=connect_timeout, read=max(timeout, _CREATE_VM_READ_TIMEOUT_FLOOR) + ) self._client: httpx.AsyncClient | None = None @classmethod @@ -99,26 +100,7 @@ async def create_vm( if env: for key, value in env.items(): command_parts.extend(["--env", shlex.quote(f"{key}={value}")]) - return await self._exec_dict(" ".join(command_parts), timeout=self._creation_timeout()) - - def _creation_timeout(self) -> httpx.Timeout: - """Read-timeout floor for VM creation, derived from the general timeout. - - Raise only the read budget when it falls below the creation floor; never - shorten a configured read timeout already at or above it, and preserve the - connect, write, and pool budgets. Returns a per-request timeout — the - cached client is never mutated, so ordinary commands keep the general - timeout. - """ - read = self.timeout.read - if read is None or read >= _CREATE_VM_READ_TIMEOUT_FLOOR: - return self.timeout - return httpx.Timeout( - connect=self.timeout.connect, - read=_CREATE_VM_READ_TIMEOUT_FLOOR, - write=self.timeout.write, - pool=self.timeout.pool, - ) + return await self._exec_dict(" ".join(command_parts), timeout=self.create_vm_timeout) async def list_vms( self, @@ -251,15 +233,12 @@ async def _exec_dict( async def _request( self, command: str, *, timeout: httpx.Timeout | None = None ) -> httpx.Response: - client = self._get_client() try: - if timeout is None: - response = await client.post("/exec", content=command) - else: - response = await client.post("/exec", content=command, timeout=timeout) + response = await self._get_client().post( + "/exec", content=command, timeout=timeout or self.timeout + ) except httpx.RequestError as exc: - # str(exc) is empty for bare transport errors like httpx.ReadTimeout(); - # {exc!r} always names the exception type so the failure is diagnosable. + # {exc!r}, not {exc}: bare transport errors like ReadTimeout() stringify empty. raise ExeResponseError(f"exe.dev API transport failed: {exc!r}") from exc if response.status_code == 401: diff --git a/src/providers/exe/tests/test_api.py b/src/providers/exe/tests/test_api.py index c521c3f..cc9b850 100644 --- a/src/providers/exe/tests/test_api.py +++ b/src/providers/exe/tests/test_api.py @@ -165,8 +165,6 @@ async def test_request_sends_authorization_and_text_body(respx_mock): @pytest.mark.asyncio @respx.mock(base_url="https://exe.dev") async def test_request_names_exception_type_for_blank_transport_error(respx_mock): - # str(httpx.ReadTimeout("")) is an empty string; the wrapped ExeResponseError - # must still name the exception type and keep the original chained. read_timeout = httpx.ReadTimeout("") respx_mock.post("/exec").mock(side_effect=read_timeout) @@ -183,14 +181,13 @@ async def test_create_vm_raises_read_timeout_floor_when_general_timeout_below(re return_value=httpx.Response(200, content=b'{"vm_name": "sb-1", "ssh_port": 22}'), ) - # Default general timeout is 30s, below the creation floor. + # The default 30s timeout sits below the creation floor. await ExeAPI(base_url="https://exe.dev", token="token").create_vm( name="sb-1", image="ubuntu:22.04" ) timeout = route.calls.last.request.extensions["timeout"] assert timeout["read"] == 90.0 - # Connect, write, and pool budgets stay on the configured general timeout. assert timeout["connect"] == 5.0 assert timeout["write"] == 30.0 assert timeout["pool"] == 30.0 @@ -203,7 +200,6 @@ async def test_create_vm_keeps_configured_timeout_when_above_floor(respx_mock): return_value=httpx.Response(200, content=b'{"vm_name": "sb-1", "ssh_port": 22}'), ) - # A configured general read timeout above the floor must not be shortened. await ExeAPI(base_url="https://exe.dev", token="token", timeout=120.0).create_vm( name="sb-1", image="ubuntu:22.04" ) @@ -218,8 +214,7 @@ async def test_ordinary_command_after_create_vm_uses_general_timeout(respx_mock) return_value=httpx.Response(200, content=b'{"vm_name": "sb-1", "ssh_port": 22}'), ) - # Creation must not mutate the cached client: a later ordinary command on the - # same instance still uses the configured general timeout. + # Creation must not leak its bigger budget into later commands on the client. api = ExeAPI(base_url="https://exe.dev", token="token") await api.create_vm(name="sb-1", image="ubuntu:22.04") await api.whoami()