From 9a101a0172e2710bd60f8b5975ac1bd04be586e1 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:57:53 +0500 Subject: [PATCH 1/2] fix(openai): gate native tools by model support --- hud/agents/openai/tools/coding.py | 13 ++++- hud/agents/openai/tools/computer.py | 13 ++++- .../tests/test_provider_native_tools.py | 57 +++++++++++++++---- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/hud/agents/openai/tools/coding.py b/hud/agents/openai/tools/coding.py index a1d970cd0..af397fd07 100644 --- a/hud/agents/openai/tools/coding.py +++ b/hud/agents/openai/tools/coding.py @@ -16,6 +16,14 @@ OPENAI_SHELL_SPEC = OpenAIToolSpec( api_type="shell", api_name="shell", + supported_models=( + "gpt-5.4", + "gpt-5.4-*", + "gpt-5.5", + "gpt-5.5-*", + "gpt-5.6", + "gpt-5.6-*", + ), ) @@ -23,9 +31,8 @@ class OpenAIShellTool(SSHTool): name = "shell" @classmethod - def default_spec(cls, model: str) -> OpenAIToolSpec: - del model - return OPENAI_SHELL_SPEC + def default_spec(cls, model: str) -> OpenAIToolSpec | None: + return OPENAI_SHELL_SPEC if OPENAI_SHELL_SPEC.supports_model(model) else None def to_params(self) -> Any: # openai.types.responses.FunctionShellToolParam, as a plain dict (TypedDicts diff --git a/hud/agents/openai/tools/computer.py b/hud/agents/openai/tools/computer.py index 6b8196f00..3b3d2a8a6 100644 --- a/hud/agents/openai/tools/computer.py +++ b/hud/agents/openai/tools/computer.py @@ -18,6 +18,14 @@ OPENAI_COMPUTER_SPEC = OpenAIToolSpec( api_type="computer", api_name="computer", + supported_models=( + "gpt-5.4", + "gpt-5.4-*", + "gpt-5.5", + "gpt-5.5-*", + "gpt-5.6", + "gpt-5.6-*", + ), ) @@ -81,9 +89,8 @@ class OpenAIComputerTool(RFBTool): name = "computer" @classmethod - def default_spec(cls, model: str) -> OpenAIToolSpec: - del model - return OPENAI_COMPUTER_SPEC + def default_spec(cls, model: str) -> OpenAIToolSpec | None: + return OPENAI_COMPUTER_SPEC if OPENAI_COMPUTER_SPEC.supports_model(model) else None def to_params(self) -> Any: return {"type": "computer"} diff --git a/hud/agents/tests/test_provider_native_tools.py b/hud/agents/tests/test_provider_native_tools.py index ce1d37687..8d76264d1 100644 --- a/hud/agents/tests/test_provider_native_tools.py +++ b/hud/agents/tests/test_provider_native_tools.py @@ -17,14 +17,15 @@ from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool from hud.agents.gemini.tools.coding import GeminiEditTool, GeminiShellTool -from hud.agents.openai.tools.coding import OpenAIShellTool +from hud.agents.openai.agent import OpenAIAgent +from hud.agents.openai.tools.coding import OPENAI_SHELL_SPEC, OpenAIShellTool from hud.agents.openai_compatible.agent import OpenAIChatAgent from hud.agents.openai_compatible.tools import BashTool, EditTool, ReadTool, WriteTool from hud.agents.tool_agent import RunState from hud.agents.tools.base import result_text from hud.agents.tools.ssh import bound_shell_output -from hud.agents.types import OpenAIChatConfig -from hud.capabilities import Capability, SSHClient +from hud.agents.types import OpenAIChatConfig, OpenAIConfig +from hud.capabilities import Capability, RFBClient, SSHClient from hud.types import MCPToolCall @@ -172,9 +173,43 @@ async def build_tools_for_test(self, ssh: SSHClient) -> tuple[dict[str, Any], li return await self._build_tools({"ssh": ssh}) +class _OpenAIAgentForTest(OpenAIAgent): + async def build_native_tools_for_test( + self, + ssh: SSHClient, + rfb: RFBClient, + ) -> tuple[dict[str, Any], list[Any]]: + return await self._build_tools({"shell": ssh, "computer": rfb}) + + # ─── OpenAI shell ───────────────────────────────────────────────────── +@pytest.mark.parametrize( + ("model", "expected_types"), + [ + ("gpt-4o-mini", []), + ("gpt-5.4-mini", ["shell", "computer"]), + ("gpt-5.5", ["shell", "computer"]), + ("gpt-5.6", ["shell", "computer"]), + ("gpt-5.6-2026-08-07", ["shell", "computer"]), + ], +) +async def test_openai_native_tools_are_registered_only_for_supported_models( + model: str, + expected_types: list[str], +) -> None: + agent = _OpenAIAgentForTest(OpenAIConfig(model=model, model_client=cast("Any", object()))) + + tools, params = await agent.build_native_tools_for_test( + _ssh(), + object.__new__(RFBClient), + ) + + assert list(tools) == expected_types + assert [param["type"] for param in params] == expected_types + + @pytest.mark.parametrize( ("command", "timeout_ms", "expected"), [ @@ -192,7 +227,7 @@ async def test_openai_shell_applies_requested_timeout_to_entire_command( timeout_ms: int, expected: str, ) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) result = await tool.execute({"commands": [command], "timeout_ms": timeout_ms}) @@ -204,7 +239,7 @@ async def test_openai_shell_applies_requested_timeout_to_entire_command( async def test_openai_shell_runs_each_command_without_timeout() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) await tool.execute({"commands": ["echo a", "echo b"]}) @@ -214,7 +249,7 @@ async def test_openai_shell_runs_each_command_without_timeout() -> None: async def test_openai_shell_has_no_hidden_timeout_across_command_batch( monkeypatch: pytest.MonkeyPatch, ) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) calls: list[dict[str, Any]] = [] async def run(*args: object, **kwargs: Any) -> _Completed: @@ -239,7 +274,7 @@ async def run(*args: object, **kwargs: Any) -> _Completed: async def test_openai_shell_applies_limit_independently_to_each_command() -> None: limit = 80 tool = OpenAIShellTool( - spec=OpenAIShellTool.default_spec("gpt-5.5"), + spec=OPENAI_SHELL_SPEC, client=_ssh(stdout="stdout-start-" + "a" * 100, stderr="b" * 100 + "-stderr-end"), ) @@ -274,7 +309,7 @@ def test_shared_output_bound_handles_limits_smaller_than_marker( async def test_openai_shell_rejects_invalid_output_limits_without_running( max_output_length: Any, ) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) result = await tool.execute( {"commands": ["echo should-not-run"], "max_output_length": max_output_length} @@ -289,7 +324,7 @@ async def test_openai_shell_rejects_invalid_output_limits_without_running( @pytest.mark.parametrize("max_output_length", [None, 20 * 1024 * 1024]) async def test_openai_shell_uses_safe_effective_limit(max_output_length: int | None) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) arguments: dict[str, Any] = {"commands": ["echo ok"]} if max_output_length is not None: arguments["max_output_length"] = max_output_length @@ -301,7 +336,7 @@ async def test_openai_shell_uses_safe_effective_limit(max_output_length: int | N async def test_openai_shell_rejects_non_list_commands_without_running() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) result = await tool.execute({"commands": 123}) @@ -310,7 +345,7 @@ async def test_openai_shell_rejects_non_list_commands_without_running() -> None: def test_openai_shell_to_params_is_shell_type() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) + tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh()) assert tool.to_params()["type"] == "shell" From 785316d8f322bade75b1bd31eaf47f5299c22128 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:41:57 +0500 Subject: [PATCH 2/2] fix(agents): warn when model skips environment tool --- .../tests/test_provider_native_tools.py | 54 ++++++++++++++++--- hud/agents/tool_agent.py | 17 ++++-- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/hud/agents/tests/test_provider_native_tools.py b/hud/agents/tests/test_provider_native_tools.py index 8d76264d1..e87286c95 100644 --- a/hud/agents/tests/test_provider_native_tools.py +++ b/hud/agents/tests/test_provider_native_tools.py @@ -7,6 +7,7 @@ from __future__ import annotations +import logging import shlex from types import SimpleNamespace from typing import Any, cast @@ -25,7 +26,7 @@ from hud.agents.tools.base import result_text from hud.agents.tools.ssh import bound_shell_output from hud.agents.types import OpenAIChatConfig, OpenAIConfig -from hud.capabilities import Capability, RFBClient, SSHClient +from hud.capabilities import Capability, CapabilityClient, RFBClient, SSHClient from hud.types import MCPToolCall @@ -176,10 +177,9 @@ async def build_tools_for_test(self, ssh: SSHClient) -> tuple[dict[str, Any], li class _OpenAIAgentForTest(OpenAIAgent): async def build_native_tools_for_test( self, - ssh: SSHClient, - rfb: RFBClient, + connections: dict[str, CapabilityClient], ) -> tuple[dict[str, Any], list[Any]]: - return await self._build_tools({"shell": ssh, "computer": rfb}) + return await self._build_tools(connections) # ─── OpenAI shell ───────────────────────────────────────────────────── @@ -202,14 +202,56 @@ async def test_openai_native_tools_are_registered_only_for_supported_models( agent = _OpenAIAgentForTest(OpenAIConfig(model=model, model_client=cast("Any", object()))) tools, params = await agent.build_native_tools_for_test( - _ssh(), - object.__new__(RFBClient), + { + "shell": _ssh(), + "computer": object.__new__(RFBClient), + } ) assert list(tools) == expected_types assert [param["type"] for param in params] == expected_types +async def test_unsupported_native_tool_warns_when_environment_exposes_capability( + caplog: pytest.LogCaptureFixture, +) -> None: + agent = _OpenAIAgentForTest( + OpenAIConfig(model="gpt-4o-mini", model_client=cast("Any", object())) + ) + caplog.set_level(logging.WARNING, logger="hud.agents.tool_agent") + + await agent.build_native_tools_for_test({"shell": _ssh()}) + + assert [record.getMessage() for record in caplog.records] == [ + "Skipping tool 'shell' for model 'gpt-4o-mini' because the model does not support it; " + "the rollout will continue without the matching environment capability" + ] + + +async def test_unsupported_native_tool_does_not_warn_without_matching_capability( + caplog: pytest.LogCaptureFixture, +) -> None: + agent = _OpenAIAgentForTest( + OpenAIConfig(model="gpt-4o-mini", model_client=cast("Any", object())) + ) + caplog.set_level(logging.WARNING, logger="hud.agents.tool_agent") + + await agent.build_native_tools_for_test({}) + + assert caplog.records == [] + + +async def test_supported_native_tool_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + agent = _OpenAIAgentForTest(OpenAIConfig(model="gpt-5.6", model_client=cast("Any", object()))) + caplog.set_level(logging.WARNING, logger="hud.agents.tool_agent") + + await agent.build_native_tools_for_test({"shell": _ssh()}) + + assert caplog.records == [] + + @pytest.mark.parametrize( ("command", "timeout_ms", "expected"), [ diff --git a/hud/agents/tool_agent.py b/hud/agents/tool_agent.py index 5c2b31f53..ec85ad5bc 100644 --- a/hud/agents/tool_agent.py +++ b/hud/agents/tool_agent.py @@ -168,12 +168,23 @@ async def _build_tools( qualify_mcp_names = len(mcp_clients) > 1 for tool_cls in type(self).tool_catalog: + matching_connections = [ + (connection_name, client) + for connection_name, client in connections.items() + if isinstance(client, tool_cls.client_type) + ] + if not matching_connections: + continue spec = tool_cls.default_spec(model) if spec is None: + logger.warning( + "Skipping tool %r for model %r because the model does not support it; " + "the rollout will continue without the matching environment capability", + tool_cls.name, + model, + ) continue - for connection_name, client in connections.items(): - if not isinstance(client, tool_cls.client_type): - continue + for connection_name, client in matching_connections: if issubclass(tool_cls, MCPTool): assert isinstance(client, MCPClient) for mt in mcp_by_client[client]: