Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions hud/agents/openai/tools/coding.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,23 @@
OPENAI_SHELL_SPEC = OpenAIToolSpec(
api_type="shell",
api_name="shell",
supported_models=(
"gpt-5.4",
"gpt-5.4-*",
"gpt-5.5",
"gpt-5.5-*",
"gpt-5.6",
"gpt-5.6-*",
),
Comment thread
cursor[bot] marked this conversation as resolved.
)


class OpenAIShellTool(SSHTool):
name = "shell"

@classmethod
def default_spec(cls, model: str) -> OpenAIToolSpec:
del model
return OPENAI_SHELL_SPEC
def default_spec(cls, model: str) -> OpenAIToolSpec | None:
return OPENAI_SHELL_SPEC if OPENAI_SHELL_SPEC.supports_model(model) else None

def to_params(self) -> Any:
# openai.types.responses.FunctionShellToolParam, as a plain dict (TypedDicts
Expand Down
13 changes: 10 additions & 3 deletions hud/agents/openai/tools/computer.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@
OPENAI_COMPUTER_SPEC = OpenAIToolSpec(
api_type="computer",
api_name="computer",
supported_models=(
"gpt-5.4",
"gpt-5.4-*",
"gpt-5.5",
"gpt-5.5-*",
"gpt-5.6",
"gpt-5.6-*",
),
)


Expand Down Expand Up @@ -81,9 +89,8 @@ class OpenAIComputerTool(RFBTool):
name = "computer"

@classmethod
def default_spec(cls, model: str) -> OpenAIToolSpec:
del model
return OPENAI_COMPUTER_SPEC
def default_spec(cls, model: str) -> OpenAIToolSpec | None:
return OPENAI_COMPUTER_SPEC if OPENAI_COMPUTER_SPEC.supports_model(model) else None

def to_params(self) -> Any:
return {"type": "computer"}
Expand Down
99 changes: 88 additions & 11 deletions hud/agents/tests/test_provider_native_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from __future__ import annotations

import logging
import shlex
from types import SimpleNamespace
from typing import Any, cast
Expand All @@ -17,14 +18,15 @@

from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool
from hud.agents.gemini.tools.coding import GeminiEditTool, GeminiShellTool
from hud.agents.openai.tools.coding import OpenAIShellTool
from hud.agents.openai.agent import OpenAIAgent
from hud.agents.openai.tools.coding import OPENAI_SHELL_SPEC, OpenAIShellTool
from hud.agents.openai_compatible.agent import OpenAIChatAgent
from hud.agents.openai_compatible.tools import BashTool, EditTool, ReadTool, WriteTool
from hud.agents.tool_agent import RunState
from hud.agents.tools.base import result_text
from hud.agents.tools.ssh import bound_shell_output
from hud.agents.types import OpenAIChatConfig
from hud.capabilities import Capability, SSHClient
from hud.agents.types import OpenAIChatConfig, OpenAIConfig
from hud.capabilities import Capability, CapabilityClient, RFBClient, SSHClient
from hud.types import MCPToolCall


Expand Down Expand Up @@ -172,9 +174,84 @@ async def build_tools_for_test(self, ssh: SSHClient) -> tuple[dict[str, Any], li
return await self._build_tools({"ssh": ssh})


class _OpenAIAgentForTest(OpenAIAgent):
async def build_native_tools_for_test(
self,
connections: dict[str, CapabilityClient],
) -> tuple[dict[str, Any], list[Any]]:
return await self._build_tools(connections)


# ─── OpenAI shell ─────────────────────────────────────────────────────


@pytest.mark.parametrize(
("model", "expected_types"),
[
("gpt-4o-mini", []),
("gpt-5.4-mini", ["shell", "computer"]),
("gpt-5.5", ["shell", "computer"]),
("gpt-5.6", ["shell", "computer"]),
("gpt-5.6-2026-08-07", ["shell", "computer"]),
],
)
async def test_openai_native_tools_are_registered_only_for_supported_models(
model: str,
expected_types: list[str],
) -> None:
agent = _OpenAIAgentForTest(OpenAIConfig(model=model, model_client=cast("Any", object())))

tools, params = await agent.build_native_tools_for_test(
{
"shell": _ssh(),
"computer": object.__new__(RFBClient),
}
)

assert list(tools) == expected_types
assert [param["type"] for param in params] == expected_types


async def test_unsupported_native_tool_warns_when_environment_exposes_capability(
caplog: pytest.LogCaptureFixture,
) -> None:
agent = _OpenAIAgentForTest(
OpenAIConfig(model="gpt-4o-mini", model_client=cast("Any", object()))
)
caplog.set_level(logging.WARNING, logger="hud.agents.tool_agent")

await agent.build_native_tools_for_test({"shell": _ssh()})

assert [record.getMessage() for record in caplog.records] == [
"Skipping tool 'shell' for model 'gpt-4o-mini' because the model does not support it; "
"the rollout will continue without the matching environment capability"
]


async def test_unsupported_native_tool_does_not_warn_without_matching_capability(
caplog: pytest.LogCaptureFixture,
) -> None:
agent = _OpenAIAgentForTest(
OpenAIConfig(model="gpt-4o-mini", model_client=cast("Any", object()))
)
caplog.set_level(logging.WARNING, logger="hud.agents.tool_agent")

await agent.build_native_tools_for_test({})

assert caplog.records == []


async def test_supported_native_tool_does_not_warn(
caplog: pytest.LogCaptureFixture,
) -> None:
agent = _OpenAIAgentForTest(OpenAIConfig(model="gpt-5.6", model_client=cast("Any", object())))
caplog.set_level(logging.WARNING, logger="hud.agents.tool_agent")

await agent.build_native_tools_for_test({"shell": _ssh()})

assert caplog.records == []


@pytest.mark.parametrize(
("command", "timeout_ms", "expected"),
[
Expand All @@ -192,7 +269,7 @@ async def test_openai_shell_applies_requested_timeout_to_entire_command(
timeout_ms: int,
expected: str,
) -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())

result = await tool.execute({"commands": [command], "timeout_ms": timeout_ms})

Expand All @@ -204,7 +281,7 @@ async def test_openai_shell_applies_requested_timeout_to_entire_command(


async def test_openai_shell_runs_each_command_without_timeout() -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())

await tool.execute({"commands": ["echo a", "echo b"]})

Expand All @@ -214,7 +291,7 @@ async def test_openai_shell_runs_each_command_without_timeout() -> None:
async def test_openai_shell_has_no_hidden_timeout_across_command_batch(
monkeypatch: pytest.MonkeyPatch,
) -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())
calls: list[dict[str, Any]] = []

async def run(*args: object, **kwargs: Any) -> _Completed:
Expand All @@ -239,7 +316,7 @@ async def run(*args: object, **kwargs: Any) -> _Completed:
async def test_openai_shell_applies_limit_independently_to_each_command() -> None:
limit = 80
tool = OpenAIShellTool(
spec=OpenAIShellTool.default_spec("gpt-5.5"),
spec=OPENAI_SHELL_SPEC,
client=_ssh(stdout="stdout-start-" + "a" * 100, stderr="b" * 100 + "-stderr-end"),
)

Expand Down Expand Up @@ -274,7 +351,7 @@ def test_shared_output_bound_handles_limits_smaller_than_marker(
async def test_openai_shell_rejects_invalid_output_limits_without_running(
max_output_length: Any,
) -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())

result = await tool.execute(
{"commands": ["echo should-not-run"], "max_output_length": max_output_length}
Expand All @@ -289,7 +366,7 @@ async def test_openai_shell_rejects_invalid_output_limits_without_running(

@pytest.mark.parametrize("max_output_length", [None, 20 * 1024 * 1024])
async def test_openai_shell_uses_safe_effective_limit(max_output_length: int | None) -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())
arguments: dict[str, Any] = {"commands": ["echo ok"]}
if max_output_length is not None:
arguments["max_output_length"] = max_output_length
Expand All @@ -301,7 +378,7 @@ async def test_openai_shell_uses_safe_effective_limit(max_output_length: int | N


async def test_openai_shell_rejects_non_list_commands_without_running() -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())

result = await tool.execute({"commands": 123})

Expand All @@ -310,7 +387,7 @@ async def test_openai_shell_rejects_non_list_commands_without_running() -> None:


def test_openai_shell_to_params_is_shell_type() -> None:
tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh())
tool = OpenAIShellTool(spec=OPENAI_SHELL_SPEC, client=_ssh())
assert tool.to_params()["type"] == "shell"


Expand Down
17 changes: 14 additions & 3 deletions hud/agents/tool_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,12 +168,23 @@ async def _build_tools(
qualify_mcp_names = len(mcp_clients) > 1

for tool_cls in type(self).tool_catalog:
matching_connections = [
(connection_name, client)
for connection_name, client in connections.items()
if isinstance(client, tool_cls.client_type)
]
if not matching_connections:
continue
spec = tool_cls.default_spec(model)
if spec is None:
logger.warning(
"Skipping tool %r for model %r because the model does not support it; "
"the rollout will continue without the matching environment capability",
tool_cls.name,
model,
)
continue
for connection_name, client in connections.items():
if not isinstance(client, tool_cls.client_type):
continue
for connection_name, client in matching_connections:
if issubclass(tool_cls, MCPTool):
assert isinstance(client, MCPClient)
for mt in mcp_by_client[client]:
Expand Down
Loading