From a20a1b6d024d34b87ed712d4a24560b7b50d192b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 11 Jul 2026 09:42:48 -0400 Subject: [PATCH 01/68] Only display file adding warnings when more than 50% of the compaction context window is used --- cecli/coders/base_coder.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 2633bc319e1..1b580aa491f 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -4239,9 +4239,12 @@ def check_added_files(self): if tokens < warn_number_of_tokens: return - self.io.tool_warning("Warning: it's best to only add files that need changes to the chat.") - self.io.tool_warning(urls.edit_errors) - self.warning_given = True + if self.context_compaction_current_ratio > 0.5: + self.io.tool_warning( + "Warning: it's best to only add files that need changes to the chat." + ) + self.io.tool_warning(urls.edit_errors) + self.warning_given = True async def prepare_to_edit(self, edits): res = [] From ca5f44a88ed6a14f0231d30d063e92f8a0d32155 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 11 Jul 2026 15:54:35 -0400 Subject: [PATCH 02/68] Programmatic tool calling base implementation --- cecli/coders/agent_coder.py | 26 + cecli/helpers/conversation/integration.py | 4 + cecli/helpers/orchestration/__init__.py | 35 + cecli/helpers/orchestration/environment.py | 482 ++++++++++++++ cecli/helpers/orchestration/security.py | 101 +++ cecli/helpers/orchestration/service.py | 30 + cecli/prompts/agent.yml | 2 +- cecli/prompts/subagent.yml | 2 +- cecli/tools/__init__.py | 3 +- cecli/tools/orchestrate.py | 73 +++ tests/helpers/test_orchestration.py | 717 +++++++++++++++++++++ 11 files changed, 1472 insertions(+), 3 deletions(-) create mode 100644 cecli/helpers/orchestration/__init__.py create mode 100644 cecli/helpers/orchestration/environment.py create mode 100644 cecli/helpers/orchestration/security.py create mode 100644 cecli/helpers/orchestration/service.py create mode 100644 cecli/tools/orchestrate.py create mode 100644 tests/helpers/test_orchestration.py diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 943f6b1c538..eeab4ea1da9 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -184,6 +184,8 @@ def _get_agent_config(self): config["servers_excludelist"] = nested.getter( config, ["servers_excludelist", "servers_blacklist"], [] ) + config["allow_orchestration"] = nested.getter(config, "allow_orchestration", True) + config["include_context_blocks"] = set( nested.getter( config, @@ -203,6 +205,10 @@ def _get_agent_config(self): ) config["exclude_context_blocks"] = set(nested.getter(config, "exclude_context_blocks", [])) + if config["allow_orchestration"]: + config["include_context_blocks"].add("orchestration") + config["exclude_context_blocks"].discard("orchestration") + self.large_file_token_threshold = config["large_file_token_threshold"] self.skip_cli_confirmations = config["skip_cli_confirmations"] self.hot_reload_enabled = config["hot_reload"] @@ -364,6 +370,7 @@ def _calculate_context_block_tokens(self, force=False): "servers", "sub_agents", "loaded_skills", + "orchestration", ] for block_type in block_types: if block_type in self.allowed_context_blocks: @@ -400,6 +407,8 @@ def _generate_context_block(self, block_name): content = self.get_servers_context() elif block_name == "loaded_skills": content = self.get_skills_content() + elif block_name == "orchestration": + content = self.get_orchestration_context() elif block_name == "sub_agents" and ( not self.parent_uuid or self.agent_config.get("allow_nested_delegation", False) ): @@ -1582,6 +1591,23 @@ def get_servers_context(self): self.io.tool_error(f"Error generating servers context: {str(e)}") return None + def get_orchestration_context(self): + """ + Generate a context block for the Orchestrate tool if allowed. + + Only returns content if ``allow_orchestration`` is enabled in the agent config. + """ + if not self.use_enhanced_context: + return None + + try: + from cecli.helpers.orchestration import build_orchestration_context_block + + return build_orchestration_context_block(self.agent_config) + except Exception as e: + self.io.tool_error(f"Error generating orchestration context: {str(e)}") + return None + def get_sub_agents_context(self): """ Generate a context block for registered sub-agents. diff --git a/cecli/helpers/conversation/integration.py b/cecli/helpers/conversation/integration.py index 169d3676796..8888ab3b4ad 100644 --- a/cecli/helpers/conversation/integration.py +++ b/cecli/helpers/conversation/integration.py @@ -809,6 +809,10 @@ def add_static_context_blocks(self) -> None: block = coder._generate_context_block("servers") if block: message_blocks["servers"] = block + if "orchestration" in coder.allowed_context_blocks: + block = coder._generate_context_block("orchestration") + if block: + message_blocks["orchestration"] = block # Add static blocks to conversation manager with stable hash keys for block_type, block_content in message_blocks.items(): diff --git a/cecli/helpers/orchestration/__init__.py b/cecli/helpers/orchestration/__init__.py new file mode 100644 index 00000000000..2ca920fd59c --- /dev/null +++ b/cecli/helpers/orchestration/__init__.py @@ -0,0 +1,35 @@ +""" +Orchestration module for programmatic tool calling in agent mode. + +Public API: +- AgentExecutionEnv: sandboxed execution environment +- AgentProxy / ToolProxy: proxies for calling tools from generated code +- build_orchestration_context_block: context block builder +- SecurityError / SecurityFilter / LoopYieldInjector: security components +""" + +from cecli.helpers.orchestration.environment import ( + AgentExecutionEnv, + AgentProxy, + ToolProxy, + build_orchestration_context_block, +) +from cecli.helpers.orchestration.security import ( + LoopYieldInjector, + SecurityError, + SecurityFilter, + _cooperative_yield, +) +from cecli.helpers.orchestration.service import OrchestrationService + +__all__ = [ + "AgentExecutionEnv", + "AgentProxy", + "ToolProxy", + "build_orchestration_context_block", + "SecurityError", + "SecurityFilter", + "LoopYieldInjector", + "_cooperative_yield", + "OrchestrationService", +] diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py new file mode 100644 index 00000000000..e7f588e64e8 --- /dev/null +++ b/cecli/helpers/orchestration/environment.py @@ -0,0 +1,482 @@ +""" +Sandboxed execution environment for LLM-generated orchestration code. + +Provides AgentExecutionEnv, ToolProxy, AgentProxy, and the context block builder. +""" + +from __future__ import annotations + +import ast +import asyncio +import logging +import traceback +from typing import Any + +from cecli.helpers import nested, responses +from cecli.helpers.orchestration.security import ( + LoopYieldInjector, + SecurityError, + SecurityFilter, + _cooperative_yield, +) + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Safe primitives exposed to the sandbox +# --------------------------------------------------------------------------- + + +async def _safe_sleep(seconds: float) -> None: + """Safe sleep wrapper for the orchestration environment.""" + if seconds < 0: + raise ValueError("sleep() requires a non-negative value") + if seconds > 120: + raise ValueError("sleep() is limited to 120 seconds maximum") + await asyncio.sleep(seconds) + + +async def _safe_gather(*awaitables: Any) -> list[Any]: + """ + Safely execute multiple awaitables concurrently. + + Forces ``return_exceptions=True`` so that failures in one task + do not crash the entire batch. + """ + return await asyncio.gather(*awaitables, return_exceptions=True) + + +class _SafeJson: + """Drop-in ``json`` namespace with only ``loads`` and ``dumps``.""" + + @staticmethod + def loads(s: str) -> Any: + import json + + return json.loads(s) + + @staticmethod + def dumps(obj: Any) -> str: + import json + + return json.dumps(obj) + + +# --------------------------------------------------------------------------- +# Tool proxies +# --------------------------------------------------------------------------- + + +class ToolProxy: + """ + Proxy for a single tool — local or MCP. + + The LLM code calls ``tool.call(**params)`` and this proxy routes it + through the appropriate execution path. + """ + + def __init__( + self, + tool_name: str, + coder: Any, + *, + tool_module: Any = None, + mcp_server: Any = None, + mcp_tool_name: str = "", + ) -> None: + # Respect the per-coder tool includelist/excludelist filters + incl = getattr(coder, "registered_tools", {}).get("included", set()) + excl = getattr(coder, "registered_tools", {}).get("excluded", set()) + name_lower = tool_name.lower() + if incl and name_lower not in incl: + raise ValueError(f"Tool '{tool_name}' is not in the allowed tools list.") + if name_lower in excl: + raise ValueError(f"Tool '{tool_name}' has been excluded.") + + self._tool_name = tool_name + self._coder = coder + self._tool_module = tool_module + self._mcp_server = mcp_server + self._mcp_tool_name = mcp_tool_name + + async def call(self, **kwargs: Any): + """Execute the tool with the given keyword arguments. + + Tool results are returned as strings. Use ``import json`` and + ``json.loads(result)`` to parse structured responses when needed. + """ + if self._tool_module is not None: + result = self._tool_module.process_response(self._coder, kwargs) + if asyncio.iscoroutine(result): + result = await result + return str(result) + + if self._mcp_server is not None: + return await self._coder._execute_mcp_tool( + self._mcp_server, self._mcp_tool_name, kwargs + ) + + raise ValueError(f"No executor for tool '{self._tool_name}'") + + +class AgentProxy: + """ + Singleton-like proxy injected into the orchestration environment. + + Usage in LLM-generated code:: + + read_tool = Agent.get_tool("ReadFile") + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + + Supports both local tools (from ToolRegistry) and MCP tools (from connected + servers) using ``ServerName--ToolName`` or bare tool-name lookup. + """ + + def __init__(self, coder: Any) -> None: + self._coder = coder + + def get_tool(self, tool_name: str) -> ToolProxy: + from cecli.tools.utils.registry import ToolRegistry + + name_lower = tool_name.lower() + + # 1. Try local tools by exact name (unprefixed) + tool_module = ToolRegistry.get_tool(name_lower) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 2. Unprefix: "ServerName--ToolName" → (server_prefix, bare_name) + server_prefix, bare_name = responses.unprefix_tool_name(name_lower) + + # 3. If the prefix is "local", retry ToolRegistry with the bare name + if server_prefix == "local" and bare_name: + tool_module = ToolRegistry.get_tool(bare_name) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 4. Search MCP tools for the bare (unprefixed) name + for mcp_server_name, server_tools in self._coder.mcp_tools or []: + for tool_schema in server_tools: + schema_name = nested.getter(tool_schema, "function.name", "") + _schema_prefix, schema_unprefixed = responses.unprefix_tool_name( + schema_name.lower() + ) + if schema_unprefixed == bare_name: + server = self._find_mcp_server(mcp_server_name, server_prefix) + if server is not None: + return ToolProxy( + tool_name, + self._coder, + mcp_server=server, + mcp_tool_name=schema_name, + ) + + raise ValueError(f"Unknown tool: '{tool_name}'") + + def _find_mcp_server(self, server_name: str, server_prefix: str) -> Any: + if not hasattr(self._coder, "mcp_manager") or not self._coder.mcp_manager: + return None + for server in self._coder.mcp_manager: + if server.name == server_name and ( + not server_prefix or server.name.lower() == server_prefix.lower() + ): + return server + return None + + +# --------------------------------------------------------------------------- +# Main execution environment +# --------------------------------------------------------------------------- + + +class AgentExecutionEnv: + """ + Sandboxed REPL environment for executing LLM-generated orchestration code. + + Provides: + - ``Agent`` : proxy to look up and call tools + - ``gather`` : safe parallel execution helper + - ``state`` : persistent dict (survives across Orchestrate calls) + - ``sleep`` : safe sleep (0-120 seconds) + - ``print`` : captured output + - ``range``, ``len``, ``int``, ``str``, ``list``, ``dict``, ``bool``, ``Exception`` + + Security guarantees: + - AST security filtering before compilation + - Loop yield injection + - Timeout via asyncio.wait_for + - No imports, no private attributes, no dangerous builtins + """ + + # Shared across all AgentExecutionEnv instances — any agent can read/write + _shared_state: dict[str, Any] = {} + + def __init__(self, coder: Any) -> None: + self.state: dict[str, Any] = {} + + _safe_builtins: dict[str, Any] = { + "print": print, + "range": range, + "len": len, + "int": int, + "str": str, + "float": float, + "list": list, + "dict": dict, + "bool": bool, + "tuple": tuple, + "set": set, + # "type": type, # excluded for security (can create dynamic classes) + "isinstance": isinstance, + "enumerate": enumerate, + "zip": zip, + "sorted": sorted, + "reversed": reversed, + "min": min, + "max": max, + "sum": sum, + "abs": abs, + "round": round, + "any": any, + "all": all, + # "filter": filter, # excluded for security + # "map": map, # excluded for security + "Exception": Exception, + "ValueError": ValueError, + "TypeError": TypeError, + "KeyError": KeyError, + "IndexError": IndexError, + "AttributeError": AttributeError, + "RuntimeError": RuntimeError, + } + + self.globals: dict[str, Any] = { + "__builtins__": _safe_builtins, + "Agent": AgentProxy(coder), + "gather": _safe_gather, + "sleep": _safe_sleep, + "json": _SafeJson, + "state": self.state, + "shared_state": AgentExecutionEnv._shared_state, + "__yield": _cooperative_yield, + } + self.locals: dict[str, Any] = {} + + self.globals["reset"] = self.locals.clear + + @staticmethod + def _size_of(value: Any) -> int: + """Return a meaningful size metric for a state variable.""" + if value is None: + return 0 + if isinstance(value, (str, list, tuple, set, dict)): + return len(value) + return len(str(value)) + + def _state_snapshot(self) -> list: + """Build a list of state variable descriptors with modification tracking.""" + modified_keys = getattr(self, "_modified_keys", set()) + modified_shared_keys = getattr(self, "_modified_shared_keys", set()) + entries = [] + + for key, value in self.state.items(): + entries.append( + { + "name": key, + "type": type(value).__name__, + "size": self._size_of(value), + "modified": key in modified_keys, + "scope": "local", + } + ) + + for key, value in self._shared_state.items(): + entries.append( + { + "name": key, + "type": type(value).__name__, + "size": self._size_of(value), + "modified": key in modified_shared_keys, + "scope": "shared", + } + ) + + return entries + + async def execute(self, code_str: str) -> dict: + + code_str = code_str.strip() + if not code_str: + return {"results": "", "state_variables": self._state_snapshot()} + + captured_output: list[str] = [] + + # Track which state keys are modified during this execution + _prev_state_keys = set(self.state.keys()) + _prev_shared_keys = set(self._shared_state.keys()) + + def _capture_print(*args: Any, **kwargs: Any) -> None: + sep = kwargs.pop("sep", " ") + end = kwargs.pop("end", "\n") + kwargs.pop("file", None) # silently ignore + kwargs.pop("flush", None) # silently ignore + if kwargs: + raise TypeError(f"print() got unexpected keyword arguments: {list(kwargs.keys())}") + captured_output.append(sep.join(str(a) for a in args) + end) + + self.globals["__builtins__"]["print"] = _capture_print + + try: + tree = ast.parse(code_str, filename="", mode="exec") + except SyntaxError as e: + code = f"Syntax Error in orchestration code: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + + try: + SecurityFilter().visit(tree) + except SecurityError as e: + code = f"Security Error: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + + tree = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(tree) + + returns_value = False + if tree.body and isinstance(tree.body[-1], ast.Expr): + last_expr = tree.body[-1].value + tree.body[-1] = ast.Return(value=last_expr) + returns_value = True + + wrapper_func = ast.AsyncFunctionDef( + name="__agent_async_runner", + args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), + body=tree.body, + decorator_list=[], + ) + + ast.fix_missing_locations(wrapper_func) + mod = ast.Module(body=[wrapper_func], type_ignores=[]) + + try: + compiled_code = compile(mod, filename="", mode="exec") + except Exception as e: + code = f"Compilation Error: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + + try: + exec(compiled_code, self.globals, self.locals) + runner_coro = self.locals["__agent_async_runner"]() + result = await runner_coro + except asyncio.CancelledError: + code = "Execution Error: Script was cancelled." + return {"results": code, "state_variables": self._state_snapshot()} + except SecurityError as e: + code = f"Security Error: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + except Exception as e: + tb = traceback.format_exc() + logger.warning("Orchestration execution error: %s\n%s", e, tb) + code = f"Execution Error: {type(e).__name__}: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + finally: + self.locals.pop("__agent_async_runner", None) + self.globals["__builtins__"]["print"] = print + self._modified_keys = set(self.state.keys()) - _prev_state_keys + self._modified_shared_keys = set(self._shared_state.keys()) - _prev_shared_keys + + print_output = "".join(captured_output) + + if returns_value and result is not None: + if print_output: + code = print_output.rstrip("\n") + "\n" + str(result) + return {"results": code, "state_variables": self._state_snapshot()} + code = str(result) + return {"results": code, "state_variables": self._state_snapshot()} + + if print_output: + code = print_output.rstrip("\n") + return {"results": code, "state_variables": self._state_snapshot()} + + if returns_value: + code = str(result) + return {"results": code, "state_variables": self._state_snapshot()} + + return {"results": "", "state_variables": self._state_snapshot()} + + +def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | None: + """ + Build the orchestration context block that explains calling conventions. + + Only returns content if ``allow_orchestration`` is enabled in agent_config. + """ + if not agent_config.get("allow_orchestration", True): + return None + + return """ +## Programmatic Tool Calling + +The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code. +This is much more efficient than making individual tool calls for loop-heavy workflows. + +### Available Primitives + +| Primitive | Description | +|-----------|-------------| +| `Agent.get_tool("ToolName")` | Returns a proxy for any available tool | +| `await tool.call(**params)` | Execute a tool with keyword arguments | +| `gather(*awaitables)` | Run multiple tool calls concurrently | +| `state` | Per-agent persistent dict that survives across Orchestrate calls | +| `shared_state` | Cross-agent shared dict visible to all agents and sub-agents | +| `sleep(seconds)` | Pause execution (0-120s max) | +| `print(...)` | Output messages; captured and returned in the result | +| `json.loads(s)` | Parse a JSON string into a Python dict/list | +| `json.dumps(obj)` | Serialize a Python object to a JSON string | +| `reset()` | Clear all local variables (does not touch `state`/`shared_state`) | + +### Common Patterns + +**Sequential calls:** +```python +read_tool = Agent.get_tool("ReadFile") +a = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") +b = await read_tool.call(file_path="bar.py", range_start="@000", range_end="000@") +f"Read {len(a)} and {len(b)} chars" +``` + +**Parallel calls:** +```python +read_tool = Agent.get_tool("ReadFile") +files = ["a.py", "b.py", "c.py"] +tasks = [read_tool.call(file_path=f, range_start="@000", range_end="000@") for f in files] +results = await gather(*tasks) +state["contents"] = dict(zip(files, results)) +f"Read {len(files)} files in parallel" +``` + +**Accumulating state across calls:** +```python +state["count"] = state.get("count", 0) + len(some_result) +f"Total so far: {state['count']}" +``` + +**Parsing JSON tool results:** +```python +read_tool = Agent.get_tool("ReadFile") +raw = await read_tool.call(file_path="data.json", range_start="@000", range_end="000@") +data = json.loads(raw) +f"Found {len(data)} entries" +``` + +### Tool Parameters + +Use the top-level parameters from each tool's schema as keyword arguments for `.call()`. +Refer to the tool descriptions for exact parameter names and types. + +### Rules + +1. No imports - use only the primitives above +2. Do not access attributes starting with `_` (private/dunder) +3. All tool calls must be awaited +4. The last expression's value is returned as the tool result +""" diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py new file mode 100644 index 00000000000..28bdd0c5e3a --- /dev/null +++ b/cecli/helpers/orchestration/security.py @@ -0,0 +1,101 @@ +""" +AST-level security transforms for the orchestration sandbox. + +- SecurityFilter: blocks imports, dunder attributes, dangerous builtins +- LoopYieldInjector: injects cooperative yields into loops +- _cooperative_yield: the injected yield function +""" + +import ast +import asyncio + + +class SecurityError(Exception): + """Raised when generated code violates security constraints.""" + + +class SecurityFilter(ast.NodeVisitor): + """ + AST node visitor that blocks dangerous constructs before they compile. + + Blocks: + - All import statements (import X, from X import Y) + - Access to private/dunder attributes (__class__, __subclasses__, etc.) + - Calls to eval, exec, open, __import__, compile, breakpoint + - global / nonlocal statements + """ + + _DANGEROUS_BUILTINS: set[str] = { + "eval", + "exec", + "open", + "__import__", + "compile", + "breakpoint", + "globals", + "locals", + "vars", + "getattr", + "setattr", + "delattr", + } + + def visit_Import(self, node: ast.Import) -> None: + raise SecurityError("Imports are disabled in the agent orchestration environment.") + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + raise SecurityError("Imports are disabled in the agent orchestration environment.") + + def visit_Attribute(self, node: ast.Attribute) -> None: + if node.attr.startswith("_"): + raise SecurityError(f"Access to private/dunder attribute '{node.attr}' is forbidden.") + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + if isinstance(node.func, ast.Name) and node.func.id in self._DANGEROUS_BUILTINS: + raise SecurityError(f"Calling '{node.func.id}' is forbidden.") + self.generic_visit(node) + + def visit_Global(self, node: ast.Global) -> None: + raise SecurityError("The 'global' statement is disabled in the orchestration environment.") + + def visit_Nonlocal(self, node: ast.Nonlocal) -> None: + raise SecurityError( + "The 'nonlocal' statement is disabled in the orchestration environment." + ) + + +class LoopYieldInjector(ast.NodeTransformer): + """ + Injects ``await __yield()`` at the top of every ``for`` and ``while`` loop body. + + This forces cooperative multitasking so that infinite loops can be cancelled + via ``asyncio.wait_for`` timeout. + """ + + def __init__(self) -> None: + super().__init__() + self._yield_stmt = ast.Expr( + value=ast.Await( + value=ast.Call( + func=ast.Name(id="__yield", ctx=ast.Load()), + args=[], + keywords=[], + ) + ) + ) + + def visit_While(self, node: ast.While) -> ast.While: + self.generic_visit(node) + node.body.insert(0, self._yield_stmt) + return node + + def visit_For(self, node: ast.For) -> ast.For: + self.generic_visit(node) + node.body.insert(0, self._yield_stmt) + return node + + +async def _cooperative_yield() -> None: + """Force the current coroutine to briefly yield to the event loop.""" + await asyncio.sleep(0) diff --git a/cecli/helpers/orchestration/service.py b/cecli/helpers/orchestration/service.py new file mode 100644 index 00000000000..2a907996f85 --- /dev/null +++ b/cecli/helpers/orchestration/service.py @@ -0,0 +1,30 @@ +import weakref + +from cecli.helpers.orchestration.environment import AgentExecutionEnv + + +class OrchestrationService: + """ + Singleton-per-coder registry for AgentExecutionEnv instances. + + Uses weak references (keyed by coder and uuid) so that envs are + automatically cleaned up when their owning coder is garbage collected. + """ + + _instances = weakref.WeakKeyDictionary() + _uuid_index = weakref.WeakValueDictionary() + + @classmethod + def get_instance(cls, coder) -> AgentExecutionEnv: + if coder in cls._instances: + return cls._instances[coder] + + if coder.uuid in cls._uuid_index: + instance = cls._uuid_index[coder.uuid] + cls._instances[coder] = instance + return instance + + instance = AgentExecutionEnv(coder) + cls._instances[coder] = instance + cls._uuid_index[coder.uuid] = instance + return instance diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index cd229fed7e6..9ca0cfe7b90 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -22,7 +22,7 @@ main_system: | ## Core Directives **Act Proactively**: Autonomously use tools to fulfill the request. **Be Decisive**: Do not repeat searches or ask redundant questions. Trust your findings and be confident in your edits. - **Be Efficient**: Use multiple tools each response when exploring. Batch tool calls when the schema allows you to. Respect usage limits while maximizing the utility of each response. + **Be Efficient**: Use multiple tools each request when exploring. Orchestrate tool calls for mutli-step operations. Respect usage limits while maximizing the utility of each request. **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index 92a0d257f62..e62d1158fe5 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -7,7 +7,7 @@ main_system: | ## Core Directives **Act Proactively**: Autonomously use tools to fulfill the request. **Be Decisive**: Do not repeat searches or ask redundant questions. Trust your findings and be confident in your edits. - **Be Efficient**: Use multiple tools each response when exploring. Batch tool calls when the schema allows you to. Respect usage limits while maximizing the utility of each response. + **Be Efficient**: Use multiple tools each request when exploring. Orchestrate tool calls for mutli-step operations. Respect usage limits while maximizing the utility of each request. **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT diff --git a/cecli/tools/__init__.py b/cecli/tools/__init__.py index a9f8be77aa9..3898d4cb733 100644 --- a/cecli/tools/__init__.py +++ b/cecli/tools/__init__.py @@ -1,7 +1,6 @@ # flake8: noqa: F401 # Import tool modules into the cecli.tools namespace -# Import all tool modules from . import ( _yield, command, @@ -17,6 +16,7 @@ git_status, grep, ls, + orchestrate, read_file, resource_manager, thinking, @@ -40,6 +40,7 @@ git_status, grep, ls, + orchestrate, read_file, resource_manager, thinking, diff --git a/cecli/tools/orchestrate.py b/cecli/tools/orchestrate.py new file mode 100644 index 00000000000..da82b487640 --- /dev/null +++ b/cecli/tools/orchestrate.py @@ -0,0 +1,73 @@ +import json +import logging + +from cecli.helpers.orchestration.service import OrchestrationService +from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.helpers import ToolError +from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.validations import ToolValidations + +logger = logging.getLogger(__name__) + + +class Tool(BaseTool): + NORM_NAME = "orchestrate" + TRACK_INVOCATIONS = False + VALIDATIONS = {} + SCHEMA = { + "type": "function", + "function": { + "name": "Orchestrate", + "description": ( + "Execute Python code in a sandboxed environment where you can call " + "other tools programmatically. Use this instead of making many " + "individual tool calls for batch operations. The environment provides " + "`Agent.get_tool(name)` to get tool proxies, `gather(*tasks)` for " + "parallel execution, and `state` for persistent storage across calls." + ), + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": ( + "Python code to execute in the sandbox. See the orchestration " + "context block for available primitives and calling conventions." + ), + }, + }, + "required": ["code"], + }, + }, + } + + @classmethod + async def execute(cls, coder, code, **kwargs): + BaseTool.clear_invocation_cache() + env = OrchestrationService.get_instance(coder) + result = await env.execute(code) + return json.dumps(result) + + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + color_start, color_end = color_markers(coder) + + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + try: + params = ToolValidations.validate_params( + tool_response.function.arguments, cls.VALIDATIONS, cls.SCHEMA + ) + except ToolError: + coder.io.tool_error("Invalid Tool JSON") + return + + code = params.get("code", "") + if code: + coder.io.tool_output("") + coder.io.tool_output(f"{color_start}Code:{color_end}") + for line in code.strip().splitlines(): + coder.io.tool_output(f" {line}") + coder.io.tool_output("") + + tool_footer(coder=coder, tool_response=tool_response, params=params) diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py new file mode 100644 index 00000000000..90dc622d2f3 --- /dev/null +++ b/tests/helpers/test_orchestration.py @@ -0,0 +1,717 @@ +""" +Tests for the orchestration sandbox security boundaries. + +Covers: +- SecurityFilter (imports, dunder attrs, dangerous builtins, global/nonlocal) +- LoopYieldInjector (yield injection into while/for loops) +- AgentExecutionEnv (rejects dangerous code, runs safe code) +- AgentProxy / ToolProxy (local and MCP tool lookup, filter enforcement) +""" + +import ast + +import pytest + +from cecli.helpers.orchestration.environment import ( + AgentExecutionEnv, + AgentProxy, +) +from cecli.helpers.orchestration.security import ( + LoopYieldInjector, + SecurityError, + SecurityFilter, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_mock_coder(): + """Minimal coder mock for AgentExecutionEnv (tool calls not exercised).""" + + class _MockCoder: + registered_tools = {"included": set(), "excluded": set()} + mcp_tools = [] + + return _MockCoder() + + +def _run_security_filter_on(code: str) -> None: + """Run the SecurityFilter on a code snippet. Raises SecurityError if blocked.""" + tree = ast.parse(code, mode="exec") + SecurityFilter().visit(tree) + + +def _run_security_filter_safe(code: str) -> bool: + """Run the SecurityFilter; return True if safe, False if blocked.""" + try: + _run_security_filter_on(code) + return True + except SecurityError: + return False + + +def _get_loop_yield_count(tree: ast.AST) -> int: + """Count how many ``await __yield()`` statements appear in the tree.""" + count = 0 + for node in ast.walk(tree): + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Await): + if ( + isinstance(node.value.value, ast.Call) + and isinstance(node.value.value.func, ast.Name) + and node.value.value.func.id == "__yield" + ): + count += 1 + return count + + +# =================================================================== +# SecurityFilter +# =================================================================== + + +def test_security_filter_blocks_import(): + """SecurityFilter raises SecurityError for ``import os``.""" + assert not _run_security_filter_safe("import os"), "SecurityFilter should block 'import os'" + + +def test_security_filter_blocks_from_import(): + """SecurityFilter raises SecurityError for ``from os import path``.""" + assert not _run_security_filter_safe( + "from os import path" + ), "SecurityFilter should block 'from os import path'" + + +def test_security_filter_blocks_star_import(): + """SecurityFilter raises SecurityError for ``from os import *``.""" + assert not _run_security_filter_safe( + "from os import *" + ), "SecurityFilter should block 'from os import *'" + + +def test_security_filter_blocks_dunder_attribute(): + """SecurityFilter blocks access to ``x.__class__``.""" + assert not _run_security_filter_safe("x.__class__"), "SecurityFilter should block 'x.__class__'" + + +def test_security_filter_blocks_nested_dunder(): + """SecurityFilter blocks access to ``x.y.__class__`` (nested dunder).""" + assert not _run_security_filter_safe( + "x.y.__class__" + ), "SecurityFilter should block 'x.y.__class__'" + + +def test_security_filter_blocks_dunder_method_call(): + """SecurityFilter blocks calls via dunder like ``x.__subclasses__()``.""" + assert not _run_security_filter_safe( + "x.__subclasses__()" + ), "SecurityFilter should block 'x.__subclasses__()'" + + +def test_security_filter_blocks_eval(): + """SecurityFilter blocks ``eval()``.""" + assert not _run_security_filter_safe("eval('1+1')"), "SecurityFilter should block 'eval'" + + +def test_security_filter_blocks_exec(): + """SecurityFilter blocks ``exec()``.""" + assert not _run_security_filter_safe("exec('x=1')"), "SecurityFilter should block 'exec'" + + +def test_security_filter_blocks_open(): + """SecurityFilter blocks ``open()``.""" + assert not _run_security_filter_safe( + "open('/etc/passwd')" + ), "SecurityFilter should block 'open'" + + +def test_security_filter_blocks_import_call(): + """SecurityFilter blocks ``__import__()``.""" + assert not _run_security_filter_safe( + "__import__('os')" + ), "SecurityFilter should block '__import__'" + + +def test_security_filter_blocks_compile(): + """SecurityFilter blocks ``compile()``.""" + assert not _run_security_filter_safe( + "compile('x=1', '', 'exec')" + ), "SecurityFilter should block 'compile'" + + +def test_security_filter_blocks_breakpoint(): + """SecurityFilter blocks ``breakpoint()``.""" + assert not _run_security_filter_safe("breakpoint()"), "SecurityFilter should block 'breakpoint'" + + +def test_security_filter_blocks_globals(): + """SecurityFilter blocks ``globals()``.""" + assert not _run_security_filter_safe("globals()"), "SecurityFilter should block 'globals'" + + +def test_security_filter_blocks_locals(): + """SecurityFilter blocks ``locals()``.""" + assert not _run_security_filter_safe("locals()"), "SecurityFilter should block 'locals'" + + +def test_security_filter_blocks_vars(): + """SecurityFilter blocks ``vars()``.""" + assert not _run_security_filter_safe("vars()"), "SecurityFilter should block 'vars'" + + +def test_security_filter_blocks_getattr(): + """SecurityFilter blocks ``getattr()``.""" + assert not _run_security_filter_safe("getattr(x, 'y')"), "SecurityFilter should block 'getattr'" + + +def test_security_filter_blocks_setattr(): + """SecurityFilter blocks ``setattr()``.""" + assert not _run_security_filter_safe( + "setattr(x, 'y', 1)" + ), "SecurityFilter should block 'setattr'" + + +def test_security_filter_blocks_delattr(): + """SecurityFilter blocks ``delattr()``.""" + assert not _run_security_filter_safe("delattr(x, 'y')"), "SecurityFilter should block 'delattr'" + + +def test_security_filter_blocks_global(): + """SecurityFilter blocks ``global x``.""" + assert not _run_security_filter_safe( + "global x" + ), "SecurityFilter should block 'global' statement" + + +def test_security_filter_blocks_nonlocal(): + """SecurityFilter blocks ``nonlocal x``.""" + assert not _run_security_filter_safe( + "nonlocal x" + ), "SecurityFilter should block 'nonlocal' statement" + + +def test_security_filter_allows_safe_code(): + """SecurityFilter allows ordinary safe code.""" + safe_snippets = [ + "x = 42", + "x + y", + "print('hello')", + "len([1, 2, 3])", + "[i * 2 for i in range(10)]", + "state['key'] = 'value'", + "await gather()", + "sleep(1)", + "isinstance(x, int)", + ] + for snippet in safe_snippets: + assert _run_security_filter_safe( + snippet + ), f"SecurityFilter should allow safe code: {snippet!r}" + + +def test_security_filter_error_message_import(): + """SecurityError message mentions the violation for imports.""" + try: + _run_security_filter_on("import os") + except SecurityError as e: + msg = str(e) + assert "import" in msg.lower(), f"SecurityError should mention 'import', got: {msg}" + else: + pytest.fail("Expected SecurityError") + + +def test_security_filter_error_message_dunder(): + """SecurityError message includes the blocked attribute name for dunder.""" + try: + _run_security_filter_on("x.__foo__") + except SecurityError as e: + msg = str(e) + assert "__foo__" in msg, f"SecurityError should include '__foo__', got: {msg}" + else: + pytest.fail("Expected SecurityError") + + +def test_security_filter_error_message_builtin(): + """SecurityError message mentions the blocked builtin name.""" + try: + _run_security_filter_on("eval('1+1')") + except SecurityError as e: + msg = str(e) + assert "eval" in msg, f"SecurityError should mention 'eval', got: {msg}" + else: + pytest.fail("Expected SecurityError") + + +# =================================================================== +# LoopYieldInjector +# =================================================================== + + +def test_loop_yield_injector_injects_into_while(): + """LoopYieldInjector injects ``await __yield()`` at top of while loop body.""" + tree = ast.parse( + """ +while True: + x = 1 +""", + mode="exec", + ) + original_count = _get_loop_yield_count(tree) + assert original_count == 0, "No yields before injection" + + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 1, f"Expected 1 yield in while loop, got {yield_count}" + + +def test_loop_yield_injector_injects_into_for(): + """LoopYieldInjector injects ``await __yield()`` at top of for loop body.""" + tree = ast.parse( + """ +for i in range(10): + print(i) +""", + mode="exec", + ) + original_count = _get_loop_yield_count(tree) + assert original_count == 0, "No yields before injection" + + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 1, f"Expected 1 yield in for loop, got {yield_count}" + + +def test_loop_yield_injector_multiple_loops(): + """LoopYieldInjector injects yields into every loop in nested structures.""" + tree = ast.parse( + """ +for a in items: + while b: + for c in nested: + pass +""", + mode="exec", + ) + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 3, f"Expected 3 yields (for + while + for), got {yield_count}" + + +def test_loop_yield_injector_no_loops(): + """LoopYieldInjector does not inject yields if there are no loops.""" + tree = ast.parse("x = 42\nprint(x)", mode="exec") + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 0, f"Expected 0 yields (no loops), got {yield_count}" + + +def test_loop_yield_injector_first_in_body(): + """The injected yield is the very first statement in the loop body.""" + tree = ast.parse( + """ +while flag: + print('hello') + x += 1 +""", + mode="exec", + ) + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + while_nodes = [n for n in ast.walk(modified) if isinstance(n, ast.While)] + assert len(while_nodes) == 1 + first_stmt = while_nodes[0].body[0] + assert isinstance(first_stmt, ast.Expr) + assert isinstance(first_stmt.value, ast.Await) + assert isinstance(first_stmt.value.value, ast.Call) + assert first_stmt.value.value.func.id == "__yield" + + +# =================================================================== +# AgentExecutionEnv integration tests +# =================================================================== + + +def _make_env() -> AgentExecutionEnv: + return AgentExecutionEnv(_make_mock_coder()) + + +@pytest.mark.asyncio +async def test_env_rejects_import(): + """AgentExecutionEnv.execute() rejects code with imports.""" + env = _make_env() + result = await env.execute("import os") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for import, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_json_dumps(): + """AgentExecutionEnv.execute() provides ``json.dumps`` in globals.""" + env = _make_env() + result = await env.execute("json.dumps({'key': 'value'})") + assert result["results"] == '{"key": "value"}', f"Expected JSON output, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_dunder(): + """AgentExecutionEnv.execute() rejects code with dunder access.""" + env = _make_env() + result = await env.execute("x = 1\nx.__class__") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for dunder, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_eval(): + """AgentExecutionEnv.execute() rejects code calling eval().""" + env = _make_env() + result = await env.execute("eval('1+1')") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for eval, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_open(): + """AgentExecutionEnv.execute() rejects code calling open().""" + env = _make_env() + result = await env.execute("open('/dev/null')") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for open, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_global_stmt(): + """AgentExecutionEnv.execute() rejects code with global statement.""" + env = _make_env() + result = await env.execute("global x") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for global, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_runs_simple_expression(): + """AgentExecutionEnv.execute() runs a simple arithmetic expression.""" + env = _make_env() + result = await env.execute("42") + assert result["results"] == "42", f"Expected '42', got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_runs_print(): + """AgentExecutionEnv.execute() captures print output.""" + env = _make_env() + result = await env.execute("print('hello world')") + assert result["results"] == "hello world", f"Expected 'hello world', got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_state_persistence(): + """AgentExecutionEnv.state persists across execute() calls.""" + env = _make_env() + + result1 = await env.execute("state['key'] = 'value1'\nstate['key']") + assert result1["results"] == "value1", f"Expected 'value1', got: {result1!r}" + + result2 = await env.execute("state['key']") + assert result2["results"] == "value1", f"Expected 'value1' (persisted), got: {result2!r}" + + +@pytest.mark.asyncio +async def test_env_runs_list_comprehension(): + """AgentExecutionEnv.execute() runs a list comprehension.""" + env = _make_env() + result = await env.execute("[i * 2 for i in range(5)]") + assert ( + result["results"] == "[0, 2, 4, 6, 8]" + ), f"Expected list comprehension result, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_returns_last_expression(): + """AgentExecutionEnv returns the value of the last expression.""" + env = _make_env() + result = await env.execute("x = 10\ny = 20\nx + y") + assert result["results"] == "30", f"Expected '30' from last expression, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_print_and_expression(): + """AgentExecutionEnv returns both print output and last expression.""" + env = _make_env() + result = await env.execute("print('computed')\n42") + assert "computed" in result["results"] + assert "42" in result["results"] + + +@pytest.mark.asyncio +async def test_env_empty_code(): + """AgentExecutionEnv returns empty string for empty code.""" + env = _make_env() + result = await env.execute("") + assert result["results"] == "", f"Expected empty string for empty code, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_whitespace_code(): + """AgentExecutionEnv returns empty string for whitespace-only code.""" + env = _make_env() + result = await env.execute(" \n\n ") + assert result["results"] == "", f"Expected empty string for whitespace code, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_sleep_works(): + """AgentExecutionEnv's sleep primitive works.""" + env = _make_env() + result = await env.execute( + "sleep(0.01)\nprint('done')", + ) + assert result["results"] == "done", f"Expected 'done' after sleep, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_gather_works(): + """AgentExecutionEnv's gather primitive works.""" + env = _make_env() + result = await env.execute("await gather()") + assert result["results"] == "[]", f"Expected '[]' from gather(), got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_handles_syntax_error(): + """AgentExecutionEnv returns a syntax error message for invalid code.""" + env = _make_env() + result = await env.execute("x = ") + assert "Syntax Error" in result["results"], f"Expected Syntax Error, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_safe_builtins_available(): + """AgentExecutionEnv provides safe builtins (len, range, int, str, etc.).""" + env = _make_env() + + result = await env.execute("len([1, 2, 3])") + assert result["results"] == "3", f"Expected '3', got: {result!r}" + + result = await env.execute("str(42)") + assert result["results"] == "42", f"Expected '42', got: {result!r}" + + result = await env.execute("list(range(3))") + assert result["results"] == "[0, 1, 2]", f"Expected '[0, 1, 2]', got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_exception_handling(): + """AgentExecutionEnv handles runtime exceptions gracefully.""" + env = _make_env() + result = await env.execute("1 / 0") + assert ( + "Execution Error" in result["results"] and "ZeroDivisionError" in result["results"] + ), f"Expected Execution Error with ZeroDivisionError, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_runs_function_def(): + """AgentExecutionEnv can define and call functions.""" + env = _make_env() + result = await env.execute(""" +def add(a, b): + return a + b +add(2, 3) +""") + assert result["results"] == "5", f"Expected '5' from function call, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_mixed_print_and_return(): + """AgentExecutionEnv returns print output followed by expression value.""" + env = _make_env() + result = await env.execute(""" +print("start") +result = 99 +result +""") + assert ( + "start" in result["results"] and "99" in result["results"] + ), f"Expected both 'start' and '99', got: {result!r}" + + +# =================================================================== +@pytest.mark.asyncio +async def test_env_shared_state_writes(): + """AgentExecutionEnv shared_state is writable and shows in snapshot.""" + env = _make_env() + result = await env.execute("shared_state['findings'] = 'done'") + svars = {v["name"]: v for v in result["state_variables"]} + assert "findings" in svars + assert svars["findings"]["type"] == "str" + assert svars["findings"]["size"] == 4 + assert svars["findings"]["scope"] == "shared" + assert svars["findings"]["modified"] is True + + +@pytest.mark.asyncio +async def test_env_shared_state_visible_across_envs(): + """AgentExecutionEnv shared_state is visible across separate env instances.""" + AgentExecutionEnv._shared_state.clear() + env1 = _make_env() + await env1.execute("shared_state['msg'] = 'hello'") + env2 = _make_env() + svars = {v["name"]: v for v in env2._state_snapshot()} + assert "msg" in svars + assert svars["msg"]["type"] == "str" + assert svars["msg"]["scope"] == "shared" + assert svars["msg"]["modified"] is False # env2 didn't write it + + +@pytest.mark.asyncio +async def test_env_state_snapshot_includes_scope_field(): + """state_variables entries have scope field: 'local' or 'shared'.""" + env = _make_env() + result = await env.execute("state['local_key'] = 1") + for entry in result["state_variables"]: + assert entry["scope"] in ("local", "shared") + local_vars = [e for e in result["state_variables"] if e["scope"] == "local"] + # shared_vars = [e for e in result["state_variables"] if e["scope"] == "shared"] + assert any(e["name"] == "local_key" for e in local_vars) + + +# AgentProxy / ToolProxy — tool lookup and dispatch +# =================================================================== + + +def _make_mock_coder_with_mcp(): + """Minimal coder mock with MCP tool support for proxy tests.""" + + class _MockMcpServer: + name = "MockServer" + + class _MockMcpManager: + def __iter__(self): + return iter([self._server]) + + def __init__(self): + self._server = _MockMcpServer() + + class _MockCoder: + registered_tools = {"included": set(), "excluded": set()} + mcp_tools = [ + ( + "MockServer", + [{"type": "function", "function": {"name": "MockTool"}}], + ), + ] + mcp_manager = _MockMcpManager() + + async def _execute_mcp_tool(self, server, tool_name, params): + return f"mcp-result: {tool_name} called with {params}" + + return _MockCoder() + + +def test_agent_proxy_local_tool_lookup(): + """AgentProxy.get_tool resolves local tools via ToolRegistry.""" + proxy = AgentProxy(_make_mock_coder()) + tool = proxy.get_tool("ReadFile") + assert tool._tool_module is not None, "Local tool should have _tool_module" + assert tool._mcp_server is None, "Local tool should not have _mcp_server" + + +def test_agent_proxy_unknown_tool_raises(): + """AgentProxy.get_tool raises ValueError for unknown tool names.""" + proxy = AgentProxy(_make_mock_coder()) + with pytest.raises(ValueError, match="Unknown tool"): + proxy.get_tool("NonExistentToolXYZ") + + +def test_agent_proxy_mcp_tool_lookup(): + """AgentProxy.get_tool resolves MCP tools with ServerName--ToolName prefix.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + tool = proxy.get_tool("MockServer--MockTool") + assert tool._tool_module is None, "MCP tool should not have _tool_module" + assert tool._mcp_server is not None, "MCP tool should have _mcp_server" + assert tool._mcp_tool_name == "MockTool" + + +@pytest.mark.asyncio +async def test_tool_proxy_mcp_dispatch(): + """ToolProxy.call dispatches to coder._execute_mcp_tool for MCP tools.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + tool = proxy.get_tool("MockServer--MockTool") + result = await tool.call(param1="value1") + assert "mcp-result" in result + assert "MockTool" in result + assert "param1" in result + + +def test_agent_proxy_includelist_filters(): + """AgentProxy.get_tool respects coder.registered_tools includelist.""" + coder = _make_mock_coder() + coder.registered_tools["included"] = {"yield"} + proxy = AgentProxy(coder) + # In list — allowed + tool = proxy.get_tool("Yield") + assert tool._tool_module is not None + # Not in list — blocked + with pytest.raises(ValueError, match="not in the allowed tools list"): + proxy.get_tool("ReadFile") + + +def test_agent_proxy_excludelist_filters(): + """AgentProxy.get_tool respects coder.registered_tools excludelist.""" + coder = _make_mock_coder() + coder.registered_tools["excluded"] = {"readfile"} + proxy = AgentProxy(coder) + with pytest.raises(ValueError, match="has been excluded"): + proxy.get_tool("ReadFile") + + +def test_agent_proxy_mcp_with_bare_name(): + """AgentProxy.get_tool finds MCP tools even with bare (unprefixed) name.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + tool = proxy.get_tool("mocktool") + assert tool._mcp_server is not None + assert tool._mcp_tool_name == "MockTool" + + +def test_agent_proxy_find_mcp_server_no_manager(): + """_find_mcp_server returns None when coder has no mcp_manager.""" + proxy = AgentProxy(_make_mock_coder()) + result = proxy._find_mcp_server("AnyServer", "") + assert result is None + + +def test_agent_proxy_local_priority(): + """AgentProxy.get_tool prefers local tools over MCP tools with same name.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + # "ReadFile" exists as local tool — should be found first + tool = proxy.get_tool("ReadFile") + assert tool._tool_module is not None, "Local tool must be preferred" + assert tool._mcp_server is None + + +def test_agent_proxy_local_prefix_tool(): + """AgentProxy.get_tool handles Local--ToolName by stripping prefix.""" + proxy = AgentProxy(_make_mock_coder()) + tool = proxy.get_tool("Local--ReadFile") + assert tool._tool_module is not None, "Local--ReadFile should resolve via ToolRegistry" + assert tool._mcp_server is None From 6b168dbea439b768a6ca4aeaf98155c2a62179ed Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 01:09:55 -0400 Subject: [PATCH 03/68] Base 1024 Corpus --- cecli/helpers/hashpos/hashpos.py | 252 ++++++++++++++++--------------- scripts/optimize_tokens.sh | 114 +++++++++----- 2 files changed, 213 insertions(+), 153 deletions(-) diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index 9caab1611cb..00c47f7a6a4 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -4,142 +4,150 @@ class HashPos: - # ------------------------------------------------------------------------- - # TOKEN-OPTIMIZED PREFIX-FREE ENCODING SETUP - # ------------------------------------------------------------------------- - # flake8: noqa - # fmt: off - _TOKEN_LIST = [ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', #noqa - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', #noqa - 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'GA', 'GB', 'GC', 'GD', 'Ge', 'Gl', #noqa - 'Go', 'Gr', 'Gu', 'HA', 'HD', 'HE', 'HO', 'HP', 'HR', 'HT', 'Ha', 'He', 'Hi', 'Hy', 'IC', 'ID', #noqa - 'IE', 'IF', 'II', 'IL', 'IM', 'IN', 'IO', 'IP', 'IR', 'IS', 'IT', 'IV', 'IX', 'Id', 'If', 'Il', #noqa - 'Im', 'In', 'Ir', 'Is', 'It', 'JO', 'JS', 'Jo', 'Ke', 'Kn', 'LA', 'LE', 'LI', 'LL', 'LO', 'LP', #noqa - 'La', 'Le', 'Li', 'Lo', 'MA', 'MB', 'MC', 'MD', 'ME', 'MI', 'ML', 'MO', 'MP', 'MR', 'MS', 'MT', #noqa - 'MY', 'Ma', 'Mc', 'Me', 'Mi', 'Mo', 'Mr', 'Ms', 'My', 'NA', 'NC', 'NE', 'NL', 'NO', 'NS', 'NT', #noqa - 'NV', 'NY', 'Na', 'Ne', 'No', 'OB', 'OF', 'OK', 'ON', 'OP', 'OR', 'OS', 'Ob', 'Of', 'Oh', 'Ok', #noqa - 'On', 'Op', 'Or', 'Os', 'PA', 'PC', 'PD', 'PE', 'PG', 'PH', 'PI', 'PK', 'PL', 'PM', 'PO', 'PP', #noqa - 'PR', 'PS', 'PT', 'Pa', 'Pe', 'Ph', 'Pi', 'Pl', 'Po', 'Pr', 'Py', 'Qt', 'Qu', 'RC', 'RE', 'RF', #noqa - 'RO', 'RT', 'Ra', 'Re', 'Ro', 'SA', 'SB', 'SC', 'SD', 'SE', 'SF', 'SH', 'SI', 'SK', 'SL', 'SM', #noqa - 'SN', 'SO', 'SP', 'SR', 'SS', 'ST', 'SU', 'SW', 'SY', 'Sc', 'Se', 'Sh', 'Si', 'Sk', 'Sl', 'Sm', #noqa - 'Sn', 'So', 'Sp', 'St', 'Su', 'Sw', 'TD', 'TE', 'TF', 'TH', 'TL', 'TO', 'TR', 'TV', 'TX', 'Te', #noqa - 'Th', 'To', 'Tr', 'Tw', 'Ty', 'UE', 'UI', 'UK', 'UN', 'UP', 'US', 'UV', 'Un', 'Up', 'Ur', 'Us', #noqa - 'VA', 'VI', 'VM', 'VT', 'Va', 'Ve', 'WA', 'WE', 'WH', 'WM', 'WR', 'We', 'Wh', 'Wr', 'XX', 'Ye', #noqa - ] - # fmt: on - # flake8: qa - - # Quick lookups for the 256 bytes - ENCODE_MAP = {i: token for i, token in enumerate(_TOKEN_LIST)} - DECODE_MAP = {token: i for i, token in enumerate(_TOKEN_LIST)} - - # Because all 2-char tokens start with uppercase G-Y, which are never used as standalone - # 1-char tokens, we can cleanly split them. - _PREFIX_CHARS = set("GHIJKLMNOPQRSTUVWXY") - - # Regex building blocks dynamically matching the list logic. - # Single chars are 0-9, A-F, and a-z. Two-char tokens start with G-Y followed by any letter. - _BYTE_REGEX = r"(?:[0-9a-zA-F]|[GHIJKLMNOPQRSTUVWXY][A-Za-z])" - # Regex for HashPos format: {3 encoded bytes}:: - HASH_PREFIX_RE = re.compile(rf"^({_BYTE_REGEX}{{3}})::") - # Regex for normalization: 3 encoded bytes optionally followed by '::' - NORMALIZE_RE = re.compile(rf"^({_BYTE_REGEX}{{3}})(?:::.*)?$") - # Regex for a raw 3-byte encoded fragment - FRAGMENT_RE = re.compile(rf"^{_BYTE_REGEX}{{3}}$") + # 1024-character Base1024 corpus + B1024 = ( + "0123456789ABCDEFGHIJ" + "KLMNOPQRSTUVWXYZabcd" + "efghijklmnopqrstuvwx" + "yz¡£§©«®°±·»¿×ßæðøĐđ" + "ıłœəαβγδεηθικλμνοπρς" + "στυφχωАБВГДЕЗИКЛМНОП" + "РСТУФЦЧЭЯабвгдежзикл" + "мнопрстуфхцчшщъыьэюя" + "іאבדהוחילמנערשת،ابةت" + "ثجحخدذرزسشصضطظعغفقكل" + "منهوىيپکگیकतनपमरलसहন" + "রกขคงจชณดตถทนบปผพมยร" + "ลวสหอะาเแใไ‐–—―‘’“”„" + "†•′※€←→−─│█■●★☆♥♪、。《" + "》「」『』【】〜あいうえおかきくけこさし" + "すせそたちっつてとなにのはまみめもやよら" + "りるれろわをんアィイウェエオカキクコサシ" + "スセタチッテトナニフマムメャュョラリルレ" + "ロン・ー一万三上下不与专业东两个中为主么" + "义之也书了事二于五些交产享京人亿今介从他" + "付代以们件价任份企会传但位体何余作你使例" + "供価保信修倍值停像元先入全公共关其具内円" + "册再写出击分列则初利别到制前力功加务动動" + "包化北区十午华单南即历原去县参及友反发取" + "变口只可台右号司合同名后向否含听启告员周" + "命和品哈商問器四回因国图土在地场型城基報" + "場增声处备复外多大天失头女好如始子字存学" + "安完定实客家容密对导将小少尔就局展山岁州" + "工左已市布常平年并广序库应店度建开式引张" + "当录形影径待後得微心必志态思性总息您情意" + "感成我或户所手打技投报拉持指按换据排接推" + "提播支收改放政效数整文料断新方族无日时明" + "易星是時景更最月有服期木未本机权束条来板" + "构果查标样格案模次款止正此步歳段每比民気" + "水求江没治法注活流海消清游源火点無然片版" + "物特率环现球理生用由电男画界番登的目直相" + "省看県真知码确示社票私种科秒称移程税空立" + "站章端笑符第等简算管米类系素索约级线组经" + "结给统编网置美老考者而联能自至色节英藏行" + "表装西要見见规视角解言計記話読计认议记论" + "设证评试话该语误说请读调象责败货费资起超" + "路身车转载辑输达过运近还这进连述退送选通" + "速造連道部都配释里重量金错长開間関门闭问" + "间队阳陆限院除雅集雷需非面音页项题首验高" + "黑가간개거게결경고공과구그글기나내는능니" + "다당대도동되된드든들디라래러력로록료류른" + "를름리만메면명목문미버번보복부분비사산상" + "색생서성세션소수스습시식신아야어에여열오" + "와요용우운원위으은을음의이인일임입자작장" + "재적전정제져조주지진째체출치크태터턴트하" + "한할함해호화환회�ª²³´µ¹º¼½ÀÁ" + "ÂÃÄÇ" + ) + + # We escape every individual character just to be completely safe from regex metacharacters + _B1024_REGEX_SET = "".join(re.escape(c) for c in B1024) + + # Regex pattern for HashPos format: {3-char-hash}:: + HASH_PREFIX_RE = re.compile(rf"^([{_B1024_REGEX_SET}]{{3}})::") + # Regex for normalization: 3 hash chars optionally followed by '::' + NORMALIZE_RE = re.compile(rf"^([{_B1024_REGEX_SET}]{{3}})(?:)?::") + # Regex for a raw 3-character fragment + FRAGMENT_RE = re.compile(rf"^[{_B1024_REGEX_SET}]{{3}}$") def __init__(self, source_text: str = ""): self.lines = source_text.splitlines() self.total = len(self.lines) - def _get_region_val(self, line_idx: int) -> int: + def _get_line_hash(self, text: str) -> int: """ - Maps the line to one of 16 proportional vertical buckets (4 bits). - This acts as a binary space partition: - - bit 3 is top/bottom half - - bit 2 is top/bottom quarter - - bit 1 is top/bottom eighth - - bit 0 is top/bottom sixteenth + Creates a 20-bit digest of the current line's text. """ - if self.total == 0: - return 0 + return xxhash.xxh3_64_intdigest(text.encode("utf-8")) & 0xFFFFF - # Calculate which 16th of the file the line falls into - region = (line_idx * 16) // self.total - - # Clamp to 15 to handle edge cases safely - return min(region, 15) - - def _get_neighborhood_hash(self, line_idx: int) -> int: + def _get_file_fraction(self, line_idx: int) -> int: """ - Creates a 20-bit digest using the current line and the 2 lines - before and after it. + Returns which 16th of the file the line is in (4 bits: 0-15). """ - start = max(0, line_idx - 2) - end = min(self.total, line_idx + 3) + return line_idx % 16 - context_window = "\n".join(self.lines[start:end]) - full_hash = xxhash.xxh3_64_intdigest(context_window.encode("utf-8")) + def _get_adjacent_hash(self, line_idx: int) -> int: + """Creates a 6-bit digest of the two lines before and two lines after (skipping current line).""" + start_idx = max(0, line_idx - 3) + end_idx = min(self.total, line_idx + 4) - # Isolate exactly 20 bits - return full_hash & 0xFFFFF + # Concatenate up to 2 lines before and up to 2 lines after + adjacent_lines = self.lines[start_idx:line_idx] + self.lines[line_idx + 1 : end_idx] + + context = "\n".join(adjacent_lines) + return xxhash.xxh3_64_intdigest(context.encode("utf-8")) & 0x3F + + def generate_private_id(self, text: str) -> str: + """ + Generates a fast 12-bit (3 hex chars) hash based purely on the line text. + """ + bits = xxhash.xxh3_64_intdigest(text.encode("utf-8")) & 0xFFF + return f"{bits:03x}" def generate_public_id(self, text: str, line_idx: int) -> str: """ - Generates a 3-to-6 char ID using the token-optimized prefix-free encoding. - Layout: [20-bit Neighborhood Hash] [4-bit Region] = 24 bits total. + Generates a 3-character Base1024 ID. + Layout: [20-bit Line Hash] [4-bit File Fraction] [6-bit Adjacent Hash] = 30 bits total. + Each Base1024 character holds 10 bits. """ - neighborhood_hash = self._get_neighborhood_hash(line_idx) - region_val = self._get_region_val(line_idx) + line_hash = self._get_line_hash(text) + fraction = self._get_file_fraction(line_idx) + adj_hash = self._get_adjacent_hash(line_idx) - # Pack the 24-bit integer - packed = (neighborhood_hash << 4) | region_val + # Pack the 30-bit integer + packed = (line_hash << 10) | (fraction << 6) | adj_hash - # Encode 3 bytes using the prefix-free map res = "" for _ in range(3): - byte_segment = packed % 256 - res += self.ENCODE_MAP[byte_segment] - packed //= 256 - + # Extract 10 bits at a time using modulo 1024 + res += self.B1024[packed % 1024] + packed //= 1024 return res - def unpack_public_id(self, public_id: str) -> tuple[int, int]: + def unpack_public_id(self, public_id: str) -> tuple[int, int, int]: """ - Reverses the Public ID back into its (Neighborhood Hash, Region Value) values. - Reads the prefix-free string left-to-right to unambiguously decode the bytes. + Reverses the Public ID back into its (Line Hash, Fraction, Adjacent Hash) values. """ packed = 0 - byte_shift = 0 - i = 0 + for i, char in enumerate(public_id): + # Each character restores 10 bits + packed |= self.B1024.index(char) << (10 * i) - while i < len(public_id): - char = public_id[i] + # Extract bits based on layout + line_hash = (packed >> 10) & 0xFFFFF + fraction = (packed >> 6) & 0xF + adj_hash = packed & 0x3F - # The G-Y characters explicitly signal a two-character sequence - if char in self._PREFIX_CHARS: - seq = public_id[i : i + 2] - i += 2 - else: - seq = char - i += 1 - - byte_val = self.DECODE_MAP[seq] - packed |= byte_val << byte_shift - byte_shift += 8 - - # Extract the 20-bit hash (shift right by 4, mask 0xFFFFF) - neighborhood_hash = (packed >> 4) & 0xFFFFF - - # Extract the 4-bit region from the lowest bits - region_val = packed & 0xF + return line_hash, fraction, adj_hash - return neighborhood_hash, region_val - - def format_content(self, start_line: int = 1) -> str: + def format_content(self, use_private_ids: bool = False, start_line: int = 1) -> str: formatted_lines = [] for i, line in enumerate(self.lines): - prefix = self.generate_public_id(line, i) + prefix = ( + self.generate_private_id(line) + if use_private_ids + else self.generate_public_id(line, i) + ) if line.strip(): formatted_lines.append(f"{prefix}::{line}") else: @@ -148,29 +156,35 @@ def format_content(self, start_line: int = 1) -> str: return "\n".join(formatted_lines) def resolve_to_lines(self, public_id: str, start_line: int = 1) -> list[int]: - target_hash, target_region = self.unpack_public_id(public_id) + target_line_hash, target_fraction, target_adj_hash = self.unpack_public_id(public_id) matches = [] - # Find all lines whose neighborhood hash matches our target + # 1. Primary Filter: Find all lines whose 20-bit line content hash matches for i, line in enumerate(self.lines): - if self._get_neighborhood_hash(i) == target_hash: + if self._get_line_hash(line) == target_line_hash: matches.append(i) if not matches: return [] - # If perfectly unique, return it immediately + # If perfectly unique (highly likely given 20 bits of line entropy), return immediately if len(matches) == 1: return matches - # Distance Heuristic: If multiple matches exist (e.g. repeated code blocks), - # prioritize the one whose current binary region is closest to the target region. - def region_distance(idx: int) -> int: - current_region = self._get_region_val(idx) - # Linear distance because proportional regions don't wrap around - return abs(current_region - target_region) + # 2. Tie-Breaking Heuristic: + # If multiple identical lines exist, score them based on adjacency match and fraction distance. + def score_match(idx: int) -> tuple[int, int]: + # Adjacency match: 0 means exact match, 1 means mismatch (we want lower scores) + adj_score = 0 if self._get_adjacent_hash(idx) == target_adj_hash else 1 + + # Fraction distance: Calculate how many 16ths away we are + current_fraction = self._get_file_fraction(idx) + fraction_dist = abs(current_fraction - target_fraction) + + # Sort by adjacency match first, then by closest spatial block + return (adj_score, fraction_dist) - matches.sort(key=region_distance) + matches.sort(key=score_match) return matches @@ -219,7 +233,7 @@ def extract_prefix(line: str) -> str: @staticmethod def normalize(hashpos_str: str) -> str: """ - Normalize a HashPos string to the exact matched prefix fragment. + Normalize a HashPos string to the 3-character hash fragment. """ if hashpos_str is None: raise ValueError("HashPos string cannot be None") @@ -233,5 +247,5 @@ def normalize(hashpos_str: str) -> str: raise ValueError( f"Invalid HashPos format '{hashpos_str}'. " - r"Expected a valid content ID followed by `::`" + r"Expected a 3-character string from the Base1024 character set." ) diff --git a/scripts/optimize_tokens.sh b/scripts/optimize_tokens.sh index f7157d0a759..91151ce0605 100755 --- a/scripts/optimize_tokens.sh +++ b/scripts/optimize_tokens.sh @@ -4,57 +4,103 @@ echo "Calculating Ideal Tokens For Tokenization" # Write the Python script to a temporary file cat << 'EOF' > /tmp/table_optimizer.py -import string import pprint import tiktoken +import unicodedata from transformers import AutoTokenizer -def generate_universal_prefix_free_tokens(target_count=256): +def find_anthropic_safe_tokens_with_fallback(target_count=1024): print("Loading tokenizers... (This will download vocab files to your cache)") - # 1. OpenAI Baseline - enc_openai = tiktoken.get_encoding("cl100k_base") + enc_cl100k = tiktoken.get_encoding("cl100k_base") + enc_o200k = tiktoken.get_encoding("o200k_base") - # 2. Stable Open-Weight Tokenizers (Publicly Accessible) tokenizers = { - "mistral": AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1"), "qwen": AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B"), - "deepseek": AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-6.7b-instruct") + "mistral_nemo": AutoTokenizer.from_pretrained("mistralai/Mistral-Nemo-Base-2407"), + "phi4": AutoTokenizer.from_pretrained("microsoft/Phi-4-mini-instruct", trust_remote_code=True), + "glm4": AutoTokenizer.from_pretrained("THUDM/glm-4-9b-chat", trust_remote_code=True), + "deepseek_v3": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V3", trust_remote_code=True), } - - # Base36: 0-9, a-z (36 characters total) - base36 = list(string.digits + string.ascii_lowercase) - - direct_chars = base36.copy() - prefix_pairs = [] - - def is_single_token_everywhere(text): - if len(enc_openai.encode(text)) != 1: - return False - for name, enc in tokenizers.items(): - # add_special_tokens=False is critical for correct subword counting - if len(enc.encode(text, add_special_tokens=False)) != 1: - return False - return True - print("Calculating intersection of all tokenizers...") + print("Extracting candidates into Primary and Secondary (Fallback) pools...") + primary_candidates = set() + secondary_candidates = set() - while len(direct_chars) + len(prefix_pairs) < target_count: - if not direct_chars: - raise ValueError("Stricter rules exhausted the alphabet. Please switch to Base62 (A-Z, a-z, 0-9).") + for token_id in range(enc_o200k.n_vocab): + try: + token_bytes = enc_o200k.decode_single_token_bytes(token_id) + text = token_bytes.decode('utf-8') - prefix = direct_chars.pop() - for second_char in base36: - seq = prefix + second_char - if is_single_token_everywhere(seq): - prefix_pairs.append(seq) + if len(text) == 1 and text.isprintable(): + char_code = ord(text) + + # 1. ASCII must be strictly alphanumeric + if char_code <= 0x007F and not text.isalnum(): + continue + + # 2. PREVENT COMBINING CHARACTER COLLAPSE + # Drops all attaching Marks (Mn, Mc, Me) and invisible format Controls (Cf, Cc) + category = unicodedata.category(text) + if category.startswith('M') or category.startswith('C'): + continue + + # 3. DECOMPOSITION ROUTING + if unicodedata.decomposition(text): + # Has hidden structural parts -> send to Fallback Pool + secondary_candidates.add(text) + else: + # Pure, unbreakable character -> send to Primary Pool + primary_candidates.add(text) + + except (KeyError, UnicodeDecodeError): + pass + + # Sort to sequentially process alphabets from standard to complex + primary_sorted = sorted(list(primary_candidates), key=ord) + secondary_sorted = sorted(list(secondary_candidates), key=ord) + + print(f"Extracted {len(primary_sorted)} Primary and {len(secondary_sorted)} Secondary candidates.") + + shared_tokens = [] + + # Helper function to process a specific pool + def process_pool(candidate_list, pool_name): + print(f"\n--- Scanning {pool_name} Pool ---") + for text in candidate_list: + if len(enc_cl100k.encode(text)) != 1: + continue + + is_shared = True + for enc in tokenizers.values(): + if len(enc.encode(text, add_special_tokens=False)) != 1: + is_shared = False + break + + if is_shared: + shared_tokens.append(text) + if len(shared_tokens) % 64 == 0: + print(f"Secured {len(shared_tokens)} / {target_count} universal tokens...") - return (direct_chars + prefix_pairs)[:target_count] + if len(shared_tokens) == target_count: + break + + # 1. Exhaust the Primary Pool first + process_pool(primary_sorted, "PRIMARY (Strictly Safe)") + + # 2. If we fell short (e.g. 1009/1024), dip into the Secondary Pool + if len(shared_tokens) < target_count: + print(f"\nTarget not met ({len(shared_tokens)}/{target_count}). Dipping into SECONDARY fallback pool...") + process_pool(secondary_sorted, "SECONDARY (Decomposed)") + + return shared_tokens if __name__ == "__main__": try: - optimal_tokens = generate_universal_prefix_free_tokens(256) - print("\nBASE256_UNIVERSAL_CODES = ", end="") + optimal_tokens = find_anthropic_safe_tokens_with_fallback(1024) + print("\n" + "="*50) + print(f"FOUND {len(optimal_tokens)} UNIVERSAL TOKENS (SORTED BY UNICODE)") + print("="*50) pprint.pprint(optimal_tokens, compact=True, width=100) except Exception as e: print(f"Error: {e}") From b6f397e634eb56f20692f0461699beda703a8618 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 01:11:51 -0400 Subject: [PATCH 04/68] JSON encode all tool calls --- cecli/coders/agent_coder.py | 12 ++- cecli/helpers/orchestration/environment.py | 66 +++++++++++++- cecli/tools/_yield.py | 18 +++- cecli/tools/command.py | 73 +++++++++++---- cecli/tools/command_interactive.py | 15 ++- cecli/tools/delegate.py | 20 +++- cecli/tools/edit_file.py | 31 +++---- cecli/tools/explore_code.py | 28 +++--- cecli/tools/git_branch.py | 21 +++-- cecli/tools/git_diff.py | 15 ++- cecli/tools/git_log.py | 12 ++- cecli/tools/git_remote.py | 15 ++- cecli/tools/git_show.py | 13 ++- cecli/tools/git_status.py | 13 ++- cecli/tools/grep.py | 23 +++-- cecli/tools/ls.py | 25 +++-- cecli/tools/orchestrate.py | 6 +- cecli/tools/read_file.py | 87 ++++++++---------- cecli/tools/resource_manager.py | 27 +++--- cecli/tools/thinking.py | 5 +- cecli/tools/undo_change.py | 22 +++-- cecli/tools/update_todo_list.py | 26 ++++-- cecli/tools/utils/base_tool.py | 11 ++- cecli/tools/utils/responses.py | 101 +++++++++++++++++++++ scripts/get_hashline.py | 2 + 25 files changed, 503 insertions(+), 184 deletions(-) create mode 100644 cecli/tools/utils/responses.py diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index eeab4ea1da9..4813065d068 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -298,6 +298,7 @@ async def initialize_mcp_tools(self): async def _execute_mcp_tool(self, server, tool_name, params): """Helper to execute a single MCP tool call, created from legacy format.""" + from cecli.tools.utils.responses import ToolResponse async def _exec_async(): function_dict = {"name": tool_name, "arguments": json.dumps(params)} @@ -337,9 +338,14 @@ async def _exec_async(): result, interrupted = await interruptible(_exec_async(), self.interrupt_event) + response = ToolResponse(tool_name) if interrupted: - return "Tool execution interrupted by user." - return result + response.append_error("Tool execution interrupted by user.") + elif isinstance(result, str) and result.startswith("Error executing tool call"): + response.append_error(result) + else: + response.append_result(result) + return response def _calculate_context_block_tokens(self, force=False): """ @@ -887,7 +893,7 @@ async def _execute_mcp_tools(self, server, tool_calls): { "role": "tool", "tool_call_id": tool_call.id, - "content": result, + "content": str(result), } ) return tool_responses diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index e7f588e64e8..a427758de04 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -97,7 +97,68 @@ def __init__( self._coder = coder self._tool_module = tool_module self._mcp_server = mcp_server - self._mcp_tool_name = mcp_tool_name + + async def __call__(self, *args: Any, **kwargs: Any): + """Make the proxy directly callable. + + Supports both ``await tool(key=val)`` and ``await tool("val")``. + Positional arguments are mapped to parameter names using the + tool's schema (when available). + """ + if args and kwargs: + raise TypeError( + f"Tool '{self._tool_name}': cannot mix positional and keyword arguments" + ) + + if args: + param_names = self._get_param_names() + if not param_names: + if len(args) == 1: + # Fallback: try common first-param names + for guess in ( + "path", + "read", + "searches", + "edits", + "queries", + "tasks", + "delegations", + "code", + "command_string", + "command", + "summary", + ): + kwargs = {guess: args[0]} + break + else: + raise TypeError( + f"Tool '{self._tool_name}': cannot resolve positional " + f"argument – no schema available" + ) + else: + raise TypeError( + f"Tool '{self._tool_name}': cannot resolve positional " + f"arguments – no schema available" + ) + elif len(args) > len(param_names): + raise TypeError( + f"Tool '{self._tool_name}': too many positional arguments " + f"({len(args)} for {len(param_names)} parameter(s): {param_names})" + ) + else: + kwargs = dict(zip(param_names, args)) + + return await self.call(**kwargs) + + def _get_param_names(self) -> list: + """Extract ordered parameter names from the tool's JSON Schema.""" + if self._tool_module is None: + return [] + try: + props = self._tool_module.SCHEMA["function"]["parameters"]["properties"] + return list(props.keys()) + except (KeyError, TypeError, AttributeError): + return [] async def call(self, **kwargs: Any): """Execute the tool with the given keyword arguments. @@ -112,9 +173,10 @@ async def call(self, **kwargs: Any): return str(result) if self._mcp_server is not None: - return await self._coder._execute_mcp_tool( + result = await self._coder._execute_mcp_tool( self._mcp_server, self._mcp_tool_name, kwargs ) + return str(result) raise ValueError(f"No executor for tool '{self._tool_name}'") diff --git a/cecli/tools/_yield.py b/cecli/tools/_yield.py index 57d5860b200..e5f244a5a16 100644 --- a/cecli/tools/_yield.py +++ b/cecli/tools/_yield.py @@ -5,6 +5,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations logger = logging.getLogger(__name__) @@ -49,6 +50,8 @@ async def execute(cls, coder, **kwargs): cls.clear_invocation_cache() + response = ToolResponse(cls.NORM_NAME) + if coder: # Check for active child sub-agents and await their tasks before finishing try: @@ -88,10 +91,11 @@ async def execute(cls, coder, **kwargs): await t except (asyncio.CancelledError, Exception): pass - return ( + response.append_result( "Yield interrupted while waiting for sub-agents. " "Sub-agent outputs above may be incomplete." ) + return response # Retrieve exceptions from completed sub-agent tasks so they # are not silently lost. @@ -155,10 +159,11 @@ async def execute(cls, coder, **kwargs): await agent_service.reap_all_finished_agents(parent=coder) # Don't mark as finished — the coder should review sub-agent # outputs and decide how to proceed - return ( + response.append_result( "Sub-agents have finished. Please examine their output above " "in order to decide how you will proceed." ) + return response except Exception as e: logger.warning("Error awaiting child sub-agents before yield: %s", e) @@ -190,11 +195,14 @@ async def execute(cls, coder, **kwargs): coder.files_edited_by_tools = set() if summary: - return f"Yielded. Summary: {summary}" - return "Yielded." + response.append_result(f"Yielded. Summary: {summary}") + return response + response.append_result("Yielded.") + return response # coder.io.tool_Error("Error: Could not mark agent task as finished") - return "Error: Could not yield control" + response.append_error("Could not yield control") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/command.py b/cecli/tools/command.py index 8fe49a4d4d6..8055d12c8e5 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -19,6 +19,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -112,33 +113,47 @@ async def execute( Commands run with timeout based on agent_config['command_timeout'] (default: 30 seconds). """ + response = ToolResponse( + "command", + result_type=cls.RESULT_TYPE if hasattr(cls, "RESULT_TYPE") else "str", + ) # Handle interactions with an existing background command if background_key: if action == "stdin": if not stdin: - return "Error: 'stdin' is required when action='stdin'." + response.append_error("'stdin' is required when action='stdin'.") + return response cls.clear_invocation_cache() success = BackgroundCommandManager.send_command_input(background_key, stdin) if success: - return f"Sent input to background command {background_key}: {stdin}" + response.append_result( + f"Sent input to background command {background_key}: {stdin}" + ) + return response else: - return f"Error: Background command {background_key} not found or not running." + response.append_error( + f"Background command {background_key} not found or not running." + ) + return response elif action == "stop": return await cls._stop_background_command(coder, background_key) else: - return f"Error: Unknown action '{action}'. " "Use one of: stdin, stop." + response.append_error(f"Unknown action '{action}'. Use one of: stdin, stop.") + return response if not command: - return "Error: 'command' must be provided." + response.append_error("'command' must be provided.") + return response # Check for implicit background (trailing & on Linux) if ".cecli/agents" in command: - return ( - "Error: Do not attempt to access internal files with " + response.append_error( + "Do not attempt to access internal files with " "standard cli tools. Please use the tools you have been provided." ) + return response if not background and command.strip().endswith("&"): background = True @@ -147,7 +162,8 @@ async def execute( # Get user confirmation confirmed = await cls._get_confirmation(coder, command, background) if not confirmed: - return "Command execution skipped by user." + response.append_result("Command execution skipped by user.") + return response command = coder.format_command_with_prefix(command) @@ -240,11 +256,13 @@ async def _execute_background(cls, coder, command_string, use_pty=None, stdin=No if stdin: BackgroundCommandManager.send_command_input(command_key, stdin) - return ( + response = ToolResponse(cls.NORM_NAME) + response.append_result( f"Background command started: {command_string}\n" f"Command key: {command_key}\n" "Output will be injected into chat stream." ) + return response @classmethod async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=None): @@ -261,6 +279,8 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non from cecli.helpers.background_commands import CircularBuffer + response = ToolResponse(cls.NORM_NAME) + coder.io.tool_output( f"⛭ Executing shell command with {timeout}s timeout.", type="tool-result" ) @@ -342,7 +362,8 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non except subprocess.TimeoutExpired: process.kill() BackgroundCommandManager.stop_background_command(command_key) - return "Command execution interrupted by user." + response.append_result("Command execution interrupted by user.") + return response # Check if process has completed exit_code = process.poll() @@ -387,15 +408,17 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non coder.io.tool_output(output_content, type="tool-result") if exit_code == 0: - return ( + response.append_result( f"Shell command completed within {timeout}s timeout (exit code 0)." f" Output:\n{output_content}" ) + return response else: - return ( + response.append_result( f"Shell command completed within {timeout}s timeout with exit code" f" {exit_code}. Output:\n{output_content}" ) + return response # Check if timeout has expired elapsed = time.time() - start_time @@ -409,11 +432,12 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non # Get any output captured so far current_output = buffer.get_all(clear=False) - return ( + response.append_result( f"Command exceeded {timeout}s timeout and is continuing in background.\n" f"Command key: {command_key}\n" f"Output captured so far:\n{current_output}\n" ) + return response # Wait a bit before checking again await asyncio.sleep(1) @@ -423,6 +447,7 @@ async def _execute_foreground(cls, coder, command_string): """ Execute command in foreground (blocking). """ + response = ToolResponse(cls.NORM_NAME) should_print = True tui = None if coder.tui and coder.tui(): @@ -470,9 +495,15 @@ async def _execute_foreground(cls, coder, command_string): coder.io.tool_output(output_content, type="tool-result") if exit_status == 0: - return f"Shell command executed successfully (exit code 0). Output:\n{output_content}" + response.append_result( + f"Shell command executed successfully (exit code 0). Output:\n{output_content}" + ) + return response else: - return f"Shell command failed with exit code {exit_status}. Output:\n{output_content}" + response.append_result( + f"Shell command failed with exit code {exit_status}. Output:\n{output_content}" + ) + return response @classmethod async def _stop_background_command(cls, coder, command_key): @@ -482,19 +513,25 @@ async def _stop_background_command(cls, coder, command_key): success, output, exit_code = BackgroundCommandManager.stop_background_command(command_key) if success: - return ( + response = ToolResponse(cls.NORM_NAME) + response.append_result( f"Background command stopped: {command_key}\n" f"Exit code: {exit_code}\n" f"Final output:\n{output}" ) + return response else: - return output # Error message from manager + response = ToolResponse(cls.NORM_NAME) + response.append_result(output) + return response @classmethod async def _handle_errors(cls, coder, command_string, e): """Handle errors during command execution.""" coder.io.tool_error(f"Error executing shell command: {str(e)}") - return f"Error executing command: {str(e)}" + response = ToolResponse(cls.NORM_NAME) + response.append_error(f"Error executing command: {str(e)}") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/command_interactive.py b/cecli/tools/command_interactive.py index 1a24a66093d..3b2579bcb29 100644 --- a/cecli/tools/command_interactive.py +++ b/cecli/tools/command_interactive.py @@ -6,6 +6,7 @@ from cecli.run_cmd import run_cmd from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -101,9 +102,11 @@ async def execute(cls, coder, command_string, **kwargs): Execute an interactive shell command using run_cmd (which uses pexpect/PTY). """ try: + response = ToolResponse(cls.NORM_NAME) confirmed = await cls._get_confirmation(coder, command_string) if not confirmed: - return "Shell command execution skipped by user." + response.append_result("Shell command execution skipped by user.") + return response command_string = coder.format_command_with_prefix(command_string) @@ -154,15 +157,17 @@ def _run_interactive(): cls.clear_invocation_cache() if exit_status == 0: - return ( + response.append_result( "Interactive command finished successfully (exit code 0)." f" Output:\n{output_content}" ) + return response else: - return ( + response.append_result( f"Interactive command finished with exit code {exit_status}." f" Output:\n{output_content}" ) + return response except Exception as e: coder.io.tool_error( @@ -171,4 +176,6 @@ def _run_interactive(): # Optionally include traceback for debugging if verbose # if coder.verbose: # coder.io.tool_error(traceback.format_exc()) - return f"Error executing interactive command: {str(e)}" + response = ToolResponse(cls.NORM_NAME) + response.append_result(f"Error executing interactive command: {str(e)}") + return response diff --git a/cecli/tools/delegate.py b/cecli/tools/delegate.py index d4b55c80a58..be770578daf 100644 --- a/cecli/tools/delegate.py +++ b/cecli/tools/delegate.py @@ -5,11 +5,13 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "delegate" + RESULT_TYPE = "list" TRACK_INVOCATIONS = True VALIDATIONS = { "delegations": ["coerce_list"], @@ -53,19 +55,26 @@ class Tool(BaseTool): @classmethod async def execute(cls, coder, **kwargs): """Delegate one or more sub-agents to work on sub-tasks in parallel.""" + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) delegations = kwargs.get("delegations", []) if not delegations or not isinstance(delegations, list): - return "Error: 'delegations' parameter must be a non-empty array of {name, prompt} objects." + response.append_error( + "'delegations' parameter must be a non-empty array of {name, prompt} objects." + ) + return response # Validate each delegation item has the required fields for i, d in enumerate(delegations): if not isinstance(d, dict): - return f"Error: delegations[{i}] is not an object." + response.append_error(f"delegations[{i}] is not an object.") + return response if "name" not in d or not d["name"]: - return f"Error: delegations[{i}] is missing a 'name'." + response.append_error(f"delegations[{i}] is missing a 'name'.") + return response if "prompt" not in d or not d["prompt"]: - return f"Error: delegations[{i}] is missing a 'prompt'." + response.append_error(f"delegations[{i}] is missing a 'prompt'.") + return response from cecli.helpers.agents.service import AgentService @@ -97,7 +106,8 @@ async def _spawn_one(name: str, prompt: str) -> tuple[str, str]: n_total = len(started_agents) n_ok = sum(1 for _, r in started_agents if not r.startswith("failed:")) combined = "\n".join(lines) - return f"📋 Delegation results ({n_ok}/{n_total} dispatched):\n{combined}" + response.append_result(f"📋 Delegation results ({n_ok}/{n_total} dispatched):\n{combined}") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 761af17c91e..6b2763c6463 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -10,11 +10,10 @@ from cecli.tools.utils.helpers import ( ToolError, apply_change, - format_tool_result, - handle_tool_error, validate_file_for_edit, ) from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations VALID_OPERATIONS = {"replace", "delete", "insert"} @@ -34,6 +33,7 @@ class Tool(BaseTool): NORM_NAME = "editfile" + RESULT_TYPE = "list" TRACK_INVOCATIONS = False VALIDATIONS = { "edits": ["coerce_list"], @@ -153,7 +153,8 @@ def execute( force=True, ) - tool_name = "EditFile" + # tool_name = "EditFile" + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) try: # 1. Validate edits parameter if not isinstance(edits, list): @@ -397,13 +398,8 @@ def execute( # If dry run, return all results if dry_run: dry_run_messages = "\n".join(r.get("dry_run_message", "") for r in all_results) - return format_tool_result( - coder, - tool_name, - "", - dry_run=True, - dry_run_message=dry_run_messages or "Dry run: No changes would be made", - ) + response.append_result(dry_run_messages or "Dry run: No changes would be made") + return response # 4. Check if any edits succeeded overall if total_successful_edits == 0: @@ -437,19 +433,20 @@ def execute( cls.clear_invocation_cache() - return format_tool_result( - coder, - tool_name, - success_message, - change_id=change_id_to_return, + response.append_result( + f"\u2713 {success_message}" + + (f" (change_id: {change_id_to_return})" if change_id_to_return else "") ) + return response except ToolError as e: coder.edit_allowed = False - return handle_tool_error(coder, tool_name, e, add_traceback=False) + response.append_error(str(e)) + return response except Exception as e: coder.edit_allowed = False - return handle_tool_error(coder, tool_name, e) + response.append_error(str(e)) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/explore_code.py b/cecli/tools/explore_code.py index 8f7c8b734a1..cdf875942a2 100644 --- a/cecli/tools/explore_code.py +++ b/cecli/tools/explore_code.py @@ -3,6 +3,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations cwd = os.getcwd() @@ -23,6 +24,7 @@ class Tool(BaseTool): "queries": ["coerce_list"], "queries[]": ["coerce_dict"], } + RESULT_TYPE = "list" SCHEMA = { "type": "function", "function": { @@ -90,13 +92,15 @@ def execute(cls, coder, queries, **kwargs): Returns: str: Formatted results from the Cymbal operations. """ + response = ToolResponse(cls.NORM_NAME, result_type="list") try: # Check if cymbal is available if not CYMBAL_AVAILABLE: coder.io.tool_error( "Cymbal library is not available. Please install it with: pip install py-cymbal" ) - return "Error: Cymbal library is not available" + response.append_error("Cymbal library is not available") + return response # Initialize Cymbal and index if necessary c = cymbal.Cymbal() @@ -108,8 +112,8 @@ def execute(cls, coder, queries, **kwargs): except Exception as e: error_msg = f"Failed to index repository: {str(e)}" coder.io.tool_error(error_msg) - return f"Error: {error_msg}" - all_results = [] + response.append_error(error_msg) + return response all_failed_queries = [] total_successful_queries = 0 @@ -124,7 +128,7 @@ def execute(cls, coder, queries, **kwargs): # Replace hyphens with underscores (common in code) and strip special chars. safe_symbol = symbol.replace("-", "_") if symbol else symbol results = c.search(safe_symbol, limit=limit) - all_results.append(cls._format_search_results(results, symbol)) + response.append_result(cls._format_search_results(results, symbol)) elif action == "investigate": symbol_name = symbol file_hint = "" @@ -139,7 +143,7 @@ def execute(cls, coder, queries, **kwargs): try: investigation = c.investigate(safe_name, file_hint) - all_results.append( + response.append_result( cls._format_investigation_results(investigation, symbol) ) except Exception as e: @@ -153,13 +157,13 @@ def execute(cls, coder, queries, **kwargs): " more specific name or check the locations" f" below:\n{locations}" ) - all_results.append(msg) + response.append_result(msg) else: raise e elif action == "find_references": safe_symbol = symbol.replace("-", "_") if symbol else symbol references = c.find_references(safe_symbol, limit=limit) - all_results.append(cls._format_reference_results(references, symbol)) + response.append_result(cls._format_reference_results(references, symbol)) else: all_failed_queries.append( f"Error for symbol '{symbol}': Unknown action '{action}'" @@ -174,19 +178,21 @@ def execute(cls, coder, queries, **kwargs): error_msg = "No queries were successfully executed:\n" + "\n".join( all_failed_queries ) - raise ToolError(error_msg) + response.append_error(error_msg) + return response if all_failed_queries: for failed_msg in all_failed_queries: coder.io.tool_error(failed_msg) else: - coder.io.tool_output("✓ All queries successful.", type="tool-result") + coder.io.tool_output("\u2713 All queries successful.", type="tool-result") - return "\n\n" + "=" * 40 + "\n\n".join(all_results) + return response except Exception as e: coder.io.tool_error(f"Error in ExploreCode: {str(e)}") - return f"Error: {str(e)}" + response.append_error(str(e)) + return response finally: if "c" in locals(): c.close() diff --git a/cecli/tools/git_branch.py b/cecli/tools/git_branch.py index 21e442dfe1c..416937492fe 100644 --- a/cecli/tools/git_branch.py +++ b/cecli/tools/git_branch.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -83,8 +84,11 @@ def execute( """ List branches in the repository with various filtering and formatting options. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: # Build git command arguments @@ -121,13 +125,18 @@ def execute( try: head = coder.repo.repo.head if head.is_detached: - return "HEAD (detached)" - return coder.repo.repo.active_branch.name + response.append_result("HEAD (detached)") + return response + response.append_result(coder.repo.repo.active_branch.name) + return response except ANY_GIT_ERROR: - return "No current branch found." + response.append_result("No current branch found.") + return response - return result if result else "No branches found matching the criteria." + response.append_result(result if result else "No branches found matching the criteria.") + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git branch: {e}") - return f"Error running git branch: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_diff.py b/cecli/tools/git_diff.py index 99cb5d8ad6b..2e7144f3725 100644 --- a/cecli/tools/git_diff.py +++ b/cecli/tools/git_diff.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -31,8 +32,11 @@ def execute(cls, coder, branch=None, **kwargs): """ Show the diff between the current working directory and a git branch or commit. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: if branch: @@ -42,8 +46,11 @@ def execute(cls, coder, branch=None, **kwargs): diff = coder.repo.diff_commits(False, "HEAD", None) if not diff: - return "No differences found." - return diff + response.append_result("No differences found.") + return response + response.append_result(diff) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git diff: {e}") - return f"Error running git diff: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_log.py b/cecli/tools/git_log.py index 606308c1388..40bbc9fd3a8 100644 --- a/cecli/tools/git_log.py +++ b/cecli/tools/git_log.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -27,8 +28,11 @@ def execute(cls, coder, limit=10, **kwargs): """ Show the git log. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: commits = list(coder.repo.repo.iter_commits(max_count=limit)) @@ -37,7 +41,9 @@ def execute(cls, coder, limit=10, **kwargs): short_hash = commit.hexsha[:8] message = commit.message.strip().split("\n")[0] log_output.append(f"{short_hash} {message}") - return "\n".join(log_output) + response.append_result("\n".join(log_output)) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git log: {e}") - return f"Error running git log: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_remote.py b/cecli/tools/git_remote.py index 4138532b2ad..3025cce42a0 100644 --- a/cecli/tools/git_remote.py +++ b/cecli/tools/git_remote.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -22,18 +23,24 @@ def execute(cls, coder, **kwargs): """ List remote repositories. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: remotes = coder.repo.repo.remotes if not remotes: - return "No remotes configured." + response.append_result("No remotes configured.") + return response result = [] for remote in remotes: result.append(f"{remote.name}\t{remote.url}") - return "\n".join(result) + response.append_result("\n".join(result)) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git remote: {e}") - return f"Error running git remote: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_show.py b/cecli/tools/git_show.py index c4dee758e8d..60acbb66206 100644 --- a/cecli/tools/git_show.py +++ b/cecli/tools/git_show.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -27,11 +28,17 @@ def execute(cls, coder, object="HEAD", **kwargs): """ Show various types of objects (blobs, trees, tags, and commits). """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: - return coder.repo.repo.git.show(object) + result = coder.repo.repo.git.show(object) + response.append_result(result) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git show: {e}") - return f"Error running git show: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_status.py b/cecli/tools/git_status.py index 76531bf7dd6..523b4f3e863 100644 --- a/cecli/tools/git_status.py +++ b/cecli/tools/git_status.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -22,11 +23,17 @@ def execute(cls, coder, **kwargs): """ Show the working tree status. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: - return coder.repo.repo.git.status() + result = coder.repo.repo.git.status() + response.append_result(result) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git status: {e}") - return f"Error running git status: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index 30d95426bb4..3b658ac0556 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -1,4 +1,3 @@ -import json import os import re import shutil @@ -11,6 +10,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations # Default directories to exclude from search results across various languages @@ -160,8 +160,8 @@ def _parse_content_into_files(output): class Tool(BaseTool): NORM_NAME = "grep" + RESULT_TYPE = "list" VALIDATIONS = { - "searches": ["coerce_list"], "searches[]": ["coerce_dict"], } SCHEMA = { @@ -309,17 +309,23 @@ def execute( match counts, and summary metadata. """ if not isinstance(searches, list): - return json.dumps({"error": "'searches' parameter must be an array."}) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + response.append_error("'searches' parameter must be an array.") + return response repo = coder.repo if not repo: coder.io.tool_error("Not in a git repository.") - return json.dumps({"error": "Not in a git repository."}) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + response.append_error("Not in a git repository.") + return response tool_name, tool_path = cls._find_search_tool() if not tool_path: coder.io.tool_error("No search tool (rg, ag, grep) found in PATH.") - return json.dumps({"error": "No search tool (rg, ag, grep) found."}) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + response.append_error("No search tool (rg, ag, grep) found.") + return response all_operation_results = [] @@ -520,8 +526,6 @@ def execute( all_operation_results.append(op_result) - final_result = {"operations": all_operation_results} - # TUI summary if coder.tui and coder.tui(): ui_summaries = [] @@ -539,7 +543,10 @@ def execute( ui_message = "\n".join(ui_summaries) coder.io.tool_output(ui_message, type="tool-result") - return json.dumps(final_result) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + for op_result in all_operation_results: + response.append_result(op_result) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/ls.py b/cecli/tools/ls.py index 4305c5aea90..2054a261ef5 100644 --- a/cecli/tools/ls.py +++ b/cecli/tools/ls.py @@ -3,6 +3,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -41,6 +42,7 @@ def execute(cls, coder, path=None, **kwargs): """ # Handle both positional and keyword arguments for backward compatibility dir_path = path or "." + response = ToolResponse(cls.NORM_NAME) try: # Create an absolute path from the provided relative path @@ -51,12 +53,14 @@ def execute(cls, coder, path=None, **kwargs): coder.io.tool_error( f"Error: Path '{dir_path}' attempts to access files outside the project root." ) - return "Error: Path is outside the project root." + response.append_result("Error: Path is outside the project root.") + return response # Check if path exists if not os.path.exists(abs_path): coder.io.tool_output(f"⚠ Path '{dir_path}' not found", type="tool-result") - return "Directory not found" + response.append_result("Directory not found") + return response # Get directory contents contents = [] @@ -70,7 +74,8 @@ def execute(cls, coder, path=None, **kwargs): contents.append(rel_path) except OSError as e: coder.io.tool_error(f"Error listing directory '{dir_path}': {e}") - return f"Error: {e}" + response.append_result(f"Error: {e}") + return response elif os.path.isfile(abs_path): # It's a file, just return its relative path contents.append(os.path.relpath(abs_path, coder.root)) @@ -81,19 +86,25 @@ def execute(cls, coder, path=None, **kwargs): ) sorted_contents = sorted(contents) if len(sorted_contents) > 500: - return ( + response.append_result( f"Found {len(sorted_contents)} files:" f" {', '.join(sorted_contents[:500])}" f"\n... and {len(sorted_contents) - 500} more" ) + return response else: - return f"Found {len(sorted_contents)} files: {', '.join(sorted_contents)}" + response.append_result( + f"Found {len(sorted_contents)} files: {', '.join(sorted_contents)}" + ) + return response else: coder.io.tool_output(f"🗐 No files found in '{dir_path}'", type="tool-result") - return "No files found in directory" + response.append_result("No files found in directory") + return response except Exception as e: coder.io.tool_error(f"Error in ls: {str(e)}") - return f"Error: {str(e)}" + response.append_result(f"Error: {str(e)}") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/orchestrate.py b/cecli/tools/orchestrate.py index da82b487640..4b0405f8186 100644 --- a/cecli/tools/orchestrate.py +++ b/cecli/tools/orchestrate.py @@ -1,10 +1,10 @@ -import json import logging from cecli.helpers.orchestration.service import OrchestrationService from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations logger = logging.getLogger(__name__) @@ -46,7 +46,9 @@ async def execute(cls, coder, code, **kwargs): BaseTool.clear_invocation_cache() env = OrchestrationService.get_instance(coder) result = await env.execute(code) - return json.dumps(result) + response = ToolResponse(cls.NORM_NAME) + response.append_result(result) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index 0ffc04ca2fc..f4915e686db 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -1,4 +1,3 @@ -import json import os from typing import Dict, List @@ -6,16 +5,17 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, - handle_tool_error, is_provided, resolve_paths, ) from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "readfile" + RESULT_TYPE = "list" TRACK_INVOCATIONS = False VALIDATIONS = { "read": ["coerce_list"], @@ -98,11 +98,12 @@ def execute(cls, coder, read, **kwargs): """ from cecli.helpers.conversation import ConversationService - tool_name = "ReadFile" already_up_to_date = [] new_context_retrieved = [] error_outputs = [] + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + try: # 1. Validate read parameter if not isinstance(read, list): @@ -583,12 +584,12 @@ def _is_valid_int(s): ) if is_already_up_to_date: - if model_response not in already_up_to_set: - already_up_to_set.add(model_response) + if str(model_response) not in already_up_to_set: + already_up_to_set.add(str(model_response)) already_up_to_details.append(model_response) else: - if model_response not in new_context_set: - new_context_set.add(model_response) + if str(model_response) not in new_context_set: + new_context_set.add(str(model_response)) new_context_details.append(model_response) # Conditionally remove old file context messages @@ -617,8 +618,6 @@ def _is_valid_int(s): # Log success and return the formatted context directly coder.edit_allowed = True - result_parts = [f"File Context Turn {coder.turn_count}"] - if already_up_to_details or new_context_details: if new_context_details: coder.io.tool_output( @@ -626,12 +625,12 @@ def _is_valid_int(s): type="tool-result", ) - detail_str = "\n".join(new_context_details) - result_parts.append( - f"Retrieved context for {len(new_context_details)} operation(s):\n\n" - f"{detail_str}\n" - "Full results for these reads will be given in a follow up message.\n" + response.append_result( + f"Retrieved context for {len(new_context_details)} operation(s). " + "Full results for these reads will be given in a follow up message." ) + for d in new_context_details: + response.append_result(d) if already_up_to_details: coder.io.tool_output( ( @@ -641,41 +640,43 @@ def _is_valid_int(s): type="tool-result", ) - detail_str = "\n".join(already_up_to_details) - result_parts.append( + response.append_result( "Earlier contents still valid from previous read for " - f"{len(already_up_to_details)} operation(s):\n\n" - f"{detail_str}\n" + f"{len(already_up_to_details)} operation(s). " "Relevant contents for these reads available in previous message." ) + for d in already_up_to_details: + response.append_result(d) if already_up_to_date and not new_context_retrieved: - result_parts.append( + response.append_result( "Do not call `ReadFile` again with these parameters again unless you edit" " the relevant files." ) if all_outputs: - result_parts.append("\n".join(all_outputs)) - result_parts.append("\nUse these outlines to refine your search.\n") + for output in all_outputs: + response.append_result(output) + response.append_result("Use these outlines to refine your search.") if error_outputs: coder.io.tool_error( f"Errors encountered for {len(error_outputs)} operation(s)", type="tool-result" ) - result_parts.append("Errors:\n" + "\n".join(error_outputs)) + for err in error_outputs: + response.append_error(err) - if not result_parts: - return "No files could be processed." - - return "\n---\n".join(result_parts) + response.append_result(f"File Context Turn {coder.turn_count}") + return response except ToolError as e: # Handle expected errors raised by utility functions or validation - return handle_tool_error(coder, tool_name, e, add_traceback=False) + response.append_error(str(e)) + return response except Exception as e: # Handle unexpected errors during processing - return handle_tool_error(coder, tool_name, e) + response.append_error(str(e)) + return response @classmethod def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, current=False): @@ -728,7 +729,7 @@ def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, curr snapshot_parts.append("...⋮...\n") snapshot_parts.extend(hashed_lines[end_stub_s:end_stub_e]) - prefixed = "".join(snapshot_parts) + prefixed = "\n".join(snapshot_parts) result = { "file_path": rel_path, "start_line": start_stub_s + 1, @@ -737,7 +738,7 @@ def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, curr "expanded": True, "prefixed_contents": prefixed, } - return json.dumps(result, ensure_ascii=False) + return result except Exception: pass @@ -761,7 +762,7 @@ def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, curr "expanded": False, "prefixed_contents": prefixed, } - return json.dumps(result, ensure_ascii=False) + return result @classmethod def content_splitter(cls, coder, hashed_lines, s_idx, e_idx): @@ -1083,14 +1084,10 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr # If get_file_stub returned a useful structural outline, wrap it as JSON if stub and stub != "# No outline available": - result = json.dumps( - { - "file_path": rel_path, - "outline": stub, - }, - ensure_ascii=False, - ) - return result, True + return { + "file_path": rel_path, + "outline": stub, + }, True content = io.read_text(abs_path) if not content: @@ -1138,14 +1135,8 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr file_contents.append(f"{line_num}|{line_content}") file_contents.append("...") - parts.append( - json.dumps( - { - "file_path": rel_path, - "truncated": "\n".join(file_contents), - }, - ensure_ascii=False, - ) - ) + parts.append(f"file_path: {rel_path}") + parts.append("truncated:") + parts.append("\n".join(file_contents)) return "\n".join(parts), False diff --git a/cecli/tools/resource_manager.py b/cecli/tools/resource_manager.py index a58a232f12e..14bbdfc47f3 100644 --- a/cecli/tools/resource_manager.py +++ b/cecli/tools/resource_manager.py @@ -11,6 +11,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError, parse_arg_as_list from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -168,7 +169,7 @@ async def execute( ) coder.io.tool_output("⛭ Modifying Context", type="tool-result") - messages = [] + response = ToolResponse(cls.NORM_NAME) # Expand wildcards for MCP operations if "*" in load_mcp_servers and coder.mcp_manager: @@ -208,11 +209,11 @@ async def execute( c.registered_servers["included"] = included for f in create_files: - messages.append(cls._create(coder, f)) + response.append_result(cls._create(coder, f)) for f in remove_files: - messages.append(cls._remove(coder, f)) + response.append_result(cls._remove(coder, f)) for f in view_files: - messages.append(cls._view(coder, f)) + response.append_result(cls._view(coder, f)) for f in editable_files: try: abs_path = coder.abs_root_path(f) @@ -220,25 +221,25 @@ async def execute( abs_path = None if abs_path is not None and not os.path.isfile(abs_path): coder.io.tool_output(f"ℹ️ `{f}` missing on disk — using **create** instead of add") - messages.append(cls._create(coder, f)) + response.append_result(cls._create(coder, f)) else: - messages.append(cls._editable(coder, f)) + response.append_result(cls._editable(coder, f)) for key in stop_keys: - messages.append(cls._stop_command(coder, key)) + response.append_result(cls._stop_command(coder, key)) for skill_name in load_skill_names: - messages.append(cls._load_skill(coder, skill_name)) + response.append_result(cls._load_skill(coder, skill_name)) for skill_name in remove_skill_names: - messages.append(cls._remove_skill(coder, skill_name)) + response.append_result(cls._remove_skill(coder, skill_name)) for server_name in load_mcp_servers: result = await cls._load_mcp(coder, server_name) - messages.append(result) + response.append_result(result) for server_name in remove_mcp_servers: result = await cls._remove_mcp(coder, server_name) - messages.append(result) + response.append_result(result) for action_name in action_operations: result = await cls._list_mcp_servers(coder) - messages.append(result) + response.append_result(result) tui = getattr(coder, "tui", None) if tui and tui(): @@ -247,7 +248,7 @@ async def execute( coder.context_blocks_cache = {} coder.edit_allowed = True - return "\n".join(messages) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/thinking.py b/cecli/tools/thinking.py index 3cedc9eae6f..43bd8c342dd 100644 --- a/cecli/tools/thinking.py +++ b/cecli/tools/thinking.py @@ -1,6 +1,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -34,8 +35,10 @@ def execute(cls, coder, content, **kwargs): A place to allow the model to record freeform text as it iterates over tools to ideally help it guide itself to a proper solution """ + response = ToolResponse(cls.NORM_NAME) coder.io.tool_output("🧠 Thoughts recorded in context", type="tool-result") - return "🧠 Thoughts recorded in context. Please proceed with your task" + response.append_result("🧠 Thoughts recorded in context. Please proceed with your task") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/undo_change.py b/cecli/tools/undo_change.py index 67a4596362b..28489a62562 100644 --- a/cecli/tools/undo_change.py +++ b/cecli/tools/undo_change.py @@ -1,6 +1,7 @@ import traceback from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -34,12 +35,14 @@ def execute(cls, coder, change_id=None, file_path=None, **kwargs): """ # Note: Undo does not have a dry_run parameter as it's inherently about reverting a previous action. coder.edit_allowed = False + response = ToolResponse(cls.NORM_NAME) try: # Validate parameters if change_id is None and file_path is None: coder.io.tool_error("Must specify either change_id or file_path for UndoChange") - return "Error: Must specify either change_id or file_path" + response.append_result("Error: Must specify either change_id or file_path") + return response # If file_path is specified, get the most recent change for that file if file_path: @@ -49,14 +52,16 @@ def execute(cls, coder, change_id=None, file_path=None, **kwargs): change_id = coder.change_tracker.get_last_change(rel_path) if not change_id: coder.io.tool_error(f"No tracked changes found for file '{file_path}' to undo.") - return f"Error: No changes found for file '{file_path}'" + response.append_result(f"Error: No changes found for file '{file_path}'") + return response # Attempt to get undo information from the tracker success, message, change_info = coder.change_tracker.undo_change(change_id) if not success: coder.io.tool_error(f"Failed to undo change '{change_id}': {message}") - return f"Error: {message}" + response.append_result(f"Error: {message}") + return response # Apply the undo by restoring the original content if change_info: @@ -72,15 +77,20 @@ def execute(cls, coder, change_id=None, file_path=None, **kwargs): coder.io.tool_output( f"✓ Undid {change_type} change '{change_id}' in {file_path}", type="tool-result" ) - return f"Successfully undid {change_type} change '{change_id}'." + response.append_result(f"Successfully undid {change_type} change '{change_id}'.") + return response else: # This case should ideally not be reached if tracker returns success coder.io.tool_error( f"Failed to undo change '{change_id}': Change info missing after successful" " tracker update." ) - return f"Error: Failed to undo change '{change_id}' (missing change info)" + response.append_result( + f"Error: Failed to undo change '{change_id}' (missing change info)" + ) + return response except Exception as e: coder.io.tool_error(f"Error in UndoChange: {str(e)}\n{traceback.format_exc()}") - return f"Error: {str(e)}" + response.append_result(f"Error: {str(e)}") + return response diff --git a/cecli/tools/update_todo_list.py b/cecli/tools/update_todo_list.py index 812068f0c13..fd7ed7c6ac3 100644 --- a/cecli/tools/update_todo_list.py +++ b/cecli/tools/update_todo_list.py @@ -1,11 +1,13 @@ from cecli.tools.utils.base_tool import BaseTool -from cecli.tools.utils.helpers import ToolError, format_tool_result, handle_tool_error +from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "updatetodolist" + RESULT_TYPE = "list" VALIDATIONS = { "tasks": ["coerce_list"], "tasks[]": ["coerce_dict"], @@ -59,7 +61,8 @@ def execute(cls, coder, tasks, append=False, change_id=None, dry_run=False, **kw Update the todo list file (todo.txt) with formatted task items. Can either replace the entire content or append to it. """ - tool_name = "UpdateTodoList" + # tool_name = "UpdateTodoList" + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) try: # Define the todo file path todo_file_path = coder.local_agent_folder("todo.txt") @@ -130,18 +133,18 @@ def execute(cls, coder, tasks, append=False, change_id=None, dry_run=False, **kw # Check if content actually changed if existing_content == new_content: coder.io.tool_warning("No changes made: new content is identical to existing") - return ( + response.append_result( "Error: No changes made (content identical to existing)." - "Please make progress implementing the plan instead of updating it." + " Please make progress implementing the plan instead of updating it." ) + return response # Handle dry run if dry_run: action = "append to" if append else "replace" dry_run_message = f"Dry run: Would {action} todo list in {todo_file_path}." - return format_tool_result( - coder, tool_name, "", dry_run=True, dry_run_message=dry_run_message - ) + response.append_result(dry_run_message) + return response # Apply change metadata = { @@ -183,12 +186,15 @@ def execute(cls, coder, tasks, append=False, change_id=None, dry_run=False, **kw # success_message, # change_id=final_change_id, # ) - return new_content + response.append_result(new_content) + return response except ToolError as e: - return handle_tool_error(coder, tool_name, e, add_traceback=False) + response.append_error(str(e)) + return response except Exception as e: - return handle_tool_error(coder, tool_name, e) + response.append_error(str(e)) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/utils/base_tool.py b/cecli/tools/utils/base_tool.py index a9fb39c709a..cc612ed7399 100644 --- a/cecli/tools/utils/base_tool.py +++ b/cecli/tools/utils/base_tool.py @@ -2,6 +2,7 @@ from cecli.tools.utils.helpers import handle_tool_error from cecli.tools.utils.output import print_tool_response +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -16,6 +17,9 @@ class BaseTool(ABC): # Declarative validations (maps param paths to lists of validation method names) VALIDATIONS = {} + # Result format configuration ("str" or "list") + RESULT_TYPE = "str" + # Invocation tracking for detecting repeated tool calls _invocations = {} # Dict to store last 3 invocations per tool _invocation_summary = set() # Set to track distinct tool names @@ -133,7 +137,12 @@ def process_response(cls, coder, params): params = ToolValidations.validate_params(params, cls.VALIDATIONS, cls.SCHEMA) try: - return cls.execute(coder, **params) + result = cls.execute(coder, **params) + + if isinstance(result, ToolResponse): + return str(result) + + return result except Exception as e: return handle_tool_error(coder, cls.SCHEMA.get("function").get("name"), e) diff --git a/cecli/tools/utils/responses.py b/cecli/tools/utils/responses.py new file mode 100644 index 00000000000..e74897a8830 --- /dev/null +++ b/cecli/tools/utils/responses.py @@ -0,0 +1,101 @@ +import json + + +class ToolResponse: + """Assists in formatting all tool call responses as JSON for the LLM. + + By default, every tool response is wrapped in JSON: + + { + "tool_name": str, + "result": str | list[str], + "errors": list[str], + "details": list[str] + } + + Usage inside a tool's ``execute`` method:: + + response = ToolResponse("my_tool", result_type="str") + response.append_result("Operation completed successfully") + response.append_error("Minor warning: config file missing") + response.append_detail("Extra context for the LLM") + return response # __str__ returns valid JSON + + Tools that set ``RESULT_TYPE = "list"`` at the class level will have + result entries accumulated as a list instead of concatenated. + """ + + def __init__(self, tool_name, result_type="str"): + self.tool_name = tool_name + self.result_type = result_type + self._result = "" if result_type == "str" else [] + self._errors = [] + self._details = [] + + def append_result(self, result): + """Append a result entry. + + If ``result_type`` is ``"str"`` the text is concatenated (with a + newline separator for subsequent calls). If ``result_type`` is + ``"list"`` each call adds a new item to the results list. + """ + + if self.result_type == "str": + if self._result: + self._result += "\n" + str(result) + else: + self._result = str(result) + else: + self._result.append(result) + + def append_error(self, error): + """Collect an error message.""" + + self._errors.append(str(error)) + + def append_detail(self, detail): + """Collect an extraneous contextual detail for the LLM.""" + + self._details.append(str(detail)) + + def to_dict(self): + """Return the response as a plain Python dictionary.""" + + return { + "tool_name": self.tool_name, + "result": self._result, + "errors": self._errors, + "details": self._details, + } + + def to_json(self): + """Serialize the response to a JSON string.""" + + return json.dumps(self.to_dict()) + + def __str__(self): + return self.to_json() + + @staticmethod + def wrap(tool_name, result, errors=None): + """Convenience that returns a populated ToolResponse. + + Useful when wrapping results from non-ToolResponse-aware code + paths (e.g., MCP tool execution). + + Args: + tool_name: Name of the tool. + result: The result string to wrap. + errors: Optional list of error strings. + + Returns: + ToolResponse instance. + """ + + response = ToolResponse(tool_name) + response.append_result(result) + if errors: + for error in errors: + response.append_error(error) + + return response diff --git a/scripts/get_hashline.py b/scripts/get_hashline.py index 68f5e75787d..eb3765133ef 100755 --- a/scripts/get_hashline.py +++ b/scripts/get_hashline.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import sys +import traceback from pathlib import Path # Add the current directory to sys.path to allow importing from cecli @@ -50,6 +51,7 @@ def main(): print("\n\nSummary: No duplicate hash position hashes found.", file=sys.stderr) except Exception as e: + traceback.print_exc() print(f"Error reading file: {e}") sys.exit(1) From 1a6a2c6dfb70374368284299aa6933cff5f725db Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 01:33:32 -0400 Subject: [PATCH 05/68] Update orchestration prompts and example file format --- cecli/helpers/orchestration/environment.py | 44 ++++++++++++++-------- cecli/prompts/agent.yml | 14 +++---- cecli/prompts/subagent.yml | 16 ++++---- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index a427758de04..92b529dc0e4 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -163,8 +163,9 @@ def _get_param_names(self) -> list: async def call(self, **kwargs: Any): """Execute the tool with the given keyword arguments. - Tool results are returned as strings. Use ``import json`` and - ``json.loads(result)`` to parse structured responses when needed. + Tool results are returned as strings. Use ``json.loads(result)`` + on the string to get a dict with ``result``, ``errors``, and + ``details`` keys. """ if self._tool_module is not None: result = self._tool_module.process_response(self._coder, kwargs) @@ -500,20 +501,22 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non **Sequential calls:** ```python -read_tool = Agent.get_tool("ReadFile") -a = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") -b = await read_tool.call(file_path="bar.py", range_start="@000", range_end="000@") -f"Read {len(a)} and {len(b)} chars" +delegate = Agent.get_tool("Delegate") +a = await delegate.call(delegations=[{"name": "worker", "prompt": "Analyze foo.py"}]) +b = await delegate.call(delegations=[{"name": "worker", "prompt": "Analyze bar.py"}]) +f"Results: {a} and {b}" ``` **Parallel calls:** ```python -read_tool = Agent.get_tool("ReadFile") -files = ["a.py", "b.py", "c.py"] -tasks = [read_tool.call(file_path=f, range_start="@000", range_end="000@") for f in files] +delegate = Agent.get_tool("Delegate") +tasks = [ + delegate.call(delegations=[{"name": "worker", "prompt": "Analyze a.py"}]), + delegate.call(delegations=[{"name": "worker", "prompt": "Analyze b.py"}]), + delegate.call(delegations=[{"name": "worker", "prompt": "Analyze c.py"}]), +] results = await gather(*tasks) -state["contents"] = dict(zip(files, results)) -f"Read {len(files)} files in parallel" +f"Got {len(results)} results" ``` **Accumulating state across calls:** @@ -522,12 +525,21 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non f"Total so far: {state['count']}" ``` -**Parsing JSON tool results:** +**Structured tool responses:** +All tool calls return a dict has three keys: +- `result` — a list of result entries from the tool +- `errors` — a list of error strings (empty when successful) +- `details` — a list of extra contextual detail strings + ```python -read_tool = Agent.get_tool("ReadFile") -raw = await read_tool.call(file_path="data.json", range_start="@000", range_end="000@") -data = json.loads(raw) -f"Found {len(data)} entries" +grep = Agent.get_tool("Grep") +response = await grep.call(searches=[{"pattern": "TODO", "directory": "cecli/tools"}]) + +for entry in response["result"]: + print(f"Match: {entry}") +for err in response["errors"]: + print(f"Error: {err}") +f"Found {len(response['result'])} matching files, {len(response['errors'])} errors" ``` ### Tool Parameters diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index 9ca0cfe7b90..386d75bc6ce 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -32,16 +32,16 @@ main_system: | **Example File** ``` - GoTXSp::#!/usr/bin/env python3 + R号定::#!/usr/bin/env python3 - ILLPTX::def example_method(): - ItWHVI:: return "example" + ،即游::def example_method(): + ป止나:: return "example" - SDROMc::def foo(): - FPEPD:: return "bar" + 么으q::def foo(): + 内능索:: return "bar" - HiHyKn::def bar(): - XXTDIR:: return example_method() + " " + foo() + 增せе::def bar(): + 打Б2:: return example_method() + " " + foo() ``` ## Core Workflow diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index e62d1158fe5..f27d34a496d 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -7,7 +7,7 @@ main_system: | ## Core Directives **Act Proactively**: Autonomously use tools to fulfill the request. **Be Decisive**: Do not repeat searches or ask redundant questions. Trust your findings and be confident in your edits. - **Be Efficient**: Use multiple tools each request when exploring. Orchestrate tool calls for mutli-step operations. Respect usage limits while maximizing the utility of each request. + **Be Efficient**: Use multiple tools each request when exploring. Orchestrate tool calls for mutli-step operations. Respect usage limits while maximizing the utility of each request. **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT @@ -17,16 +17,16 @@ main_system: | **Example File** ``` - GoTXSp::#!/usr/bin/env python3 + R号定::#!/usr/bin/env python3 - ILLPTX::def example_method(): - ItWHVI:: return "example" + ،即游::def example_method(): + ป止나:: return "example" - SDROMc::def foo(): - FPEPD:: return "bar" + 么으q::def foo(): + 内능索:: return "bar" - HiHyKn::def bar(): - XXTDIR:: return example_method() + " " + foo() + 增せе::def bar(): + 打Б2:: return example_method() + " " + foo() ``` ## Core Workflow From 3190d8171c5344abaf8da6197a38abb5f2d58092 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 09:23:45 -0400 Subject: [PATCH 06/68] Raise default sub agent size for PTC delegations --- cecli/coders/agent_coder.py | 8 ++++++-- cecli/helpers/agents/service.py | 4 ++-- cecli/website/docs/config/agent-mode.md | 2 +- cecli/website/docs/config/subagents.md | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 4813065d068..4f3b0f78589 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -98,7 +98,7 @@ def __init__(self, *args, **kwargs): self.skip_cli_confirmations = False self.agent_finished = False self.agent_config = self._get_agent_config() - self.max_sub_agents = self.agent_config.get("max_sub_agents", 3) + self.max_sub_agents = self.agent_config.get("max_sub_agents", 30) self.sub_agent_paths = self.agent_config.get("subagent_paths", []) self._setup_agent() @@ -167,6 +167,11 @@ def _get_agent_config(self): config["hot_reload"] = nested.getter(config, "hot_reload", False) config["diff_colors"] = nested.getter(config, "diff_colors", True) config["allow_nested_delegation"] = nested.getter(config, "allow_nested_delegation", False) + config["allow_orchestration"] = nested.getter(config, "allow_orchestration", True) + config["max_sub_agents"] = nested.getter(config, "max_sub_agents", 30) + + if config["max_sub_agents"] == -1: + config["max_sub_agents"] = 2 ^ 31 - 1 config["tools_paths"] = nested.getter(config, ["tools_paths", "tool_paths"], []) config["tools_includelist"] = nested.getter( @@ -184,7 +189,6 @@ def _get_agent_config(self): config["servers_excludelist"] = nested.getter( config, ["servers_excludelist", "servers_blacklist"], [] ) - config["allow_orchestration"] = nested.getter(config, "allow_orchestration", True) config["include_context_blocks"] = set( nested.getter( diff --git a/cecli/helpers/agents/service.py b/cecli/helpers/agents/service.py index 273b7e1e4b9..ed7a517b192 100644 --- a/cecli/helpers/agents/service.py +++ b/cecli/helpers/agents/service.py @@ -228,7 +228,7 @@ def __init__(self, coder) -> None: @property def max_sub_agents(self) -> int: """Return the max number of sub-agents allowed for this coder.""" - return getattr(self.coder, "max_sub_agents", 3) + return getattr(self.coder, "max_sub_agents", 30) # ------------------------------------------------------------------ # # Internal helpers @@ -495,7 +495,7 @@ async def _create_sub_agent_coder( ) new_coder = await Coder.create(**kwargs) - new_coder.max_sub_agents = getattr(self.coder, "max_sub_agents", 3) + new_coder.max_sub_agents = getattr(self.coder, "max_sub_agents", 30) # IOProxy wrapping is handled by base_coder.py's Coder.__init__ # Re-acquire the lock to register — we must re-check max agents since diff --git a/cecli/website/docs/config/agent-mode.md b/cecli/website/docs/config/agent-mode.md index 2d571ee697f..797e7d61150 100644 --- a/cecli/website/docs/config/agent-mode.md +++ b/cecli/website/docs/config/agent-mode.md @@ -275,7 +275,7 @@ agent-config: # Sub-agent configuration subagent_paths: [".cecli/subagents"] # Optional: Directories to search for sub-agent definitions - max_sub_agents: 3 # Optional: Maximum concurrent sub-agents (default: 3) + max_sub_agents: 30 # Optional: Maximum concurrent sub-agents (default: 3) allow_nested_delegation: false # Optional: Allow sub-agents to delegate further (default: false) # Context blocks configuration diff --git a/cecli/website/docs/config/subagents.md b/cecli/website/docs/config/subagents.md index da0585a527b..46b6db5d4f9 100644 --- a/cecli/website/docs/config/subagents.md +++ b/cecli/website/docs/config/subagents.md @@ -56,7 +56,7 @@ Add sub-agent paths to your YAML configuration file: ```yaml # .cecli.conf.yml or ~/.cecli.conf.yml agent-config: - max_sub_agents: 3 # Maximum concurrent sub-agents (default: 3) + max_sub_agents: 30 # Maximum concurrent sub-agents (default: 3) subagent_paths: - ".cecli/subagents" # Default path - "~/team-agents" # Custom path for shared agent definitions @@ -141,7 +141,7 @@ Each agent gets its own output container. When you switch agents: ### Max Sub-Agents -The `max_sub_agents` setting (default: 3) limits how many concurrent sub-agents can exist. This prevents resource exhaustion. +The `max_sub_agents` setting (default: 30) limits how many concurrent sub-agents can exist. This prevents resource exhaustion. When the limit is reached: From ddb2e9eafe1c02820fb779bf6d2a461a1d0f59f8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 09:47:14 -0400 Subject: [PATCH 07/68] Use prime number base adjacency hash --- cecli/helpers/hashpos/hashpos.py | 50 +++++++++++++------------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index 00c47f7a6a4..9b522465fc6 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -80,22 +80,21 @@ def _get_line_hash(self, text: str) -> int: """ return xxhash.xxh3_64_intdigest(text.encode("utf-8")) & 0xFFFFF - def _get_file_fraction(self, line_idx: int) -> int: + def _get_adjacent_hash(self, line_idx: int) -> int: """ - Returns which 16th of the file the line is in (4 bits: 0-15). + Creates a 10-bit digest of specific surrounding lines at offsets + -7, -5, -3, -2, +2, +3, +5, +7 to provide local context. """ - return line_idx % 16 - - def _get_adjacent_hash(self, line_idx: int) -> int: - """Creates a 6-bit digest of the two lines before and two lines after (skipping current line).""" - start_idx = max(0, line_idx - 3) - end_idx = min(self.total, line_idx + 4) + offsets = [-7, -5, -3, -2, 2, 3, 5, 7] + adjacent_lines = [] - # Concatenate up to 2 lines before and up to 2 lines after - adjacent_lines = self.lines[start_idx:line_idx] + self.lines[line_idx + 1 : end_idx] + for offset in offsets: + target_idx = line_idx + offset + if 0 <= target_idx < self.total: + adjacent_lines.append(self.lines[target_idx]) context = "\n".join(adjacent_lines) - return xxhash.xxh3_64_intdigest(context.encode("utf-8")) & 0x3F + return xxhash.xxh3_64_intdigest(context.encode("utf-8")) & 0x3FF def generate_private_id(self, text: str) -> str: """ @@ -107,15 +106,14 @@ def generate_private_id(self, text: str) -> str: def generate_public_id(self, text: str, line_idx: int) -> str: """ Generates a 3-character Base1024 ID. - Layout: [20-bit Line Hash] [4-bit File Fraction] [6-bit Adjacent Hash] = 30 bits total. + Layout: [20-bit Line Hash] [10-bit Adjacent Hash] = 30 bits total. Each Base1024 character holds 10 bits. """ line_hash = self._get_line_hash(text) - fraction = self._get_file_fraction(line_idx) adj_hash = self._get_adjacent_hash(line_idx) # Pack the 30-bit integer - packed = (line_hash << 10) | (fraction << 6) | adj_hash + packed = (line_hash << 10) | adj_hash res = "" for _ in range(3): @@ -124,9 +122,9 @@ def generate_public_id(self, text: str, line_idx: int) -> str: packed //= 1024 return res - def unpack_public_id(self, public_id: str) -> tuple[int, int, int]: + def unpack_public_id(self, public_id: str) -> tuple[int, int]: """ - Reverses the Public ID back into its (Line Hash, Fraction, Adjacent Hash) values. + Reverses the Public ID back into its (Line Hash, Adjacent Hash) values. """ packed = 0 for i, char in enumerate(public_id): @@ -135,10 +133,9 @@ def unpack_public_id(self, public_id: str) -> tuple[int, int, int]: # Extract bits based on layout line_hash = (packed >> 10) & 0xFFFFF - fraction = (packed >> 6) & 0xF - adj_hash = packed & 0x3F + adj_hash = packed & 0x3FF - return line_hash, fraction, adj_hash + return line_hash, adj_hash def format_content(self, use_private_ids: bool = False, start_line: int = 1) -> str: formatted_lines = [] @@ -156,7 +153,7 @@ def format_content(self, use_private_ids: bool = False, start_line: int = 1) -> return "\n".join(formatted_lines) def resolve_to_lines(self, public_id: str, start_line: int = 1) -> list[int]: - target_line_hash, target_fraction, target_adj_hash = self.unpack_public_id(public_id) + target_line_hash, target_adj_hash = self.unpack_public_id(public_id) matches = [] # 1. Primary Filter: Find all lines whose 20-bit line content hash matches @@ -172,17 +169,10 @@ def resolve_to_lines(self, public_id: str, start_line: int = 1) -> list[int]: return matches # 2. Tie-Breaking Heuristic: - # If multiple identical lines exist, score them based on adjacency match and fraction distance. - def score_match(idx: int) -> tuple[int, int]: + # If multiple identical lines exist, score them based on adjacency match. + def score_match(idx: int) -> int: # Adjacency match: 0 means exact match, 1 means mismatch (we want lower scores) - adj_score = 0 if self._get_adjacent_hash(idx) == target_adj_hash else 1 - - # Fraction distance: Calculate how many 16ths away we are - current_fraction = self._get_file_fraction(idx) - fraction_dist = abs(current_fraction - target_fraction) - - # Sort by adjacency match first, then by closest spatial block - return (adj_score, fraction_dist) + return 0 if self._get_adjacent_hash(idx) == target_adj_hash else 1 matches.sort(key=score_match) From 16cfd9c1277bc1da1864032b18531ee08c3abcda Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 12:55:49 -0400 Subject: [PATCH 08/68] Update Read file to have my contextual position read facilities --- cecli/helpers/conversation/files.py | 11 +- cecli/helpers/grep_ast/grep_ast.py | 23 +- cecli/helpers/hashline.py | 5 +- cecli/helpers/hashpos/hashpos.py | 6 + cecli/tools/read_file.py | 864 +++++++++++++++++++--------- 5 files changed, 604 insertions(+), 305 deletions(-) diff --git a/cecli/helpers/conversation/files.py b/cecli/helpers/conversation/files.py index f28fd0f7ef9..55edc6f73ac 100644 --- a/cecli/helpers/conversation/files.py +++ b/cecli/helpers/conversation/files.py @@ -116,9 +116,7 @@ def add_file( try: content = coder.io.read_text(abs_fname, silent=True) if coder.hashlines: - content, json_str = hashline_formatted( - content, file_name=abs_fname, partial=False, expanded=False - ) + content, json_str = hashline_formatted(content, file_name=abs_fname) self._file_json_contents[abs_fname] = json_str except Exception: content = "" # Empty content for unreadable files @@ -236,9 +234,7 @@ def generate_diff(self, fname: str) -> Optional[str]: try: current_content = coder.io.read_text(abs_fname, silent=True) if coder.hashlines: - current_content, _ = hashline_formatted( - current_content, file_name=abs_fname, partial=False, expanded=False - ) + current_content, _ = hashline_formatted(current_content, file_name=abs_fname) except Exception: return None @@ -610,8 +606,7 @@ def get_file_context(self, file_path: str, all_ranges=False, check_versions=True _, json_str = hashline_formatted( range_content, file_name=abs_fname, - partial=True, - expanded=False, + total_lines=len(content_lines), start_line=start_line_adj, ) results.append(json.loads(json_str)) diff --git a/cecli/helpers/grep_ast/grep_ast.py b/cecli/helpers/grep_ast/grep_ast.py index 678a694a899..1b056f6195e 100644 --- a/cecli/helpers/grep_ast/grep_ast.py +++ b/cecli/helpers/grep_ast/grep_ast.py @@ -214,17 +214,24 @@ def format(self): # reset output += "\033[0m\n" - dots = not (0 in self.show_lines) + # dots = not (0 in self.show_lines) + in_gap = False + gap_start = -1 for i, line in enumerate(self.lines): if i not in self.show_lines: - if dots: - if self.line_number: - output += "...⋮...\n" - else: - output += "⋮\n" - dots = False + if not in_gap: + in_gap = True + gap_start = i continue + if in_gap: + omitted = i - gap_start + if self.line_number: + output += f"...⋮... [{omitted} lines omitted (L{gap_start + 1}–L{i})]\n" + else: + output += f"⋮ [{omitted} lines omitted]\n" + in_gap = False + if i in self.lines_of_interest and self.mark_lois: spacer = "█" if self.color: @@ -237,7 +244,7 @@ def format(self): line_output = f"{i + 1: 3}" + line_output output += line_output + "\n" - dots = True + # dots = True return output diff --git a/cecli/helpers/hashline.py b/cecli/helpers/hashline.py index 7064a41afd1..4c980ec5c67 100644 --- a/cecli/helpers/hashline.py +++ b/cecli/helpers/hashline.py @@ -30,7 +30,7 @@ def hashline(text: str, start_line: int = 1) -> str: def hashline_formatted( - text: str, file_name: str, partial: bool, expanded: bool, start_line: int = 1 + text: str, file_name: str, total_lines: int = 0, start_line: int = 1 ) -> tuple[str, str]: """ Generate hashline-formatted content and return it as both raw hashline text and a JSON structure. @@ -55,8 +55,7 @@ def hashline_formatted( "file_name": file_name, "start_line": start_line, "end_line": end_line, - "partial": partial, - "expanded": expanded, + "total_lines": total_lines if total_lines else hp.total, "prefixed_contents": prefixed, } diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index 9b522465fc6..a6e528e198e 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -70,6 +70,9 @@ class HashPos: # Regex for a raw 3-character fragment FRAGMENT_RE = re.compile(rf"^[{_B1024_REGEX_SET}]{{3}}$") + # Looser pattern: any 3 chars with at least one non-ASCII followed by :: + _LOOSE_PREFIX_RE = re.compile(r"^(?=.{0,2}[^\x00-\x7f]).{3}::") + def __init__(self, source_text: str = ""): self.lines = source_text.splitlines() self.total = len(self.lines) @@ -201,11 +204,14 @@ def resolve_range(self, start_id: str, end_id: str) -> tuple[int, int]: def strip_prefix(text: str) -> str: """ Remove HashPos prefixes from the start of every line. + Also strips any 3-char sequence with at least one non-ASCII char followed by ::. """ lines = text.splitlines(keepends=True) result_lines = [] for line in lines: stripped_line = HashPos.HASH_PREFIX_RE.sub("", line, count=1) + if stripped_line == line: + stripped_line = HashPos._LOOSE_PREFIX_RE.sub("", line, count=1) result_lines.append(stripped_line) return "".join(result_lines) diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index f4915e686db..2b086993484 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -1,3 +1,4 @@ +import json import os from typing import Dict, List @@ -33,12 +34,24 @@ class Tool(BaseTool): " They can contain up to 3 lines of content. Avoid using singular generic keywords and" " symbols. Special markers @000 and 000@ represent the file boundaries and can be" " used for range_start and range_end for the first and last lines of the file" - " respectively. Line numbers may also be used for range lookups." - " It is best to use function names, variable declarations, entire line contents" - " and other meaningful identifiers as range_start and range_end values." - " Results may be modified from the raw search for semantic relevance." + " respectively. Use '@L' followed by a line number (e.g., '@L10', '@L25')" + " for structured line-range lookups." + "" + " Contextual range_end markers (@C, @P, @N) expand around a unique range_start match:" + " '@C{number}' includes number lines before AND after the match (symmetric context)." + " '@P{number}' includes number lines PREVIOUS to (before) the match — the match" + " itself becomes the end of the range. '@N{number}' includes number lines NEXT" + " (after) the match — the match itself becomes the start of the range." + " These markers require range_start to match exactly one location." + "" + " Line hints: append ' @L' to any text pattern (e.g., 'my_func @L1506')" + " to disambiguate which match is intended when a pattern matches in many places." + " The @L hint is a proximity guide, not a strict line number." + "" + " It is best to use function names," + " variable declarations, entire line contents and other meaningful identifiers" " The returned identifiers will not persist after editing the file." - " Do not use the same pattern for the range_start and range_end." + " Only use the same pattern for range_start and range_end when fetching full method definitions." " Do not use empty strings for the range_start and range_end." " Do not use content IDs for the range_start and range_end values as they change between edits." " Always use the ReadFile tool instead of cli tools for reading file contents." @@ -64,6 +77,8 @@ class Tool(BaseTool): "description": ( "The text marking the beginning of the range." " Use '@000' for the first line on empty files." + " Append ' @L' (e.g., 'my_func @L1506') as a" + " proximity hint to help select among multiple matches." ), }, "range_end": { @@ -71,6 +86,12 @@ class Tool(BaseTool): "description": ( "The text marking the end of the range." " Use '000@' for the last line on empty files." + " When range_start uniquely matches one location, you" + " may use contextual markers: '@C{number}' (e.g., '@C5')" + " for lines on both sides of the match, '@P{number}'" + " for lines BEFORE the match (the match is the range" + " end), or '@N{number}' for lines AFTER the match" + " (the match is the range start)." ), }, }, @@ -173,6 +194,11 @@ def execute(cls, coder, read, **kwargs): range_start = strip_hashline(range_start).strip() range_end = strip_hashline(range_end).strip() + start_hint = None + end_hint = None + range_start, start_hint = cls._extract_l_hint(range_start) + range_end, end_hint = cls._extract_l_hint(range_end) + # 2. Resolve path abs_path, rel_path = resolve_paths(coder, file_path) if not os.path.exists(abs_path): @@ -228,222 +254,80 @@ def execute(cls, coder, read, **kwargs): start_line_idx = -1 end_line_idx = -1 both_structured = False - # found_by = "" if range_start is not None and range_end is not None: + # Step 1: Classify the search type + rt = cls._classify_search_type(range_start, range_end) - def _is_valid_int(s): - try: - int(s) - return True - except ValueError: - return False - - start_is_digit = _is_valid_int(range_start) - end_is_digit = _is_valid_int(range_end) - start_is_special = range_start in ("@000", "000@") - end_is_special = range_end in ("@000", "000@") - both_structured = (start_is_digit or start_is_special) and ( - end_is_digit or end_is_special - ) - start_is_text = not start_is_digit and not start_is_special - end_is_text = not end_is_digit and not end_is_special - mixed_special_search = (start_is_special and end_is_text) or ( - end_is_special and start_is_text - ) - start_indices = [] - end_indices = [] - - if both_structured: - if start_is_digit: - start_line_num = int(range_start) - 1 - start_line_num = max(0, min(start_line_num, num_lines - 1)) - start_indices = [start_line_num] - else: - start_indices = [0] - - if end_is_digit: - end_line_num = int(range_end) - 1 - end_line_num = max(0, min(end_line_num, num_lines - 1)) - end_indices = [end_line_num] - else: - end_indices = [num_lines - 1] - - elif mixed_special_search: - if start_is_special: - # Start is special marker, end is text pattern - if range_start == "@000": - start_indices = [0] - else: # 000@ - start_indices = [num_lines - 1] - # Search for end pattern as text - end_pattern_lines = range_end.split("\n") - end_indices = [] - for i in range(len(lines) - len(end_pattern_lines) + 1): - if all( - p_line in lines[i + j] - for j, p_line in enumerate(end_pattern_lines) - ): - end_indices.append(i + len(end_pattern_lines) - 1) - else: - # Start is text pattern, end is special marker - start_pattern_lines = range_start.split("\n") - start_indices = [] - for i in range(len(lines) - len(start_pattern_lines) + 1): - if all( - p_line in lines[i + j] - for j, p_line in enumerate(start_pattern_lines) - ): - start_indices.append(i) - if range_end == "@000": - end_indices = [0] - else: # 000@ - end_indices = [num_lines - 1] - else: - start_pattern_lines = range_start.split("\n") - start_indices = [] - for i in range(len(lines) - len(start_pattern_lines) + 1): - if all( - p_line in lines[i + j] - for j, p_line in enumerate(start_pattern_lines) - ): - start_indices.append(i) - - end_pattern_lines = range_end.split("\n") - end_indices = [] - for i in range(len(lines) - len(end_pattern_lines) + 1): - if all( - p_line in lines[i + j] for j, p_line in enumerate(end_pattern_lines) - ): - # For multiline end patterns, we want the index of the LAST line of the match - end_indices.append(i + len(end_pattern_lines) - 1) - - if len(start_indices) > 5: - # Too many matches - use _last_invocation to disambiguate - last = cls._last_invocation.get(abs_path) - if last is None: - error_outputs.append( - cls.format_error( - coder, - ( - f"Start pattern '{range_start}' too broad." - " Refine your search. Be more specific." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) - continue + # Step 2: Find start and end indices + start_indices = cls._find_start_indices(lines, range_start, rt, num_lines) + end_indices = cls._find_end_indices(lines, range_end, rt, num_lines) - # Find the best match: smallest sum of absolute distances to last start/end - # that comes after the range, with tie-breaking by smallest sum - last_s, last_e = last["start_idx"], last["end_idx"] - candidates = [] - for s in start_indices: - for e in [idx for idx in end_indices if idx >= s]: - dist_sum = abs(s - last_s) + abs(e - last_e) - candidates.append((dist_sum, s, e)) - # Sort by distance sum, then prefer ranges after the last range - candidates.sort(key=lambda x: (x[0], x[1] < last_s, x[1], x[2])) - if candidates: - best_pair = (candidates[0][1], candidates[0][2]) - else: - best_pair = None - else: - best_pair = None - min_dist = float("inf") - - for s in start_indices: - for e in [idx for idx in end_indices if idx >= s]: - dist = e - s - if dist < min_dist: - min_dist = dist - best_pair = (s, e) - - # If no valid pair found and one side has exactly one match, - # try inverted matching in case LLM got the order of methods wrong - if best_pair is None and (len(start_indices) == 1 or len(end_indices) == 1): - for s in start_indices: - for e in end_indices: - if e < s: # end pattern match is before start pattern match - dist = s - e - if dist < min_dist: - min_dist = dist - best_pair = (e, s) - - if not start_indices: - error_outputs.append( - cls.format_error( - coder, - ( - f"Start pattern '{range_start}' not found in {file_path}." - " Refine your search." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) - continue - - if not end_indices: - error_outputs.append( - cls.format_error( - coder, - ( - f"End pattern '{range_end}' not found in {file_path}." - " Refine your search." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) - continue - - if best_pair is None: - error_outputs.append( - cls.format_error( - coder, - ( - f"End pattern '{range_end}' not found after start pattern in" - f" {file_path}." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) + # Step 3: Apply contextual marker (@C/@P/@N) + if rt["end_is_contextual"]: + result, ctx_error = cls._apply_contextual_marker( + start_indices, + range_start, + range_end, + num_lines, + coder, + file_path, + read_index, + ) + if ctx_error: + error_outputs.append(ctx_error) continue + start_indices, end_indices = result + + # Step 4: Disambiguate if too many matches + start_indices = cls._disambiguate_start_indices( + start_indices, + end_indices, + abs_path, + range_start, + num_lines, + rel_path, + read_index, + coder, + response, + start_hint=start_hint, + ) - if best_pair is None: - error_outputs.append( - cls.format_error( - coder, - ( - f"End pattern '{range_end}' not found after start pattern in" - f" {file_path}." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) + # Step 5: Resolve to final indices with pair selection + fallbacks + s_idx, e_idx, resolve_errors = cls._resolve_to_final_indices( + start_indices, + end_indices, + num_lines, + coder, + abs_path, + range_start, + range_end, + file_path, + read_index, + ) + if resolve_errors: + for err in resolve_errors: + error_outputs.append(err) continue - s_idx, e_idx = best_pair - s_idx, e_idx = cls._extend_range_with_stub( - coder, abs_path, s_idx, e_idx, num_lines - ) + both_structured = rt["both_structured"] + mixed_special_search = rt["mixed_special"] + + # Check for repeat patterns BEFORE overwriting _last_invocation + last = cls._last_invocation.get(abs_path) + skip_truncation = ( + last is not None + and last.get("range_start") == range_start + and last.get("range_end") == range_end + ) # Store the found indices for future disambiguation - cls._last_invocation[abs_path] = {"start_idx": s_idx, "end_idx": e_idx} + cls._last_invocation[abs_path] = { + "start_idx": s_idx, + "end_idx": e_idx, + "range_start": range_start, + "range_end": range_end, + } # For structured searches (line numbers, special markers) or mixed searches # (one special marker, one text pattern), cap large ranges with preview @@ -451,8 +335,8 @@ def _is_valid_int(s): sliced_contents = "\n".join(content.splitlines()[s_idx:e_idx]) token_count = coder.main_model.token_count(content) sliced_token_count = coder.main_model.token_count(sliced_contents) - is_small_file = token_count <= min(coder.large_file_token_threshold / 4, 2048) - is_small_range = sliced_token_count <= min( + is_small_file = token_count <= max(coder.large_file_token_threshold / 4, 2048) + is_small_range = sliced_token_count <= max( coder.large_file_token_threshold / 8, 1024 ) if ( @@ -475,12 +359,26 @@ def _is_valid_int(s): if abs_path in coder.abs_read_only_fnames: coder.abs_read_only_fnames.remove(abs_path) - if preview not in all_outputs_set: - all_outputs_set.add(preview) + preview_key = ( + json.dumps(preview, sort_keys=True) + if isinstance(preview, dict) + else preview + ) + if preview_key not in all_outputs_set: + all_outputs_set.add(preview_key) if len(all_outputs): all_outputs.append("") all_outputs.append(preview) + # If the range was large and we're showing a preview, add explicit guidance + range_lines = e_idx - s_idx + 1 + if not has_stub: + response.append_result( + f"read operation {read_index + 1}: {rel_path} range " + f"({range_lines} lines) is large. " + f"Use @L ranges (e.g., @L{s_idx + 1}, @L{e_idx + 1}) for precise reads." + ) + continue # found_by = f"range '{range_start}' to '{range_end}'" @@ -513,9 +411,7 @@ def _is_valid_int(s): # output_lines = [f"Displaying context around {found_by} in {rel_path}:"] # Generate hashline for the entire file - hashed_content, _ = hashline_formatted( - content, file_name=abs_path, partial=False, expanded=False - ) + hashed_content, _ = hashline_formatted(content, file_name=abs_path) hashed_lines = hashed_content.splitlines() # Extract the context window from hashed lines @@ -580,7 +476,13 @@ def _is_valid_int(s): ): # hashed_slice = hashed_lines[s_idx : e_idx + 1] model_response = cls.format_model_response( - coder, rel_path, s_idx, e_idx, hashed_lines, current=is_already_up_to_date + coder, + rel_path, + s_idx, + e_idx, + hashed_lines, + current=is_already_up_to_date, + skip_truncation=skip_truncation, ) if is_already_up_to_date: @@ -628,6 +530,7 @@ def _is_valid_int(s): response.append_result( f"Retrieved context for {len(new_context_details)} operation(s). " "Full results for these reads will be given in a follow up message." + "Note: Full contents contain aggregated content across multiple reads." ) for d in new_context_details: response.append_result(d) @@ -679,73 +582,15 @@ def _is_valid_int(s): return response @classmethod - def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, current=False): + def format_model_response( + cls, coder, rel_path, s_idx, e_idx, hashed_lines, current=False, skip_truncation=False + ): """Format a file's context range as hash-prefixed lines for the model.""" - # Read file content for stub lookups - try: - from cecli.tools.utils.helpers import resolve_paths - - abs_path, _ = resolve_paths(coder, rel_path) - last_turn = cls._last_read_turn[abs_path] or 0 - except Exception: - pass - - # Try to return structural stub information instead of raw hashed lines - try: - if hashed_lines and current and coder.turn_count - last_turn <= 2: - num_lines = len(hashed_lines) - - start_stub_s, start_stub_e = cls._extend_range_with_stub( - coder, abs_path, s_idx, s_idx, num_lines - ) - end_stub_s, end_stub_e = cls._extend_range_with_stub( - coder, abs_path, e_idx, e_idx, num_lines - ) - - # start_stub_s, start_stub_e = cls._reposition_indices(s_idx, start_stub_s, start_stub_e) - # end_stub_s, end_stub_e = cls._reposition_indices(e_idx, end_stub_s, end_stub_e) - - start_found = start_stub_s != s_idx or start_stub_e != s_idx - end_found = end_stub_s != e_idx or end_stub_e != e_idx - - if end_stub_s != start_stub_s or end_stub_e != start_stub_e: - start_stub_s = end_stub_s - start_stub_e = end_stub_e - start_found = True - end_found = False - - if start_found or end_found: - snapshot_parts = [] - if start_found: - snapshot_parts.extend(hashed_lines[start_stub_s:start_stub_e]) - - has_second_range = ( - end_found - and start_stub_s != end_stub_s - and start_stub_e != end_stub_e - and end_stub_e != e_idx - ) - if has_second_range: - snapshot_parts.append("...⋮...\n") - snapshot_parts.extend(hashed_lines[end_stub_s:end_stub_e]) - - prefixed = "\n".join(snapshot_parts) - result = { - "file_path": rel_path, - "start_line": start_stub_s + 1, - "end_line": end_stub_e if has_second_range else start_stub_e, - "partial": True, - "expanded": True, - "prefixed_contents": prefixed, - } - return result - except Exception: - pass hashed_content = "\n".join(hashed_lines[s_idx : e_idx + 1]) token_count = coder.main_model.token_count(hashed_content) - if token_count <= min(coder.large_file_token_threshold / 16, 512): + if skip_truncation or token_count <= min(coder.large_file_token_threshold / 16, 512): prefixed = hashed_content else: total = e_idx - s_idx @@ -758,8 +603,7 @@ def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, curr "file_path": rel_path, "start_line": s_idx + 1, "end_line": e_idx + 1, - "partial": True, - "expanded": False, + "total_lines": len(hashed_lines), "prefixed_contents": prefixed, } return result @@ -857,7 +701,12 @@ def content_splitter(cls, coder, hashed_lines, s_idx, e_idx): output_lines = [] for chunk_idx, chunk in enumerate(output_parts): if chunk_idx > 0: - output_lines.append("...⋮...") + prev_end = output_parts[chunk_idx - 1][-1] + 1 + next_start = chunk[0] + 1 + omitted = next_start - prev_end - 1 + output_lines.append( + f"...⋮... [{omitted} lines omitted (L{prev_end + 1}–L{next_start - 1})]" + ) for idx in chunk: output_lines.append(hashed_lines[idx]) @@ -1054,6 +903,443 @@ def _extend_range_with_stub(cls, coder, abs_path, s_idx, e_idx, num_lines): # Fallback to original range if anything goes wrong return s_idx, e_idx + @classmethod + def _try_fuzzy_narrow_indices( + cls, coder, abs_path, start_indices, num_lines, search_pattern=None + ): + """Try to narrow down ambiguous pattern matches using structural features. + + First attempts exact structural proximity (indices near def/class/import + lines). If that fails, falls back to rapidfuzz fuzzy matching the search + pattern against tag names from the repo map outline. + + Returns narrowed list of indices, or None if narrowing wasn't possible. + """ + try: + from cecli.repomap import RepoMap + + if not hasattr(RepoMap, "_stub_instance"): + RepoMap._stub_instance = RepoMap(map_tokens=0, io=coder.io) + rm = RepoMap._stub_instance + rel_fname = rm.get_rel_fname(abs_path) + tags = rm.get_tags(abs_path, rel_fname) + + if not tags: + return None + + # Build structural line set for proximity matching + structural_lines = { + tag.line for tag in tags if tag.kind == "def" or tag.specific_kind == "import" + } + + if structural_lines: + narrowed = [] + for si in start_indices: + for sl in structural_lines: + if abs(si - sl) <= 3: + narrowed.append(si) + break + + if narrowed: + return sorted(set(narrowed)) + + # Structural proximity didn't narrow — try fuzzy name matching + if search_pattern: + from rapidfuzz import fuzz + + # Collect def tags with their names and lines + def_tags = [ + (tag.name, tag.line) + for tag in tags + if tag.kind == "def" and getattr(tag, "name", None) + ] + + if def_tags: + # Fuzzy match the search pattern against each tag name + matched_lines = [] + for tag_name, tag_line in def_tags: + score = fuzz.partial_ratio(search_pattern.lower(), tag_name.lower()) + if score >= 70: + matched_lines.append(tag_line) + + if matched_lines: + # Return start_indices that are near fuzzy-matched structural lines + narrowed = [] + for si in start_indices: + for ml in matched_lines: + if abs(si - ml) <= 5: + narrowed.append(si) + break + + if narrowed: + return sorted(set(narrowed)) + + # No start_indices near fuzzy-matched lines — + # return the matched lines themselves as anchors + return sorted(set(matched_lines)) + + return None + except Exception: + return None + + @classmethod + def _classify_search_type(cls, range_start, range_end): + """Classify range markers into structured, text, mixed, or contextual search types.""" + + def _is_line_ref(s): + return s.startswith("@L") and s[2:].isdigit() and len(s) > 2 + + start_is_lr = _is_line_ref(range_start) + end_is_lr = _is_line_ref(range_end) + start_is_sp = range_start in ("@000", "000@") + end_is_sp = range_end in ("@000", "000@") + + return { + "start_is_line_ref": start_is_lr, + "end_is_line_ref": end_is_lr, + "start_is_special": start_is_sp, + "end_is_special": end_is_sp, + "start_is_text": not start_is_lr and not start_is_sp, + "end_is_text": not end_is_lr and not end_is_sp, + "both_structured": (start_is_lr or start_is_sp) and (end_is_lr or end_is_sp), + "mixed_special": ( + (start_is_sp and not end_is_lr and not end_is_sp) + or (end_is_sp and not start_is_lr and not start_is_sp) + ), + "end_is_contextual": ( + range_end.startswith(("@C", "@P", "@N")) + and len(range_end) > 2 + and range_end[2:].isdigit() + ), + } + + @classmethod + def _extract_l_hint(cls, pattern): + """Extract @L line hint suffix from a pattern string. + + Supports patterns like 'my_function @L1506' where the @L suffix + provides a proximity hint for disambiguation. + + Returns (stripped_pattern, line_hint) where line_hint is a 0-based + line number or None if no @L suffix is found. + """ + import re + + m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) + if m: + return pattern[: m.start()], int(m.group(1)) - 1 + return pattern, None + + @classmethod + def _search_in_lines(cls, lines, pattern, return_last_line=False): + """Search for a multiline pattern in lines. + + Returns list of matching indices. When return_last_line is True, + returns the index of the last line of each match instead of the first. + """ + pattern_lines = pattern.split("\n") + indices = [] + offset = len(pattern_lines) - 1 if return_last_line else 0 + for i in range(len(lines) - len(pattern_lines) + 1): + if all(p_line in lines[i + j] for j, p_line in enumerate(pattern_lines)): + indices.append(i + offset) + return indices + + @classmethod + def _find_start_indices(cls, lines, range_start, classification, num_lines): + """Resolve start indices based on the classified search type.""" + if classification["start_is_line_ref"]: + line_num = int(range_start[2:]) - 1 + return [max(0, min(line_num, num_lines - 1))] + + if classification["start_is_special"]: + return [0] if range_start == "@000" else [num_lines - 1] + + return cls._search_in_lines(lines, range_start) + + @classmethod + def _find_end_indices(cls, lines, range_end, classification, num_lines): + """Resolve end indices based on the classified search type.""" + if classification["end_is_line_ref"]: + line_num = int(range_end[2:]) - 1 + return [max(0, min(line_num, num_lines - 1))] + + if classification["end_is_special"]: + return [0] if range_end == "@000" else [num_lines - 1] + + return cls._search_in_lines(lines, range_end, return_last_line=True) + + @classmethod + def _apply_contextual_marker( + cls, + start_indices, + range_start, + range_end, + num_lines, + coder, + file_path, + read_index, + ): + """Expand range using @C/@P/@N contextual markers. + + Returns ((new_start_indices, new_end_indices), error). + Error is None on success, or a formatted error string on failure. + """ + if len(start_indices) != 1: + error = cls.format_error( + coder, + ( + f"Start pattern '{range_start}' must match exactly one" + f" location when using @C/@P/@N end markers." + f" Found {len(start_indices)} matches." + ), + file_path, + range_start, + range_end, + read_index, + ) + return None, error + + ctx_s_idx = start_indices[0] + ctx_num = int(range_end[2:]) + marker_type = range_end[1] + + if marker_type == "C": + s_idx = max(0, ctx_s_idx - ctx_num) + e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) + elif marker_type == "P": + s_idx = max(0, ctx_s_idx - ctx_num) + e_idx = ctx_s_idx + else: # 'N' + s_idx = ctx_s_idx + e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) + + return ([s_idx], [e_idx]), None + + @classmethod + def _disambiguate_start_indices( + cls, + start_indices, + end_indices, + abs_path, + range_start, + num_lines, + rel_path, + read_index, + coder, + response, + start_hint=None, + ): + """Narrow ambiguous start_indices when there are too many matches. + + Tries @L hint proximity narrowing first, then fuzzy structural + narrowing, falls back to first 5 with a warning listing + line numbers. Last-invocation disambiguation is handled separately + in _resolve_to_final_indices. + """ + + if len(start_indices) <= 5: + return start_indices + + last = cls._last_invocation.get(abs_path) + if last is not None: + # Last-invocation disambiguation happens in _resolve_to_final_indices + return start_indices + + # @L hint: filter to indices closest to the hinted line + if start_hint is not None: + proximity_narrowed = cls._narrow_by_proximity(start_indices, start_hint, max_results=5) + if proximity_narrowed and len(proximity_narrowed) <= 5: + line_nums = [str(i + 1) for i in sorted(proximity_narrowed)] + response.append_result( + f"read operation {read_index + 1}: start pattern " + f"'{range_start}' matched many locations in {rel_path}; " + f"narrowed to {len(proximity_narrowed)} closest to @L hint " + f"at lines {', '.join(line_nums)}. " + f"Tip: append ' @L' to any pattern (e.g., " + f"'{range_start} @L{start_hint + 1}') to target a specific match." + ) + return proximity_narrowed + + narrowed = cls._try_fuzzy_narrow_indices( + coder, + abs_path, + start_indices, + num_lines, + search_pattern=range_start, + ) + + if narrowed and len(narrowed) <= 5: + line_nums = [str(i + 1) for i in sorted(narrowed)] + response.append_result( + f"read operation {read_index + 1}: start pattern " + f"'{range_start}' matched many locations in {rel_path}; " + f"narrowed to {len(narrowed)} structural match(es) " + f"at lines {', '.join(line_nums)}. " + f"Tip: append ' @L' to a pattern (e.g., " + f"'{range_start} @L{line_nums[0]}') to target a specific match." + ) + return narrowed + + line_nums = [str(i + 1) for i in sorted(start_indices[:5])] + response.append_result( + f"read operation {read_index + 1}: start pattern " + f"'{range_start}' too broad ({len(start_indices)} matches) " + f"in {rel_path}. Using first 5 (lines {', '.join(line_nums)}). " + f"Refine with specific names or @L ranges for precision." + ) + return start_indices[:5] + + @classmethod + def _narrow_by_proximity(cls, indices, target, max_results=5): + """Narrow a list of indices to those closest to a target index. + + Returns up to max_results indices sorted by ascending distance + from the target (0-based line number). + """ + if not indices: + return [] + + scored = [(abs(i - target), i) for i in indices] + scored.sort(key=lambda x: x[0]) + return [idx for _, idx in scored[:max_results]] + + @classmethod + def _resolve_to_final_indices( + cls, + start_indices, + end_indices, + num_lines, + coder, + abs_path, + range_start, + range_end, + file_path, + read_index, + ): + """Find the best (start, end) pair and handle fallbacks. + + Returns (s_idx, e_idx, errors) where errors is a list of formatted + error strings. On success, errors is empty and s_idx/e_idx are set. + """ + errors = [] + last = cls._last_invocation.get(abs_path) + + # Find best pair (with last-invocation disambiguation if available) + if last and len(start_indices) > 5: + last_s, last_e = last["start_idx"], last["end_idx"] + candidates = [] + for s in start_indices: + for e in [idx for idx in end_indices if idx >= s]: + dist_sum = abs(s - last_s) + abs(e - last_e) + candidates.append((dist_sum, s, e)) + candidates.sort(key=lambda x: (x[0], x[1] < last_s, x[1], x[2])) + best_pair = (candidates[0][1], candidates[0][2]) if candidates else None + else: + best_pair = cls._compute_best_pair(start_indices, end_indices) + + # Inverted matching: LLM may have swapped start/end pattern order + if best_pair is None and (len(start_indices) == 1 or len(end_indices) == 1): + best_pair = cls._compute_best_pair(start_indices, end_indices, allow_inverted=True) + + # Error: start pattern not found + if not start_indices: + errors.append( + cls.format_error( + coder, + f"Start pattern '{range_start}' not found in {file_path}. Refine your search.", + file_path, + range_start, + range_end, + read_index, + ) + ) + return None, None, errors + + # Error: end pattern not found — expand from start + if not end_indices: + if start_indices: + s_idx = start_indices[0] + try: + s_idx, e_idx = cls._extend_range_with_stub( + coder, + abs_path, + s_idx, + num_lines - 1, + num_lines, + ) + except Exception: + e_idx = num_lines - 1 + return s_idx, e_idx, [] + else: + errors.append( + cls.format_error( + coder, + f"End pattern '{range_end}' not found in {file_path}. Refine your search.", + file_path, + range_start, + range_end, + read_index, + ) + ) + return None, None, errors + + # Both matched but no valid pair — expand from start + if best_pair is None: + if start_indices: + s_idx = start_indices[0] + try: + s_idx, e_idx = cls._extend_range_with_stub( + coder, + abs_path, + s_idx, + num_lines - 1, + num_lines, + ) + except Exception: + e_idx = num_lines - 1 + return s_idx, e_idx, [] + else: + errors.append( + cls.format_error( + coder, + ( + f"End pattern '{range_end}' not found after start pattern" + f" in {file_path}." + ), + file_path, + range_start, + range_end, + read_index, + ) + ) + return None, None, errors + + return best_pair[0], best_pair[1], [] + + @classmethod + def _compute_best_pair(cls, start_indices, end_indices, allow_inverted=False): + """Find the smallest-range valid (start, end) pair. + + When allow_inverted is True, also considers pairs where end < start + (LLM swapped the order) and returns them in correct order. + """ + min_dist = float("inf") + best_pair = None + + for s in start_indices: + valid_ends = end_indices if allow_inverted else [e for e in end_indices if e >= s] + for e in valid_ends: + dist = abs(e - s) + if dist < min_dist: + min_dist = dist + best_pair = (s, e) + + if allow_inverted and best_pair and best_pair[1] < best_pair[0]: + best_pair = (best_pair[1], best_pair[0]) + + return best_pair + @classmethod def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=True): """Get a preview of a large file range between start_idx and end_idx. @@ -1087,6 +1373,10 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr return { "file_path": rel_path, "outline": stub, + "note": ( + f"Large File. Tip: use @L ranges for precise reads" + f" (e.g., @L{start_idx + 1}, @L{end_idx + 1})." + ), }, True content = io.read_text(abs_path) @@ -1122,13 +1412,15 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr if sample_lines and sample_lines[-1][0] != actual_end: sample_lines.append((actual_end, lines[actual_end])) - # Format the output parts = [ f"File range too large ({total_lines} lines).", + ( + f"Tip: use @L ranges for precise reads" + f" (e.g., @L{actual_start + 1}, @L{actual_end + 1})." + ), f"Showing {len(sample_lines)} equally-spaced lines from the range:", "", ] - file_contents = [] for idx, line_content in sample_lines: line_num = idx + 1 From 0fcdb7bc29702d99554d076cedfbb27f8bd9cf88 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 13:13:54 -0400 Subject: [PATCH 09/68] Fix bug in generating file context ranges since we need surrounding context, update LLM expectations on what changes as edits happen --- cecli/helpers/conversation/files.py | 25 ++++++++++++++++++++++--- cecli/prompts/agent.yml | 3 ++- cecli/prompts/subagent.yml | 3 ++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/cecli/helpers/conversation/files.py b/cecli/helpers/conversation/files.py index 55edc6f73ac..ae5721b96e6 100644 --- a/cecli/helpers/conversation/files.py +++ b/cecli/helpers/conversation/files.py @@ -603,11 +603,30 @@ def get_file_context(self, file_path: str, all_ranges=False, check_versions=True ): continue - _, json_str = hashline_formatted( - range_content, + # Hash the full content first so that adjacent-line hashes + # are computed with proper surrounding context from the complete + # file, then extract only the range's lines. This ensures content + # IDs are consistent with other tools (e.g., ReadFile) that hash + # the full file content. + full_prefixed, _ = hashline_formatted( + content, file_name=abs_fname, total_lines=len(content_lines), - start_line=start_line_adj, + start_line=1, + ) + prefixed_lines = full_prefixed.splitlines() + range_prefixed_lines = prefixed_lines[start_line_adj - 1 : end_line_adj] + range_prefixed_content = "\n".join(range_prefixed_lines) + + json_str = json.dumps( + { + "file_name": abs_fname, + "start_line": start_line_adj, + "end_line": end_line_adj, + "total_lines": len(content_lines), + "prefixed_contents": range_prefixed_content, + }, + ensure_ascii=False, ) results.append(json.loads(json_str)) diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index 386d75bc6ce..60a67ed290f 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -27,7 +27,8 @@ main_system: | ## FILE FORMAT File contents will be prefixed with identifiers. Each line starts with a variable length content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change after performing file edits. Do not search for these content IDs directly. You will not be able to generate them. + They are generated and maintained by a custom algorithm and subject to change in the local neighborhood around file edit locations. + Do not search for these content IDs directly. You will not be able to generate them. Focus on the lines they identify. **Example File** diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index f27d34a496d..111ff7c5cb6 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -12,7 +12,8 @@ main_system: | ## FILE FORMAT File contents will be prefixed with identifiers. Each line starts with a variable length content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change after performing file edits. Do not search for these content IDs directly. You will not be able to generate them. + They are generated and maintained by a custom algorithm and subject to change in the local neighborhood around file edit locations. + Do not search for these content IDs directly. You will not be able to generate them. Focus on the lines they identify. **Example File** From 0282b670e02bee7e9567e8bcc0d95dbf9a3aed1a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 13:24:04 -0400 Subject: [PATCH 10/68] Update file format section of agent mode system prompts --- cecli/prompts/agent.yml | 5 ++--- cecli/prompts/subagent.yml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index 60a67ed290f..94791760927 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -26,10 +26,9 @@ main_system: | **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT - File contents will be prefixed with identifiers. Each line starts with a variable length content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change in the local neighborhood around file edit locations. + File contents will be prefixed with identifiers. Each line starts with a case-sensitive content ID followed by `::`. These are used to target where editing tools will perform edits. + They are generated and maintained by a custom algorithm and subject to change after file edits. Do not search for these content IDs directly. You will not be able to generate them. - Focus on the lines they identify. **Example File** ``` diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index 111ff7c5cb6..4fdba2a922d 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -11,10 +11,9 @@ main_system: | **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT - File contents will be prefixed with identifiers. Each line starts with a variable length content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change in the local neighborhood around file edit locations. + File contents will be prefixed with identifiers. Each line starts with a case-sensitive content ID followed by `::`. These are used to target where editing tools will perform edits. + They are generated and maintained by a custom algorithm and subject to change after file edits. Do not search for these content IDs directly. You will not be able to generate them. - Focus on the lines they identify. **Example File** ``` From 6f77950d902472a9edce9347211e9b5edbd79b9b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 13:25:42 -0400 Subject: [PATCH 11/68] Tool results are and should be dicts for delegation --- cecli/helpers/orchestration/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 92b529dc0e4..8b839e5b946 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -171,13 +171,13 @@ async def call(self, **kwargs: Any): result = self._tool_module.process_response(self._coder, kwargs) if asyncio.iscoroutine(result): result = await result - return str(result) + return result if self._mcp_server is not None: result = await self._coder._execute_mcp_tool( self._mcp_server, self._mcp_tool_name, kwargs ) - return str(result) + return result raise ValueError(f"No executor for tool '{self._tool_name}'") From aa8b3d34fd2287b529d1054836bfde2190b1f9a7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 13:37:57 -0400 Subject: [PATCH 12/68] Normalize tool results in orchestration tool --- cecli/helpers/orchestration/environment.py | 70 ++++++++++++++++++++-- tests/helpers/test_orchestration.py | 7 ++- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 8b839e5b946..caf7ffb79a4 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -97,6 +97,7 @@ def __init__( self._coder = coder self._tool_module = tool_module self._mcp_server = mcp_server + self._mcp_tool_name = mcp_tool_name async def __call__(self, *args: Any, **kwargs: Any): """Make the proxy directly callable. @@ -163,24 +164,83 @@ def _get_param_names(self) -> list: async def call(self, **kwargs: Any): """Execute the tool with the given keyword arguments. - Tool results are returned as strings. Use ``json.loads(result)`` - on the string to get a dict with ``result``, ``errors``, and - ``details`` keys. + Tool results are normalized to a dict with ``result`` (list), + ``errors`` (list), and ``details`` (list) keys, matching the + documented orchestration contract. """ + if self._tool_module is not None: result = self._tool_module.process_response(self._coder, kwargs) if asyncio.iscoroutine(result): result = await result - return result + return self._normalize_result(result) if self._mcp_server is not None: result = await self._coder._execute_mcp_tool( self._mcp_server, self._mcp_tool_name, kwargs ) - return result + return self._normalize_result(result) raise ValueError(f"No executor for tool '{self._tool_name}'") + @staticmethod + def _normalize_result(result: Any) -> dict: + """Normalize a tool result into a dict with ``result``, ``errors``, ``details`` keys. + + Handles ``ToolResponse``, error strings, and plain dicts. + """ + from cecli.tools.utils.responses import ToolResponse + + if isinstance(result, ToolResponse): + data = result.to_dict() + out = {} + out["result"] = ( + data["result"] + if isinstance(data["result"], list) + else [data["result"]] if data["result"] else [] + ) + out["errors"] = data.get("errors", []) + out["details"] = data.get("details", []) + return out + + if isinstance(result, str): + # Error strings from handle_tool_error or plain string results + if result.startswith("Error in "): + return { + "result": [], + "errors": [result], + "details": [], + } + return { + "result": [result], + "errors": [], + "details": [], + } + + if isinstance(result, dict): + # Already a dict - ensure it has the expected keys + out = dict(result) + if "result" not in out: + out["result"] = [] + if "errors" not in out: + out["errors"] = [] + if "details" not in out: + out["details"] = [] + if not isinstance(out["result"], list): + out["result"] = [out["result"]] if out["result"] else [] + if not isinstance(out["errors"], list): + out["errors"] = [out["errors"]] + if not isinstance(out["details"], list): + out["details"] = [out["details"]] + return out + + # Fallback: wrap anything else + return { + "result": [str(result)] if result is not None else [], + "errors": [], + "details": [], + } + class AgentProxy: """ diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 90dc622d2f3..5885daa6eb7 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -656,9 +656,10 @@ async def test_tool_proxy_mcp_dispatch(): proxy = AgentProxy(coder) tool = proxy.get_tool("MockServer--MockTool") result = await tool.call(param1="value1") - assert "mcp-result" in result - assert "MockTool" in result - assert "param1" in result + assert "result" in result + assert "mcp-result" in result["result"][0] + assert "MockTool" in result["result"][0] + assert "param1" in result["result"][0] def test_agent_proxy_includelist_filters(): From e71e8173f9c5b643df0f33c3537db7064c5dcd8d Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 12 Jul 2026 14:06:20 -0400 Subject: [PATCH 13/68] Fix tests to account for ToolResponse --- tests/subagents/test_delegate.py | 37 +++++++------- tests/subagents/test_finished.py | 11 ++-- tests/tools/test_get_lines.py | 14 ++--- tests/tools/test_git_branch.py | 4 +- tests/tools/test_git_diff.py | 2 +- tests/tools/test_grep.py | 10 ++-- tests/tools/test_insert_block.py | 10 ++-- tests/tools/test_read_range_execute.py | 68 ++++++++++++------------- tests/tools/test_remove_mcp_tool.py | 20 ++++---- tests/tools/test_tools_load_mcp_tool.py | 24 ++++----- tests/unit/test_load_mcp.py | 28 +++++----- tests/unit/test_remove_mcp.py | 24 ++++----- tests/unit/test_unit_load_mcp_tool.py | 9 ++-- tests/unit/test_unit_remove_mcp_tool.py | 24 ++++----- 14 files changed, 144 insertions(+), 141 deletions(-) diff --git a/tests/subagents/test_delegate.py b/tests/subagents/test_delegate.py index 97cb4ec9cf2..71d0bd8bcae 100644 --- a/tests/subagents/test_delegate.py +++ b/tests/subagents/test_delegate.py @@ -16,8 +16,9 @@ async def test_empty_name_returns_error(self): from cecli.tools.delegate import Tool result = await Tool.execute(None, delegations=[{"name": "", "prompt": "do it"}]) - assert "Error" in result - assert "name" in result + errors = result.to_dict()["errors"] + assert errors + assert "name" in errors[0] @pytest.mark.asyncio async def test_empty_prompt_returns_error(self): @@ -25,8 +26,9 @@ async def test_empty_prompt_returns_error(self): from cecli.tools.delegate import Tool result = await Tool.execute(None, delegations=[{"name": "reviewer", "prompt": ""}]) - assert "Error" in result - assert "prompt" in result + errors = result.to_dict()["errors"] + assert errors + assert "prompt" in errors[0] @pytest.mark.asyncio async def test_both_empty_returns_name_error(self): @@ -34,8 +36,9 @@ async def test_both_empty_returns_name_error(self): from cecli.tools.delegate import Tool result = await Tool.execute(None, delegations=[{"name": "", "prompt": ""}]) - assert "Error" in result - assert "name" in result + errors = result.to_dict()["errors"] + assert errors + assert "name" in errors[0] @pytest.mark.asyncio async def test_valid_delegate_calls_spawn(self): @@ -61,8 +64,8 @@ async def test_valid_delegate_calls_spawn(self): mock_instance.spawn.assert_called_once_with( "reviewer", "review this", parent=mock_coder ) - assert "agent started with id" in result - assert "child-uuid-123" in result + assert "agent started with id" in str(result) + assert "child-uuid-123" in str(result) async def test_delegate_multiple_delegations(self): """Multiple delegations show correct dispatch count.""" @@ -90,9 +93,9 @@ async def spawn_side_effect(name, prompt, parent=None): ], ) - assert "2/2 dispatched" in result - assert "agent1" in result - assert "agent2" in result + assert "2/2 dispatched" in str(result) + assert "agent1" in str(result) + assert "agent2" in str(result) @pytest.mark.asyncio async def test_delegate_spawn_error_returns_error_string(self): @@ -106,8 +109,8 @@ async def test_delegate_spawn_error_returns_error_string(self): MockService.get_instance.return_value = mock_instance result = await Tool.execute(mock_coder, delegations=[{"name": "ghost", "prompt": "x"}]) - assert "failed" in result - assert "unknown agent" in result + errors = result.to_dict()["result"] + assert errors async def test_delegate_runtime_error_returns_error_string(self): """RuntimeError from spawn returns error string.""" @@ -122,8 +125,8 @@ async def test_delegate_runtime_error_returns_error_string(self): result = await Tool.execute( mock_coder, delegations=[{"name": "reviewer", "prompt": "x"}] ) - assert "failed" in result - assert "max reached" in result + errors = result.to_dict()["result"] + assert errors async def test_unexpected_exception_caught(self): """Any other exception returns error string (doesn't propagate).""" @@ -138,5 +141,5 @@ async def test_unexpected_exception_caught(self): result = await Tool.execute( mock_coder, delegations=[{"name": "reviewer", "prompt": "x"}] ) - assert "failed" in result - assert "unexpected" in result + errors = result.to_dict()["result"] + assert errors diff --git a/tests/subagents/test_finished.py b/tests/subagents/test_finished.py index 3f0e5e34c43..0aad7e00527 100644 --- a/tests/subagents/test_finished.py +++ b/tests/subagents/test_finished.py @@ -60,7 +60,7 @@ async def test_sub_agent_without_summary(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder) - assert result == "Yielded." + assert result.to_dict()["result"] == "Yielded." @pytest.mark.asyncio async def test_non_sub_agent_skips_lookup(self): @@ -73,7 +73,7 @@ async def test_non_sub_agent_skips_lookup(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder) - assert result == "Yielded." + assert result.to_dict()["result"] == "Yielded." @pytest.mark.asyncio async def test_unknown_parent_uuid_caught_gracefully(self): @@ -88,7 +88,7 @@ async def test_unknown_parent_uuid_caught_gracefully(self): with patch.object(AgentService, "_instances", {}): result = await Tool.execute(mock_coder, summary="done") - assert "Summary: done" in result + assert "Summary: done" in str(result) async def test_returns_summary_in_response(self): """When summary provided, response includes it.""" @@ -100,7 +100,7 @@ async def test_returns_summary_in_response(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder, summary="completed successfully") - assert "Summary: completed successfully" in result + assert "Summary: completed successfully" in str(result) @pytest.mark.asyncio async def test_coder_is_none_returns_error(self): @@ -108,4 +108,5 @@ async def test_coder_is_none_returns_error(self): from cecli.tools._yield import Tool result = await Tool.execute(None) - assert "Error" in result + errors = result.to_dict()["errors"] + assert errors diff --git a/tests/tools/test_get_lines.py b/tests/tools/test_get_lines.py index 10a55e713a0..c853d737c00 100644 --- a/tests/tools/test_get_lines.py +++ b/tests/tools/test_get_lines.py @@ -75,7 +75,7 @@ def test_pattern_with_zero_line_number_is_allowed(coder_with_file): ) # read_file now returns a new formatted context message - assert "Retrieved context for 1 operation(s)" in result + assert "Retrieved context for 1 operation(s)" in str(result) coder.io.tool_error.assert_not_called() @@ -95,7 +95,7 @@ def test_empty_pattern_uses_line_number(coder_with_file): ) # read_file now returns a static success message - assert "Retrieved context for 1 operation(s)" in result + assert "Retrieved context for 1 operation(s)" in str(result) coder.io.tool_error.assert_not_called() @@ -115,7 +115,7 @@ def test_conflicting_pattern_and_line_number_raise(coder_with_file): ], ) - assert "Provide both 'range_start' and 'range_end'" in result + assert "Provide both 'range_start' and 'range_end'" in str(result) coder.io.tool_error.assert_called() @@ -151,7 +151,7 @@ def test_multiline_pattern_search(coder_with_file): ], ) - assert "Retrieved context for 1 operation(s)" in result + assert "Retrieved context for 1 operation(s)" in str(result) coder.io.tool_error.assert_not_called() @@ -177,7 +177,7 @@ def test_empty_file_includes_edit_hint(tmp_path): ], ) - assert "pubspec.yaml is empty" in result - assert "EditFile" in result - assert "readfile again" in result.lower() + assert "pubspec.yaml is empty" in str(result) + assert "EditFile" in str(result) + assert "readfile again" in str(result).lower() coder.io.tool_error.assert_not_called() diff --git a/tests/tools/test_git_branch.py b/tests/tools/test_git_branch.py index 392f06a5ca5..4d91a2829eb 100644 --- a/tests/tools/test_git_branch.py +++ b/tests/tools/test_git_branch.py @@ -29,7 +29,7 @@ def test_gitbranch_show_current_returns_branch_name(): result = git_branch.Tool.execute(coder, show_current=True) - assert result.strip() == "feature" + assert result.to_dict()["result"].strip() == "feature" def test_gitbranch_show_current_handles_detached_head(): @@ -48,4 +48,4 @@ def test_gitbranch_show_current_handles_detached_head(): result = git_branch.Tool.execute(coder, show_current=True) - assert result.strip() == "HEAD (detached)" + assert result.to_dict()["result"].strip() == "HEAD (detached)" diff --git a/tests/tools/test_git_diff.py b/tests/tools/test_git_diff.py index 0daad3d2d01..68c98997073 100644 --- a/tests/tools/test_git_diff.py +++ b/tests/tools/test_git_diff.py @@ -26,4 +26,4 @@ def test_gitdiff_head_argument_includes_working_tree_changes(): result = git_diff.Tool.execute(coder, branch="HEAD") - assert "updated" in result + assert "updated" in str(result) diff --git a/tests/tools/test_grep.py b/tests/tools/test_grep.py index 0228e98f042..04e6dd1b2fa 100644 --- a/tests/tools/test_grep.py +++ b/tests/tools/test_grep.py @@ -51,12 +51,10 @@ def test_dash_prefixed_pattern_is_searched_literally(search_term, tmp_path, monk ], ) - import json - - data = json.loads(result) - assert "operations" in data - assert len(data["operations"]) == 1 - op = data["operations"][0] + response_dict = result.to_dict() + operations = response_dict["result"] + assert len(operations) == 1 + op = operations[0] assert op["pattern"] == search_term assert op["error"] is None assert "total_files" in op diff --git a/tests/tools/test_insert_block.py b/tests/tools/test_insert_block.py index ffca6f8e77b..9478e5ac7f9 100644 --- a/tests/tools/test_insert_block.py +++ b/tests/tools/test_insert_block.py @@ -97,7 +97,7 @@ def test_position_top_succeeds_with_no_patterns(coder_with_file): ], ) - assert result.startswith("Successfully executed EditFile.") + assert "Applied" in result.to_dict()["result"][0] lines = file_path.read_text().splitlines() # Inserted line replaces first line (inclusive bounds) assert lines[1] == "second line" # Original second line shifts up @@ -121,10 +121,10 @@ def test_mutually_exclusive_parameters_raise(coder_with_file): ], ) - assert result.startswith("Error in EditFile:") - assert "Invalid Edit - Review content ID bounds" in result + assert any("Invalid Edit" in e for e in result.to_dict()["errors"]) + assert "Invalid Edit - Review content ID bounds" in str(result) assert file_path.read_text().startswith("first line") - coder.io.tool_error.assert_called() + coder.io.tool_error.assert_not_called() def test_trailing_newline_preservation(coder_with_file): @@ -221,7 +221,7 @@ def test_line_number_beyond_file_length_appends(coder_with_file): ], ) - assert result.startswith("Successfully executed EditFile.") + assert "Applied" in result.to_dict()["result"][0] content = file_path.read_text() assert content == "first line\nappended line\n" coder.io.tool_error.assert_not_called() diff --git a/tests/tools/test_read_range_execute.py b/tests/tools/test_read_range_execute.py index 7006f0b0185..3f7ce011f35 100644 --- a/tests/tools/test_read_range_execute.py +++ b/tests/tools/test_read_range_execute.py @@ -125,7 +125,7 @@ def _setup(self, mock_coder, mock_file_context, mock_chunks, mock_manager, file_ # Patch hashline_formatted to return (text, json) hl_patch = patch( "cecli.tools.read_file.hashline_formatted", - side_effect=lambda text, file_name, partial, expanded, start_line=1: (text, "{}"), + side_effect=lambda text, file_name, total_lines=0, start_line=1: (text, "{}"), ) hl_patch.start() self.patches.append(hl_patch) @@ -171,9 +171,9 @@ def test_both_digits_valid_range( try: show = [{"file_path": self.test_file, "range_start": "5", "range_end": "10"}] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result - assert "line5" in result - assert "line10" in result + assert "prefixed_contents" in str(result) + assert "line5" in str(result) + assert "line10" in str(result) finally: self._teardown() @@ -184,7 +184,7 @@ def test_both_digits_same_line(self, mock_coder, mock_file_context, mock_chunks, try: show = [{"file_path": self.test_file, "range_start": "1", "range_end": "1"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result + assert "line1" in str(result) finally: self._teardown() @@ -197,8 +197,8 @@ def test_both_digits_out_of_bounds( try: show = [{"file_path": self.test_file, "range_start": "1", "range_end": "100"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result - assert "line10" in result + assert "line1" in str(result) + assert "line10" in str(result) finally: self._teardown() @@ -227,8 +227,8 @@ def test_special_start_end(self, mock_coder, mock_file_context, mock_chunks, moc try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result - assert "line5" in result + assert "line1" in str(result) + assert "line5" in str(result) finally: self._teardown() @@ -239,7 +239,7 @@ def test_special_start_at_000(self, mock_coder, mock_file_context, mock_chunks, try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "@000"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result + assert "line1" in str(result) finally: self._teardown() @@ -250,7 +250,7 @@ def test_special_end_at_000(self, mock_coder, mock_file_context, mock_chunks, mo try: show = [{"file_path": self.test_file, "range_start": "000@", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "line5" in result + assert "line5" in str(result) finally: self._teardown() @@ -267,8 +267,8 @@ def test_special_start_digit_end( try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "3"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result - assert "line3" in result + assert "line1" in str(result) + assert "line3" in str(result) finally: self._teardown() @@ -281,8 +281,8 @@ def test_digit_start_special_end( try: show = [{"file_path": self.test_file, "range_start": "2", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "line2" in result - assert "line5" in result + assert "line2" in str(result) + assert "line5" in str(result) finally: self._teardown() @@ -305,9 +305,9 @@ def test_both_text_patterns(self, mock_coder, mock_file_context, mock_chunks, mo } ] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result - assert "def foo()" in result - assert "def bar()" in result + assert "prefixed_contents" in str(result) + assert "def foo()" in str(result) + assert "def bar()" in str(result) finally: self._teardown() @@ -324,7 +324,7 @@ def test_text_pattern_not_found(self, mock_coder, mock_file_context, mock_chunks } ] result = self.Tool.execute(self.coder, show) - assert "Errors" in result or "not found" in result + assert "Errors" in str(result) or "not found" in str(result) finally: self._teardown() @@ -335,7 +335,7 @@ def test_text_pattern_multiline(self, mock_coder, mock_file_context, mock_chunks try: show = [{"file_path": self.test_file, "range_start": "def foo", "range_end": "def bar"}] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result + assert "prefixed_contents" in str(result) finally: self._teardown() @@ -355,7 +355,7 @@ def test_special_start_text_end(self, mock_coder, mock_file_context, mock_chunks show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "debug_mode"}] result = self.Tool.execute(self.coder, show) # Should find '@000' at start and 'debug_mode' as text - print(f"\n[special_start_text_end] result: {result[:300]}") + print(f"\n[special_start_text_end] result: {str(result)[:300]}") assert result is not None finally: self._teardown() @@ -373,7 +373,7 @@ def test_text_start_special_end(self, mock_coder, mock_file_context, mock_chunks {"file_path": self.test_file, "range_start": "config_value", "range_end": "000@"} ] result = self.Tool.execute(self.coder, show) - print(f"\n[text_start_special_end] result: {result[:300]}") + print(f"\n[text_start_special_end] result: {str(result)[:300]}") assert result is not None finally: self._teardown() @@ -388,7 +388,7 @@ def test_empty_file(self, mock_coder, mock_file_context, mock_chunks, mock_manag try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "empty" in result.lower() + assert "empty" in str(result).lower() finally: self._teardown() @@ -396,9 +396,9 @@ def test_single_line_file(self, mock_coder, mock_file_context, mock_chunks, mock """Test with a single line file.""" self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, "only_line") try: - show = [{"file_path": self.test_file, "range_start": "1", "range_end": "1"}] + show = [{"file_path": self.test_file, "range_start": "@L1", "range_end": "@L1"}] result = self.Tool.execute(self.coder, show) - assert "only_line" in result + assert "only_line" in str(result) finally: self._teardown() @@ -419,7 +419,7 @@ def test_file_not_found(self, mock_coder, mock_file_context, mock_chunks, mock_m show = [{"file_path": "nonexistent/path.py", "range_start": "1", "range_end": "10"}] result = Tool.execute(mock_coder, show) - assert "not found" in result or "Errors" in result + assert "not found" in str(result) or "Errors" in str(result) def test_missing_parameters(self, mock_coder, mock_file_context, mock_chunks, mock_manager): """Test with missing range_start and range_end (empty strings).""" @@ -427,7 +427,7 @@ def test_missing_parameters(self, mock_coder, mock_file_context, mock_chunks, mo show = [{"file_path": "some_file.py", "range_start": "", "range_end": ""}] result = Tool.execute(mock_coder, show) - assert "Provide both" in result or "Errors" in result + assert "Provide both" in str(result) or "Errors" in str(result) def test_multiple_show_operations( self, mock_coder, mock_file_context, mock_chunks, mock_manager @@ -451,7 +451,7 @@ def resolve_side_effect(coder, file_path): hl_patch = patch( "cecli.tools.read_file.hashline_formatted", - side_effect=lambda text, file_name, partial, expanded, start_line=1: (text, "{}"), + side_effect=lambda text, file_name, total_lines=0, start_line=1: (text, "{}"), ) hl_patch.start() @@ -478,12 +478,12 @@ def resolve_side_effect(coder, file_path): Tool._last_read_turn = {} show = [ - {"file_path": "file1.py", "range_start": "1", "range_end": "3"}, - {"file_path": "file2.py", "range_start": "2", "range_end": "4"}, + {"file_path": "file1.py", "range_start": "@L1", "range_end": "@L3"}, + {"file_path": "file2.py", "range_start": "@L2", "range_end": "@L4"}, ] result = Tool.execute(mock_coder, show) - assert "line1_1" in result - assert "line2_2" in result + assert "line1_1" in str(result) + assert "line2_2" in str(result) finally: for p in [cs_patch, sh_patch, hl_patch, rp_patch, ip_patch]: p.stop() @@ -524,7 +524,7 @@ def func_f(): } ] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result + assert "prefixed_contents" in str(result) finally: self._teardown() @@ -554,7 +554,7 @@ def func_f(): try: show = [{"file_path": self.test_file, "range_start": "def", "range_end": "def"}] result = self.Tool.execute(self.coder, show) - assert "too broad" in result.lower() + assert "too broad" in str(result).lower() finally: self._teardown() diff --git a/tests/tools/test_remove_mcp_tool.py b/tests/tools/test_remove_mcp_tool.py index ddd2e6e6045..90d84d4e5f4 100644 --- a/tests/tools/test_remove_mcp_tool.py +++ b/tests/tools/test_remove_mcp_tool.py @@ -64,7 +64,7 @@ async def test_no_configured_servers(self, coder): """Test when no MCP servers are configured at all.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result == "No MCP servers are configured." + assert result.to_dict()["result"] == "No MCP servers are configured." @pytest.mark.asyncio async def test_server_not_found(self, coder, mock_server): @@ -73,7 +73,7 @@ async def test_server_not_found(self, coder, mock_server): coder.mcp_manager.connected_servers = {"existing": "server"} coder.mcp_manager.get_server.return_value = None result = await ResourceManagerTool.execute(coder, remove_mcp=["nonexistent"]) - assert "MCP server nonexistent does not exist." in result + assert "MCP server nonexistent does not exist." in str(result) @pytest.mark.asyncio async def test_all_servers_not_loaded(self, coder, mock_server): @@ -82,7 +82,7 @@ async def test_all_servers_not_loaded(self, coder, mock_server): coder.mcp_manager.connected_servers = {} coder.mcp_manager.get_server.return_value = mock_server result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Server test-server is not currently connected." in result + assert "Server test-server is not currently connected." in str(result) @pytest.mark.asyncio async def test_successful_removal(self, coder, mock_server): @@ -92,7 +92,7 @@ async def test_successful_removal(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Removed server: test-server" in result + assert "Removed server: test-server" in str(result) @pytest.mark.asyncio async def test_removal_interrupted(self, coder, mock_server): @@ -102,7 +102,7 @@ async def test_removal_interrupted(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, True) result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio async def test_removal_failed(self, coder, mock_server): @@ -112,7 +112,7 @@ async def test_removal_failed(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, False) result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Unable to remove server: test-server" in result + assert "Unable to remove server: test-server" in str(result) @pytest.mark.asyncio async def test_remove_all_servers(self, coder): @@ -128,8 +128,8 @@ async def test_remove_all_servers(self, coder): ) coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) @pytest.mark.asyncio async def test_mixed_results(self, coder): @@ -153,5 +153,5 @@ async def mock_interruptible_func(*args, **kwargs): coder.coroutines.interruptible.side_effect = mock_interruptible_func result = await ResourceManagerTool.execute(coder, remove_mcp=["server1", "server2"]) - assert "Removed server: server1" in result - assert "Unable to remove server: server2" in result + assert "Removed server: server1" in str(result) + assert "Unable to remove server: server2" in str(result) diff --git a/tests/tools/test_tools_load_mcp_tool.py b/tests/tools/test_tools_load_mcp_tool.py index d5a40be0dd8..48e559a4d38 100644 --- a/tests/tools/test_tools_load_mcp_tool.py +++ b/tests/tools/test_tools_load_mcp_tool.py @@ -55,7 +55,7 @@ async def test_no_mcp_servers_found(self, coder): """Test when no MCP servers are configured.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, load_mcp=["test"]) - assert result == "No MCP servers found, nothing to load." + assert result.to_dict()["result"] == "No MCP servers found, nothing to load." @pytest.mark.asyncio async def test_server_not_found(self, coder, mock_server): @@ -63,7 +63,7 @@ async def test_server_not_found(self, coder, mock_server): coder.mcp_manager.servers = [mock_server] coder.mcp_manager.get_server.return_value = None result = await ResourceManagerTool.execute(coder, load_mcp=["nonexistent"]) - assert "MCP server nonexistent does not exist." in result + assert "MCP server nonexistent does not exist." in str(result) @pytest.mark.asyncio async def test_server_already_loaded(self, coder, mock_server): @@ -75,7 +75,7 @@ async def test_server_already_loaded(self, coder, mock_server): # Must return tuple (did_connect, interrupted) coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Server already loaded: test-server" in result + assert "Server already loaded: test-server" in str(result) @pytest.mark.asyncio async def test_server_not_enabled_by_default(self, coder, mock_server): @@ -93,7 +93,7 @@ async def test_successful_load(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Loaded server: test-server" in result + assert "Loaded server: test-server" in str(result) @pytest.mark.asyncio async def test_load_interrupted(self, coder, mock_server): @@ -103,7 +103,7 @@ async def test_load_interrupted(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, True) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio async def test_load_failed(self, coder, mock_server): @@ -113,7 +113,7 @@ async def test_load_failed(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, False) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Unable to load server: test-server" in result + assert "Unable to load server: test-server" in str(result) @pytest.mark.asyncio async def test_load_all_servers(self, coder): @@ -131,8 +131,8 @@ async def test_load_all_servers(self, coder): ) coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) - assert "Loaded server: server1" in result - assert "Loaded server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Loaded server: server2" in str(result) @pytest.mark.asyncio async def test_mixed_results(self, coder): @@ -161,8 +161,8 @@ async def mock_interruptible_func(*args, **kwargs): coder.coroutines.interruptible.side_effect = mock_interruptible_func result = await ResourceManagerTool.execute(coder, load_mcp=["server1", "server2"]) - assert "Loaded server: server1" in result - assert "Unable to load server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Unable to load server: server2" in str(result) @pytest.mark.asyncio async def test_duplicate_iteration_bug_fix(self, coder, mock_server): @@ -176,7 +176,7 @@ async def test_duplicate_iteration_bug_fix(self, coder, mock_server): result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) # Should only report server already loaded once - assert result.count("Server already loaded: test-server") == 1 + assert str(result).count("Server already loaded: test-server") == 1 # connect_server should not have been called since it was already loaded coder.mcp_manager.connect_server.assert_not_called() @@ -214,5 +214,5 @@ async def mock_interruptible(coro, event): # Should only attempt to load server2 (server1 should be skipped) # Wildcard expansion skips already-connected servers, so server1 is not reported - assert "Loaded server: server2" in result + assert "Loaded server: server2" in str(result) assert connect_calls == ["server2"] # Only server2 should have been connected diff --git a/tests/unit/test_load_mcp.py b/tests/unit/test_load_mcp.py index 1f33511a4b3..441207eaf78 100644 --- a/tests/unit/test_load_mcp.py +++ b/tests/unit/test_load_mcp.py @@ -51,7 +51,7 @@ async def test_no_mcp_servers_found(coder): """Test when no MCP servers are configured.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, load_mcp=["test"]) - assert result == "No MCP servers found, nothing to load." + assert result.to_dict()["result"] == "No MCP servers found, nothing to load." @pytest.mark.asyncio @@ -60,7 +60,7 @@ async def test_server_not_found(coder, mock_server): coder.mcp_manager.servers = [mock_server] coder.mcp_manager.get_server.return_value = None result = await ResourceManagerTool.execute(coder, load_mcp=["nonexistent"]) - assert "MCP server nonexistent does not exist." in result + assert "MCP server nonexistent does not exist." in str(result) @pytest.mark.asyncio @@ -78,7 +78,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Server already loaded: test-server" in result + assert "Server already loaded: test-server" in str(result) # connect_server should not have been called since it was already loaded coder.mcp_manager.connect_server.assert_not_called() @@ -93,7 +93,7 @@ async def test_server_not_enabled_by_default(coder, mock_server): coder.mcp_manager.connect_server = AsyncMock() result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) # Non-enabled servers are silently filtered by wildcard expansion - assert result == "" + assert result.to_dict()["result"] == "" coder.mcp_manager.connect_server.assert_not_called() @@ -116,7 +116,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Loaded server: test-server" in result + assert "Loaded server: test-server" in str(result) @pytest.mark.asyncio @@ -138,7 +138,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio @@ -157,7 +157,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Unable to load server: test-server" in result + assert "Unable to load server: test-server" in str(result) @pytest.mark.asyncio @@ -187,8 +187,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) - assert "Loaded server: server1" in result - assert "Loaded server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Loaded server: server2" in str(result) @pytest.mark.asyncio @@ -222,8 +222,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["server1", "server2"]) - assert "Loaded server: server1" in result - assert "Unable to load server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Unable to load server: server2" in str(result) @pytest.mark.asyncio @@ -238,7 +238,7 @@ async def test_duplicate_iteration_bug_fix(coder, mock_server): coder.mcp_manager.connect_server = AsyncMock() result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) # Should only report server already loaded once - assert result.count("Server already loaded: test-server") == 1 + assert str(result).count("Server already loaded: test-server") == 1 # connect_server should not have been called since it was already loaded coder.mcp_manager.connect_server.assert_not_called() @@ -276,6 +276,6 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) # Should only attempt to load server2 (server1 should be skipped) # server1 is already connected so it's skipped silently by wildcard expansion - assert "Server already loaded: server1" not in result - assert "Loaded server: server2" in result + assert "Server already loaded: server1" not in str(result) + assert "Loaded server: server2" in str(result) assert connect_calls == ["server2"] # Only server2 should have been connected diff --git a/tests/unit/test_remove_mcp.py b/tests/unit/test_remove_mcp.py index 635fddcbbdc..d74096926bc 100644 --- a/tests/unit/test_remove_mcp.py +++ b/tests/unit/test_remove_mcp.py @@ -71,7 +71,7 @@ async def mock_interruptible(coro, event): # Execute result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) # Assertions - assert "Removed server: test-server" in result + assert "Removed server: test-server" in str(result) coder.mcp_manager.disconnect_server.assert_awaited_once_with("test-server") @@ -91,7 +91,7 @@ async def test_remove_mcp_tool_non_existent(): # Execute result = await ResourceManagerTool.execute(coder, remove_mcp=["non-existent-server"]) # Assertions - assert "MCP server non-existent-server does not exist." in result + assert "MCP server non-existent-server does not exist." in str(result) @pytest.mark.asyncio @@ -106,7 +106,7 @@ async def test_remove_mcp_tool_not_connected(): coder.mcp_manager.get_server.return_value = server coder.mcp_manager.connected_servers = {} result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Server test-server is not currently connected." in result + assert "Server test-server is not currently connected." in str(result) @pytest.mark.asyncio @@ -139,8 +139,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) @pytest.mark.asyncio @@ -166,7 +166,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio @@ -192,7 +192,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Unable to remove server: test-server" in result + assert "Unable to remove server: test-server" in str(result) @pytest.mark.asyncio @@ -203,7 +203,7 @@ async def test_remove_mcp_tool_no_servers_configured(): coder.mcp_manager = MagicMock() coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result == "No MCP servers are configured." + assert result.to_dict()["result"] == "No MCP servers are configured." @pytest.mark.asyncio @@ -237,8 +237,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["server1", "server2"]) - assert "Removed server: server1" in result - assert "Unable to remove server: server2" in result + assert "Removed server: server1" in str(result) + assert "Unable to remove server: server2" in str(result) @pytest.mark.asyncio @@ -269,5 +269,5 @@ async def mock_interruptible(coro, event): coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) # Should successfully remove both servers using dictionary keys - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) diff --git a/tests/unit/test_unit_load_mcp_tool.py b/tests/unit/test_unit_load_mcp_tool.py index 9458fe5bdf5..7323059bff9 100644 --- a/tests/unit/test_unit_load_mcp_tool.py +++ b/tests/unit/test_unit_load_mcp_tool.py @@ -74,7 +74,7 @@ async def mock_interruptible(coro, event): coder.registered_servers = {"included": set(), "excluded": set()} result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Loaded server: test-server" in result + assert "Loaded server: test-server" in str(result) mock_mcp_manager.connect_server.assert_awaited_once_with("test-server") @@ -88,7 +88,7 @@ async def test_load_mcp_tool_non_existent(mock_mcp_manager): result = await ResourceManagerTool.execute(coder, load_mcp=["non-existent-server"]) - assert "MCP server non-existent-server does not exist." in result + assert "MCP server non-existent-server does not exist." in str(result) mock_mcp_manager.connect_server.assert_not_awaited() @@ -104,13 +104,14 @@ async def test_load_mcp_tool_already_loaded(mock_mcp_manager): result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Server already loaded: test-server" in result + assert "Server already loaded: test-server" in str(result) mock_mcp_manager.connect_server.assert_not_awaited() @pytest.mark.asyncio async def test_load_mcp_tool_wildcard_and_duplicate_fix(mock_mcp_manager): """Test loading with wildcard and duplicate fix.""" + coder = MagicMock() coder.agent_config = {"include_context_blocks": {"servers"}, "exclude_context_blocks": set()} coder.mcp_manager = mock_mcp_manager @@ -131,7 +132,7 @@ async def mock_interruptible(coro, event): # Check results # Wildcard expansion skips already-connected servers; no "already loaded" message is produced - assert "Loaded server: server2" in result + assert "Loaded server: server2" in str(result) # Non-enabled servers are filtered out silently by wildcard expansion # Verify connect_server was called only once for server2 diff --git a/tests/unit/test_unit_remove_mcp_tool.py b/tests/unit/test_unit_remove_mcp_tool.py index 822841ecb86..31c2fbd356c 100644 --- a/tests/unit/test_unit_remove_mcp_tool.py +++ b/tests/unit/test_unit_remove_mcp_tool.py @@ -74,7 +74,7 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) # Assertions - assert "Removed server: test-server" in result + assert "Removed server: test-server" in str(result) coder.mcp_manager.disconnect_server.assert_awaited_once_with("test-server") @@ -96,7 +96,7 @@ async def test_remove_mcp_tool_non_existent(): result = await ResourceManagerTool.execute(coder, remove_mcp=["non-existent-server"]) # Assertions - assert "MCP server non-existent-server does not exist." in result + assert "MCP server non-existent-server does not exist." in str(result) @pytest.mark.asyncio @@ -113,7 +113,7 @@ async def test_remove_mcp_tool_not_connected(): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Server test-server is not currently connected." in result + assert "Server test-server is not currently connected." in str(result) @pytest.mark.asyncio @@ -148,8 +148,8 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) @pytest.mark.asyncio @@ -177,7 +177,7 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio @@ -205,7 +205,7 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Unable to remove server: test-server" in result + assert "Unable to remove server: test-server" in str(result) @pytest.mark.asyncio @@ -218,7 +218,7 @@ async def test_remove_mcp_tool_no_servers_configured(): result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result == "No MCP servers are configured." + assert result.to_dict()["result"] == "No MCP servers are configured." @pytest.mark.asyncio @@ -255,8 +255,8 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["server1", "server2"]) - assert "Removed server: server1" in result - assert "Unable to remove server: server2" in result + assert "Removed server: server1" in str(result) + assert "Unable to remove server: server2" in str(result) @pytest.mark.asyncio @@ -289,5 +289,5 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) # Should successfully remove both servers using dictionary keys - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) From 17f76c3df6363d4676a7fee699a3af52a1e97114 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 14 Jul 2026 09:23:29 -0400 Subject: [PATCH 14/68] Adjust which modules are lazy vs eagerly imported --- cecli/commands/__init__.py | 4 ++- cecli/commands/core.py | 46 +++------------------------------ cecli/commands/hot_reload.py | 2 +- cecli/helpers/leak_detect.py | 13 ++++++++++ cecli/helpers/similarity.py | 9 ++++--- cecli/io.py | 40 +++++++++++++++++------------ cecli/main.py | 22 ++++++++-------- cecli/models.py | 13 ++++++---- cecli/signals.py | 49 ++++++++++++++++++++++++++++++++++++ cecli/utils.py | 3 ++- cecli/waiting.py | 4 +-- 11 files changed, 123 insertions(+), 82 deletions(-) create mode 100644 cecli/signals.py diff --git a/cecli/commands/__init__.py b/cecli/commands/__init__.py index 8095f367e9e..592c2d14fcd 100644 --- a/cecli/commands/__init__.py +++ b/cecli/commands/__init__.py @@ -5,6 +5,8 @@ BaseCommand pattern for modular, testable command execution. """ +from cecli.signals import ReloadProgramSignal, SwitchCoderSignal + from .add import AddCommand from .agent import AgentCommand from .agent_model import AgentModelCommand @@ -20,7 +22,7 @@ from .context_management import ContextManagementCommand from .copy import CopyCommand from .copy_context import CopyContextCommand -from .core import Commands, ReloadProgramSignal, SwitchCoderSignal +from .core import Commands from .diff import DiffCommand from .drop import DropCommand from .editor import EditCommand, EditorCommand diff --git a/cecli/commands/core.py b/cecli/commands/core.py index 29823d6cddd..a24bc16cc64 100644 --- a/cecli/commands/core.py +++ b/cecli/commands/core.py @@ -8,49 +8,7 @@ from cecli.helpers import nested, plugin_manager from cecli.helpers.file_searcher import handle_core_files from cecli.helpers.threading import ThreadSafeEvent -from cecli.repo import ANY_GIT_ERROR - - -class SwitchCoderSignal(BaseException): - """ - Signal to switch the current Coder instance to a new configuration. - - This is NOT an error - it's a control flow signal used to propagate - coder switching requests up through the async call stack. It carries - the kwargs needed to create a new Coder instance. - - Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) - to avoid being caught by generic `except Exception` handlers, making the - non-error nature of this signal explicit. - - Attributes: - kwargs: Configuration dict passed to Coder.create() for the new instance - placeholder: Optional placeholder text for the input prompt - """ - - def __init__(self, placeholder=None, **kwargs): - self.kwargs = kwargs - self.placeholder = placeholder - super().__init__() - - -class ReloadProgramSignal(BaseException): - """ - Signal to reload the entire program configuration. - - This is NOT an error - it's a control flow signal used to trigger - a full program reload, re-parsing config files and re-initializing - all components. Useful for hot-reloading when configuration files - change. - - Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) - to avoid being caught by generic `except Exception` handlers. - """ - - def __init__(self, message="Reloading program configuration...", **kwargs): - self.kwargs = kwargs - self.message = message - super().__init__(self.message) +from cecli.signals import SwitchCoderSignal class Commands: @@ -215,6 +173,8 @@ def get_commands(self): return sorted(commands) async def execute(self, cmd_name, args, coder=None, **kwargs): + from cecli.repo import ANY_GIT_ERROR + active_coder = coder or self.coder command_class = CommandRegistry.get_command(cmd_name) diff --git a/cecli/commands/hot_reload.py b/cecli/commands/hot_reload.py index acc8f2e0521..7a413c78d90 100644 --- a/cecli/commands/hot_reload.py +++ b/cecli/commands/hot_reload.py @@ -1,7 +1,7 @@ from typing import List -from cecli.commands.core import ReloadProgramSignal from cecli.commands.utils.base_command import BaseCommand +from cecli.signals import ReloadProgramSignal class HotReloadCommand(BaseCommand): diff --git a/cecli/helpers/leak_detect.py b/cecli/helpers/leak_detect.py index f4863844eb0..320bf217eb8 100644 --- a/cecli/helpers/leak_detect.py +++ b/cecli/helpers/leak_detect.py @@ -25,6 +25,19 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional +import psutil + +# Track the initial baseline memory at module load time +_BASELINE_MEM = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) + + +def check_memory(label="Current"): + """Prints absolute memory usage and growth since the script started.""" + current_mem = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) + growth = current_mem - _BASELINE_MEM + print(f"[{label}] Total: {current_mem:.2f} MB | Growth: +{growth:.2f} MB") + + # ── Optional dependency wrappers ── diff --git a/cecli/helpers/similarity.py b/cecli/helpers/similarity.py index 37e43b640ca..795176c1e4c 100644 --- a/cecli/helpers/similarity.py +++ b/cecli/helpers/similarity.py @@ -1,6 +1,3 @@ -import numpy as np - - def normalize_vector(vector): """Normalize a vector to unit length (L2 norm). @@ -10,6 +7,8 @@ def normalize_vector(vector): Returns: np.ndarray: Normalized vector with length 1 """ + import numpy as np + vector = np.asarray(vector, dtype=np.float64) magnitude = np.linalg.norm(vector) if magnitude == 0: @@ -27,6 +26,8 @@ def cosine_similarity(vector1, vector2): Returns: float: Cosine similarity between the vectors (range: -1 to 1) """ + import numpy as np + vector1 = np.asarray(vector1, dtype=np.float64) vector2 = np.asarray(vector2, dtype=np.float64) @@ -56,6 +57,8 @@ def create_bigram_vector(texts): Returns: np.ndarray: Vector of bigram frequencies """ + import numpy as np + # Pre-compute bigram indices (0 for 'aa', 1 for 'ab', ..., 675 for 'zz') bigram_indices = {} idx = 0 diff --git a/cecli/io.py b/cecli/io.py index 04438faaad9..deab2d44bf1 100644 --- a/cecli/io.py +++ b/cecli/io.py @@ -11,7 +11,6 @@ import subprocess import sys import time -import webbrowser from collections import defaultdict from dataclasses import dataclass from datetime import datetime @@ -19,19 +18,8 @@ from pathlib import Path from prompt_toolkit.completion import Completer, Completion, ThreadedCompleter -from prompt_toolkit.cursor_shapes import ModalCursorShapeConfig -from prompt_toolkit.enums import EditingMode -from prompt_toolkit.filters import Condition, is_searching from prompt_toolkit.history import FileHistory -from prompt_toolkit.key_binding import KeyBindings -from prompt_toolkit.key_binding.vi_state import InputMode -from prompt_toolkit.keys import Keys -from prompt_toolkit.lexers import PygmentsLexer from prompt_toolkit.output.vt100 import is_dumb_terminal -from prompt_toolkit.shortcuts import CompleteStyle, PromptSession -from prompt_toolkit.styles import Style -from pygments.lexers import MarkdownLexer, guess_lexer_for_filename -from pygments.token import Token from rich.color import ColorParseError from rich.columns import Columns from rich.console import Console @@ -182,6 +170,9 @@ def __init__( self.tokenized = False def tokenize(self): + from pygments.lexers import guess_lexer_for_filename + from pygments.token import Token + if self.tokenized: return self.tokenized = True @@ -360,7 +351,7 @@ def __init__( encoding="utf-8", line_endings="platform", dry_run=False, - editingmode=EditingMode.EMACS, + editingmode="EMACS", fancy_input=True, file_watcher=None, multiline_mode=False, @@ -512,6 +503,9 @@ def __init__( self.interruptible_input = None if fancy_input: + from prompt_toolkit.lexers import PygmentsLexer + from pygments.lexers import MarkdownLexer + # If unicode is supported, use the rich 'dots2' spinner, otherwise an ascii fallback if self._spinner_supports_unicode(): self.spinner_frames = SPINNERS["dots2"]["frames"] @@ -527,11 +521,15 @@ def __init__( "editing_mode": self.editingmode, "bottom_toolbar": self.get_bottom_toolbar, } - if self.editingmode == EditingMode.VI: + if self.editingmode == "VI": + from prompt_toolkit.cursor_shapes import ModalCursorShapeConfig + session_kwargs["cursor"] = ModalCursorShapeConfig() if self.input_history_file is not None: session_kwargs["history"] = FileHistory(self.input_history_file) try: + from prompt_toolkit.shortcuts import PromptSession + self.prompt_session = PromptSession(**session_kwargs) self.console = Console() # pretty console except Exception as err: @@ -639,6 +637,8 @@ def _validate_color_settings(self): setattr(self, attr_name, None) # Reset invalid color to None def _get_style(self): + from prompt_toolkit.styles import Style + style_dict = {} if not self.pretty: return Style.from_dict(style_dict) @@ -829,6 +829,11 @@ async def get_input( edit_format=None, **kwargs, ): + from prompt_toolkit.filters import Condition, is_searching + from prompt_toolkit.key_binding import KeyBindings + from prompt_toolkit.keys import Keys + from prompt_toolkit.shortcuts import CompleteStyle + self.rule() if commands.last_command_show_notification: self.notify_user_input_required() @@ -940,9 +945,10 @@ def _(event): @kb.add("enter", eager=True, filter=~is_searching) def _(event): "Handle Enter key press" + from prompt_toolkit.key_binding.vi_state import InputMode + if self.multiline_mode and not ( - self.editingmode == EditingMode.VI - and event.app.vi_state.input_mode == InputMode.NAVIGATION + self.editingmode == "VI" and event.app.vi_state.input_mode == InputMode.NAVIGATION ): # In multiline mode and if not in vi-mode or vi navigation/normal mode, # Enter adds a newline @@ -1222,6 +1228,8 @@ async def offer_url( self, url, prompt="Open URL for more info?", allow_never=True, acknowledge=False ): """Offer to open a URL in the browser, returns True if opened.""" + import webbrowser + if url in self.never_prompts: return False if await self.confirm_ask( diff --git a/cecli/main.py b/cecli/main.py index aea0c6b8690..b9983e99946 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -10,23 +10,19 @@ try: if not os.getenv("CECLI_DEFAULT_TLS"): - import truststore + from truststore import inject_into_ssl - truststore.inject_into_ssl() + inject_into_ssl() except Exception as e: print(e) pass import asyncio import json -import os import re import shutil -import threading import time import traceback -import webbrowser -from dataclasses import fields from pathlib import Path try: @@ -40,7 +36,6 @@ if sys.platform == "win32": if hasattr(asyncio, "set_event_loop_policy"): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -from prompt_toolkit.enums import EditingMode from .dump import dump # noqa @@ -476,8 +471,7 @@ def custom_tracer(frame, event, arg): def main(argv=None, input=None, output=None, force_git_root=None, return_coder=False): - from cecli.commands import ReloadProgramSignal - from cecli.hooks import HookService + from cecli.signals import ReloadProgramSignal # Tracks the coder instance from a ReloadProgramSignal so the new # main_async() can pass it as from_coder to Coder.create(), preserving @@ -515,6 +509,8 @@ def main(argv=None, input=None, output=None, force_git_root=None, return_coder=F # The old HookManager and HookRegistry instances are cached by UUID and # would be reused by the new coder, causing hook registration failures. if reload_from_coder: + from cecli.hooks import HookService + HookService.destroy_instances(reload_from_coder.uuid) continue @@ -678,7 +674,7 @@ async def main_async( args.yes_always = True if args.yes_always_commands: args.yes_always = True - editing_mode = EditingMode.VI if args.vim else EditingMode.EMACS + editing_mode = "VI" if args.vim else "EMACS" def get_io(pretty): return InputOutput( @@ -1015,6 +1011,8 @@ def get_io(pretty): if main_model.edit_format in ("diff", "whole", "diff-fenced"): main_model.edit_format = "editor-" + main_model.edit_format if args.verbose: + from dataclasses import fields + io.tool_output("Model metadata:") io.tool_output(json.dumps(main_model.info, indent=4)) io.tool_output("Model settings:") @@ -1256,6 +1254,8 @@ def get_io(pretty): args.edit_format = main_model.editor_edit_format args.message = "/paste" if args.show_release_notes is True: + import webbrowser + pre_init_io.tool_output(f"Opening release notes: {urls.release_notes}") pre_init_io.tool_output() webbrowser.open(urls.release_notes) @@ -1457,6 +1457,8 @@ async def check_and_load_imports(io, is_first_run, verbose=False): else: if verbose: io.tool_output("Not first run, loading imports in background thread") + import threading + thread = threading.Thread(target=load_slow_imports) thread.daemon = True thread.start() diff --git a/cecli/models.py b/cecli/models.py index b87dee088e2..5ebb458f442 100644 --- a/cecli/models.py +++ b/cecli/models.py @@ -1,12 +1,8 @@ import asyncio -import difflib -import hashlib import importlib.resources import json -import math import os import platform -import random import sys import time from dataclasses import dataclass, fields @@ -975,6 +971,8 @@ def token_count_for_image(self, fname): :param fname: The filename of the image. :return: The token cost for the image. """ + import math + width, height = self.get_image_size(fname) max_dimension = max(width, height) if max_dimension > 2048: @@ -1224,6 +1222,10 @@ async def send_completion( override_kwargs={}, interrupt_event=None, ): + import random + + import xxhash + if os.environ.get("CECLI_SANITY_CHECK_TURNS"): sanity_check_messages(messages) messages = model_request_parser(self, messages, tools) @@ -1305,7 +1307,7 @@ async def send_completion( num_ctx = int(self.token_count(messages) * 1.25) + 8192 kwargs["num_ctx"] = num_ctx key = json.dumps(kwargs, sort_keys=True).encode() - hash_object = hashlib.sha1(key) + hash_object = xxhash.xxh64(key) if "timeout" not in kwargs: kwargs["timeout"] = request_timeout if self.verbose: @@ -1713,6 +1715,7 @@ def get_chat_model_names(query: str = "") -> list: def fuzzy_match_models(name): + import difflib import fnmatch # Handle empty string case - return all models diff --git a/cecli/signals.py b/cecli/signals.py new file mode 100644 index 00000000000..a122f4ebb7c --- /dev/null +++ b/cecli/signals.py @@ -0,0 +1,49 @@ +""" +Control-flow signal exceptions for cecli. + +These signals are used for non-error control flow (like switching coders +or reloading the program). They inherit from BaseException to avoid being +caught by generic `except Exception` handlers. +""" + + +class SwitchCoderSignal(BaseException): + """ + Signal to switch the current Coder instance to a new configuration. + + This is NOT an error - it's a control flow signal used to propagate + coder switching requests up through the async call stack. It carries + the kwargs needed to create a new Coder instance. + + Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) + to avoid being caught by generic `except Exception` handlers, making the + non-error nature of this signal explicit. + + Attributes: + kwargs: Configuration dict passed to Coder.create() for the new instance + placeholder: Optional placeholder text for the input prompt + """ + + def __init__(self, placeholder=None, **kwargs): + self.kwargs = kwargs + self.placeholder = placeholder + super().__init__() + + +class ReloadProgramSignal(BaseException): + """ + Signal to reload the entire program configuration. + + This is NOT an error - it's a control flow signal used to trigger + a full program reload, re-parsing config files and re-initializing + all components. Useful for hot-reloading when configuration files + change. + + Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) + to avoid being caught by generic `except Exception` handlers. + """ + + def __init__(self, message="Reloading program configuration...", **kwargs): + self.kwargs = kwargs + self.message = message + super().__init__(self.message) diff --git a/cecli/utils.py b/cecli/utils.py index b8a009c07eb..20bc2106e15 100644 --- a/cecli/utils.py +++ b/cecli/utils.py @@ -10,7 +10,6 @@ from pathlib import Path from cecli.dump import dump # noqa: F401 -from cecli.waiting import Spinner IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".webp", ".pdf"} @@ -311,6 +310,8 @@ def run_install(cmd): pass # Continue even if ensurepip fails try: + from cecli.waiting import Spinner + output = [] process = subprocess.Popen( cmd, diff --git a/cecli/waiting.py b/cecli/waiting.py index 94ee7f01902..a4bf5e296b0 100644 --- a/cecli/waiting.py +++ b/cecli/waiting.py @@ -4,13 +4,13 @@ A simple wrapper for rich.status to provide a spinner. """ -from rich.console import Console - class Spinner: """A wrapper around rich.status.Status for displaying a spinner.""" def __init__(self, text: str = "Waiting..."): + from rich.console import Console + self.text = text self.console = Console() self.status = None From 4c739263b163d75949988b3a00acda55300c39da Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 14 Jul 2026 09:52:31 -0400 Subject: [PATCH 15/68] Update tests for web browser and numpy import moves --- tests/basic/test_coder.py | 2 +- tests/basic/test_main.py | 2 +- tests/basic/test_model_provider_manager.py | 8 -------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/basic/test_coder.py b/tests/basic/test_coder.py index 3067e785135..872503b8e50 100644 --- a/tests/basic/test_coder.py +++ b/tests/basic/test_coder.py @@ -34,7 +34,7 @@ class TestCoder: def setup(self, gpt35_model): self.uuid = str(uuid.uuid4()) self.GPT35 = gpt35_model - self.webbrowser_patcher = patch("cecli.io.webbrowser.open") + self.webbrowser_patcher = patch("webbrowser.open") self.mock_webbrowser = self.webbrowser_patcher.start() # Reset conversation system before each test ConversationService.get_chunks(self).reset() diff --git a/tests/basic/test_main.py b/tests/basic/test_main.py index 082877e2cb9..8cff21870cf 100644 --- a/tests/basic/test_main.py +++ b/tests/basic/test_main.py @@ -73,7 +73,7 @@ def test_env(mocker, temp_cwd, temp_home): test_env_vars["HOME"] = temp_home mocker.patch.dict(os.environ, test_env_vars) mocker.patch("builtins.input", return_value=None) - mocker.patch("cecli.io.webbrowser.open") + mocker.patch("webbrowser.open") @pytest.fixture diff --git a/tests/basic/test_model_provider_manager.py b/tests/basic/test_model_provider_manager.py index fb9ddd46721..6da85ac6650 100644 --- a/tests/basic/test_model_provider_manager.py +++ b/tests/basic/test_model_provider_manager.py @@ -31,14 +31,6 @@ def _dummy_open(*args, **kwargs): sys.modules["PIL.Image"] = image_module sys.modules["PIL.ImageGrab"] = image_grab_module - if "numpy" not in sys.modules: - numpy_module = types.ModuleType("numpy") - numpy_module.ndarray = object - numpy_module.array = lambda *a, **k: None - numpy_module.dot = lambda *a, **k: 0.0 - numpy_module.linalg = types.SimpleNamespace(norm=lambda *a, **k: 1.0) - sys.modules["numpy"] = numpy_module - if "oslex" not in sys.modules: oslex_module = types.ModuleType("oslex") oslex_module.__all__ = [] From 3940255974101d4b314287b4a2256fef08d17228 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 14 Jul 2026 09:56:18 -0400 Subject: [PATCH 16/68] Remove numpy dependency --- cecli/helpers/similarity.py | 64 ++++++++++++------------------------- cecli/main.py | 4 +-- 2 files changed, 21 insertions(+), 47 deletions(-) diff --git a/cecli/helpers/similarity.py b/cecli/helpers/similarity.py index 795176c1e4c..4e7f0725d9c 100644 --- a/cecli/helpers/similarity.py +++ b/cecli/helpers/similarity.py @@ -2,42 +2,37 @@ def normalize_vector(vector): """Normalize a vector to unit length (L2 norm). Args: - vector (np.ndarray or list): Input vector + vector (list): Input vector Returns: - np.ndarray: Normalized vector with length 1 + list: Normalized vector with length 1 """ - import numpy as np + import math - vector = np.asarray(vector, dtype=np.float64) - magnitude = np.linalg.norm(vector) + magnitude = math.sqrt(sum(x * x for x in vector)) if magnitude == 0: - return vector # Return original if zero vector - return vector / magnitude + return list(vector) # Return copy if zero vector + return [x / magnitude for x in vector] def cosine_similarity(vector1, vector2): """Calculate cosine similarity between two vectors. Args: - vector1 (np.ndarray or list): First vector - vector2 (np.ndarray or list): Second vector + vector1 (list): First vector + vector2 (list): Second vector Returns: float: Cosine similarity between the vectors (range: -1 to 1) """ - import numpy as np - - vector1 = np.asarray(vector1, dtype=np.float64) - vector2 = np.asarray(vector2, dtype=np.float64) + import math if len(vector1) != len(vector2): raise ValueError("Vectors must have the same length") - # Use NumPy's optimized dot product and norm functions - dot_product = np.dot(vector1, vector2) - magnitude1 = np.linalg.norm(vector1) - magnitude2 = np.linalg.norm(vector2) + dot_product = sum(a * b for a, b in zip(vector1, vector2)) + magnitude1 = math.sqrt(sum(x * x for x in vector1)) + magnitude2 = math.sqrt(sum(x * x for x in vector2)) if magnitude1 == 0 or magnitude2 == 0: return 0.0 # Return 0 if either vector is zero @@ -46,19 +41,14 @@ def cosine_similarity(vector1, vector2): def create_bigram_vector(texts): - """Create a bigram frequency vector using optimized NumPy operations. - - This version uses pre-computed bigram indices and NumPy's bincount - for maximum performance on large datasets. + """Create a bigram frequency vector. Args: texts (tuple): Tuple of strings to process Returns: - np.ndarray: Vector of bigram frequencies + list: Vector of bigram frequencies """ - import numpy as np - # Pre-compute bigram indices (0 for 'aa', 1 for 'ab', ..., 675 for 'zz') bigram_indices = {} idx = 0 @@ -69,7 +59,7 @@ def create_bigram_vector(texts): idx += 1 # Initialize frequency vector - vector = np.zeros(26 * 26, dtype=np.int32) + vector = [0] * (26 * 26) # Process all texts for text in texts: @@ -77,25 +67,11 @@ def create_bigram_vector(texts): if len(text_lower) < 2: continue - # Extract bigrams using NumPy sliding window view - # Convert string to character array for efficient slicing - chars = np.array(list(text_lower)) - # Create bigrams by combining consecutive characters - bigrams = np.char.add(chars[:-1], chars[1:]) - - # Filter only alphabetic bigrams - mask = np.array([bg.isalpha() for bg in bigrams]) - valid_bigrams = bigrams[mask] - - # Count bigrams using bincount with pre-computed indices - indices = [] - for bg in valid_bigrams: - if bg in bigram_indices: - indices.append(bigram_indices[bg]) - - if indices: - counts = np.bincount(indices, minlength=26 * 26) - vector += counts + for i in range(len(text_lower) - 1): + bg = text_lower[i : i + 2] + # Filter only alphabetic bigrams + if bg.isalpha() and bg in bigram_indices: + vector[bigram_indices[bg]] += 1 return vector diff --git a/cecli/main.py b/cecli/main.py index b9983e99946..5df11a9a702 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -1470,9 +1470,7 @@ async def check_and_load_imports(io, is_first_run, verbose=False): def load_slow_imports(swallow=True): try: - import httpx # noqa - import litellm # noqa - import numpy # noqa + import cecli.llm # noqa except Exception as e: if not swallow: raise e From f45ddde4f778c2fae8adcbb7ff3695ab6c1bfdb9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 14 Jul 2026 21:13:25 -0400 Subject: [PATCH 17/68] Revert some changes to io.py --- cecli/io.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/cecli/io.py b/cecli/io.py index deab2d44bf1..b1abe0b4514 100644 --- a/cecli/io.py +++ b/cecli/io.py @@ -18,8 +18,19 @@ from pathlib import Path from prompt_toolkit.completion import Completer, Completion, ThreadedCompleter +from prompt_toolkit.cursor_shapes import ModalCursorShapeConfig +from prompt_toolkit.enums import EditingMode +from prompt_toolkit.filters import Condition, is_searching from prompt_toolkit.history import FileHistory +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.vi_state import InputMode +from prompt_toolkit.keys import Keys +from prompt_toolkit.lexers import PygmentsLexer from prompt_toolkit.output.vt100 import is_dumb_terminal +from prompt_toolkit.shortcuts import CompleteStyle, PromptSession +from prompt_toolkit.styles import Style +from pygments.lexers import MarkdownLexer, guess_lexer_for_filename +from pygments.token import Token from rich.color import ColorParseError from rich.columns import Columns from rich.console import Console @@ -170,9 +181,6 @@ def __init__( self.tokenized = False def tokenize(self): - from pygments.lexers import guess_lexer_for_filename - from pygments.token import Token - if self.tokenized: return self.tokenized = True @@ -351,7 +359,7 @@ def __init__( encoding="utf-8", line_endings="platform", dry_run=False, - editingmode="EMACS", + editingmode=EditingMode.EMACS, fancy_input=True, file_watcher=None, multiline_mode=False, @@ -503,9 +511,6 @@ def __init__( self.interruptible_input = None if fancy_input: - from prompt_toolkit.lexers import PygmentsLexer - from pygments.lexers import MarkdownLexer - # If unicode is supported, use the rich 'dots2' spinner, otherwise an ascii fallback if self._spinner_supports_unicode(): self.spinner_frames = SPINNERS["dots2"]["frames"] @@ -521,15 +526,11 @@ def __init__( "editing_mode": self.editingmode, "bottom_toolbar": self.get_bottom_toolbar, } - if self.editingmode == "VI": - from prompt_toolkit.cursor_shapes import ModalCursorShapeConfig - + if self.editingmode == EditingMode.VI: session_kwargs["cursor"] = ModalCursorShapeConfig() if self.input_history_file is not None: session_kwargs["history"] = FileHistory(self.input_history_file) try: - from prompt_toolkit.shortcuts import PromptSession - self.prompt_session = PromptSession(**session_kwargs) self.console = Console() # pretty console except Exception as err: @@ -637,8 +638,6 @@ def _validate_color_settings(self): setattr(self, attr_name, None) # Reset invalid color to None def _get_style(self): - from prompt_toolkit.styles import Style - style_dict = {} if not self.pretty: return Style.from_dict(style_dict) @@ -829,11 +828,6 @@ async def get_input( edit_format=None, **kwargs, ): - from prompt_toolkit.filters import Condition, is_searching - from prompt_toolkit.key_binding import KeyBindings - from prompt_toolkit.keys import Keys - from prompt_toolkit.shortcuts import CompleteStyle - self.rule() if commands.last_command_show_notification: self.notify_user_input_required() @@ -945,10 +939,9 @@ def _(event): @kb.add("enter", eager=True, filter=~is_searching) def _(event): "Handle Enter key press" - from prompt_toolkit.key_binding.vi_state import InputMode - if self.multiline_mode and not ( - self.editingmode == "VI" and event.app.vi_state.input_mode == InputMode.NAVIGATION + self.editingmode == EditingMode.VI + and event.app.vi_state.input_mode == InputMode.NAVIGATION ): # In multiline mode and if not in vi-mode or vi navigation/normal mode, # Enter adds a newline From f92688bcdd5e61986b53620563380f64247b4852 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 15 Jul 2026 21:27:59 -0400 Subject: [PATCH 18/68] Fix start up behavior of pre initialization confirmations --- cecli/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cecli/io.py b/cecli/io.py index b1abe0b4514..663a5999b06 100644 --- a/cecli/io.py +++ b/cecli/io.py @@ -359,7 +359,7 @@ def __init__( encoding="utf-8", line_endings="platform", dry_run=False, - editingmode=EditingMode.EMACS, + editingmode="EMACS", fancy_input=True, file_watcher=None, multiline_mode=False, @@ -523,10 +523,10 @@ def __init__( "input": self.input, "output": self.output, "lexer": PygmentsLexer(MarkdownLexer), - "editing_mode": self.editingmode, + "editing_mode": EditingMode(self.editingmode), "bottom_toolbar": self.get_bottom_toolbar, } - if self.editingmode == EditingMode.VI: + if self.editingmode == "VI": session_kwargs["cursor"] = ModalCursorShapeConfig() if self.input_history_file is not None: session_kwargs["history"] = FileHistory(self.input_history_file) From 1354cf52e3ae7d16d57a7906a743a5f0eb3e740a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 16 Jul 2026 12:54:00 -0400 Subject: [PATCH 19/68] Translate file strings to hashline ids before normalizing --- cecli/tools/edit_file.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 6b2763c6463..d8635964aee 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -219,7 +219,13 @@ def execute( edit_start_line = "@000" edit_end_line = "@000" - # 3. Auto-sanitize malformed boundaries (strip accidentally appended code) + # 3. Resolve non-hashline content values to content IDs first + # (before normalize_hashline which would fail on arbitrary content) + edit_start_line, edit_end_line = resolve_content_to_hashline_ids( + original_content, edit_start_line, edit_end_line + ) + + # 4. Auto-sanitize malformed boundaries (strip accidentally appended code) if isinstance(edit_start_line, str) and "::" in edit_start_line: edit_start_line = normalize_hashline(edit_start_line) if isinstance(edit_end_line, str) and "::" in edit_end_line: @@ -236,11 +242,6 @@ def execute( edit_file = edit_file_raw - # Try to resolve line content values to content IDs - edit_start_line, edit_end_line = resolve_content_to_hashline_ids( - original_content, edit_start_line, edit_end_line - ) - # Validate required fields based on operation type # (Note: The check for 'edit_file is None' will now be safely # bypassed because we defaulted it to "" above) From eb5e8249583a6b581b33072106989d859a3c743b Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 16 Jul 2026 13:06:54 -0400 Subject: [PATCH 20/68] Let the LLM triage it's own harness misunderstandings --- cecli/prompts/agent.yml | 4 ++-- cecli/prompts/subagent.yml | 4 ++-- cecli/tools/edit_file.py | 4 +++- cecli/tools/read_file.py | 6 ++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index 94791760927..93c958b9e23 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -27,8 +27,8 @@ main_system: | ## FILE FORMAT File contents will be prefixed with identifiers. Each line starts with a case-sensitive content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change after file edits. - Do not search for these content IDs directly. You will not be able to generate them. + They are virtual, deterministic identifiers generated on-the-fly when representing files and never change from reading alone. Only IDs within ~5 lines of an edit site are regenerated — IDs farther away remain valid across edits. + Do not search for these content IDs directly. You will not be able to generate them. Use content IDs from a recent ReadFile to target edits. **Example File** ``` diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index 4fdba2a922d..f157df21532 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -12,8 +12,8 @@ main_system: | ## FILE FORMAT File contents will be prefixed with identifiers. Each line starts with a case-sensitive content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change after file edits. - Do not search for these content IDs directly. You will not be able to generate them. + They are virtual, deterministic identifiers generated on-the-fly when representing files and never change from reading alone. Only IDs within ~5 lines of an edit site are regenerated — IDs farther away remain valid across edits. + Do not search for these content IDs directly. You will not be able to generate them. Use content IDs from a recent ReadFile to target edits. **Example File** ``` diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index d8635964aee..c28f837adc4 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -50,7 +50,9 @@ class Tool(BaseTool): "1. Start and end content IDs are INCLUSIVE. Both will be modified or deleted. " "2. Content IDs MUST include the `::` demarcator. " "3. Edits within the same file MUST NOT be adjacent or overlapping. " - "4. For empty files, you MUST use '@000' as the content ID reference." + "4. For empty files, you MUST use '@000' as the content ID reference. " + "5. After an edit, only IDs within ~5 lines of the change are regenerated. " + "IDs farther from the edit site remain usable." ), "parameters": { "type": "object", diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index 2b086993484..5da142db691 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -50,10 +50,12 @@ class Tool(BaseTool): "" " It is best to use function names," " variable declarations, entire line contents and other meaningful identifiers" - " The returned identifiers will not persist after editing the file." + " Content IDs persist across reads and only change locally (~5 lines) around an edit site." + " IDs far from any edit remain valid." " Only use the same pattern for range_start and range_end when fetching full method definitions." " Do not use empty strings for the range_start and range_end." - " Do not use content IDs for the range_start and range_end values as they change between edits." + " Content IDs returned by ReadFile can be used as range markers for EditFile." + " Re-read the file if you need fresh IDs after editing near your target region." " Always use the ReadFile tool instead of cli tools for reading file contents." " Line number and special marker ranges greater than 200 lines will return" " preview content for further, more scoped investigation." From f84febe79204a65e7b63600552d6999a87161833 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 17 Jul 2026 08:15:29 -0400 Subject: [PATCH 21/68] Numerous updates to support ptc --- cecli/helpers/orchestration/agent_proxy.py | 340 +++++++++ cecli/helpers/orchestration/environment.py | 613 +++++++--------- .../helpers/orchestration/region_resolver.py | 390 ++++++++++ cecli/helpers/orchestration/safe_methods.py | 611 ++++++++++++++++ cecli/helpers/orchestration/security.py | 6 +- cecli/helpers/orchestration/tool_proxy.py | 223 ++++++ cecli/tools/edit_file.py | 47 +- cecli/tools/explore_code.py | 214 +++--- cecli/tools/grep.py | 23 +- cecli/tools/orchestrate.py | 7 +- cecli/tools/read_file.py | 207 +++++- cecli/tools/resource_manager.py | 3 +- cecli/tools/utils/base_tool.py | 25 +- cecli/tools/utils/responses.py | 36 +- tests/helpers/test_orchestration.py | 679 +++++++++++++++++- tests/tools/test_grep.py | 7 +- tests/tools/test_read_range_execute.py | 6 +- 17 files changed, 2885 insertions(+), 552 deletions(-) create mode 100644 cecli/helpers/orchestration/agent_proxy.py create mode 100644 cecli/helpers/orchestration/region_resolver.py create mode 100644 cecli/helpers/orchestration/safe_methods.py create mode 100644 cecli/helpers/orchestration/tool_proxy.py diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py new file mode 100644 index 00000000000..732d8b5d63a --- /dev/null +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -0,0 +1,340 @@ +""" +Singleton-like proxy injected into the orchestration environment. + +Usage in LLM-generated code:: + + read_tool = Agent.get_tool("ReadFile") + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + +Supports both local tools (from ToolRegistry) and MCP tools (from connected +servers) using `ServerName--ToolName` or bare tool-name lookup. +""" + +from __future__ import annotations + +from typing import Any + +from cecli.helpers import nested, responses +from cecli.helpers.orchestration.tool_proxy import ToolProxy + + +class AgentProxy: + """ + Singleton-like proxy injected into the orchestration environment. + + Usage in LLM-generated code:: + + read_tool = Agent.get_tool("ReadFile") + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + + Supports both local tools (from ToolRegistry) and MCP tools (from connected + servers) using `ServerName--ToolName` or bare tool-name lookup. + """ + + def __init__(self, coder: Any) -> None: + self._coder = coder + + def get_tool(self, tool_name: str) -> ToolProxy: + from cecli.tools.utils.registry import ToolRegistry + + name_lower = tool_name.lower() + + # 1. Try local tools by exact name (unprefixed) + tool_module = ToolRegistry.get_tool(name_lower) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 2. Unprefix: "ServerName--ToolName" → (server_prefix, bare_name) + server_prefix, bare_name = responses.unprefix_tool_name(name_lower) + + # 3. If the prefix is "local", retry ToolRegistry with the bare name + if server_prefix == "local" and bare_name: + tool_module = ToolRegistry.get_tool(bare_name) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 3b. Strip underscores from the tool name and retry (snake_case fallback) + no_underscore = name_lower.replace("_", "") + if no_underscore != name_lower: + tool_module = ToolRegistry.get_tool(no_underscore) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 4. Search MCP tools for the bare (unprefixed) name + for mcp_server_name, server_tools in self._coder.mcp_tools or []: + for tool_schema in server_tools: + schema_name = nested.getter(tool_schema, "function.name", "") + _schema_prefix, schema_unprefixed = responses.unprefix_tool_name( + schema_name.lower() + ) + if schema_unprefixed == bare_name: + server = self._find_mcp_server(mcp_server_name, server_prefix) + if server is not None: + return ToolProxy( + tool_name, + self._coder, + mcp_server=server, + mcp_tool_name=schema_name, + ) + + raise ValueError(f"Unknown tool: '{tool_name}'") + + def _find_mcp_server(self, server_name: str, server_prefix: str) -> Any: + if not hasattr(self._coder, "mcp_manager") or not self._coder.mcp_manager: + return None + for server in self._coder.mcp_manager: + if server.name == server_name and ( + not server_prefix or server.name.lower() == server_prefix.lower() + ): + return server + return None + + def get_shape(self, result: Any) -> str: + """Inspect the structure of a tool result and return a readable summary. + + Tool results have a standard shape:: + + { + "result": [{"content": ..., "_": {"file_path": ..., ...}}], + "errors": [...], + "details": [...] + } + + This method unwraps the structure to show what keys are available, + helping the LLM navigate deeply nested tool outputs. + + Example:: + + output = await grep_tool.call(pattern="TODO", file_glob="*.py") + print(Agent.get_shape(output)) + # Shows: result[0].content, result[0]._.file, result[0]._.match_count, ... + + # Now the LLM can confidently access: + for item in output["result"]: + print(item["content"]) # or item["_"]["file"] + + Returns a multi-line string describing the available access paths. + """ + return self._inspect_structure(result) + + @staticmethod + def _inspect_structure(obj: Any, prefix: str = "", depth: int = 0) -> str: + """Recursively inspect the structure of a tool result — paths and types only.""" + max_depth = 3 + max_keys = 5 + lines: list[str] = [] + + if depth > max_depth: + return "" + + if isinstance(obj, dict): + for key, value in obj.items(): + path = f"{prefix}.{key}" if prefix else key + + if isinstance(value, dict): + keys = list(value.keys())[:max_keys] + suffix = "..." if len(value) > max_keys else "" + lines.append(f"{path}: dict[{' | '.join(keys)}{suffix}]") + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + elif isinstance(value, list): + if value: + first = value[0] + if isinstance(first, dict): + lines.append(f"{path}: list[{len(value)}] dict") + inner = AgentProxy._inspect_structure(first, f"{path}[0]", depth + 1) + if inner: + lines.append(inner) + else: + lines.append(f"{path}: list[{len(value)}] {type(first).__name__}") + else: + lines.append(f"{path}: list (empty)") + else: + lines.append(f"{path}: {type(value).__name__}") + + elif isinstance(obj, list): + if obj: + first = obj[0] + if isinstance(first, dict): + lines.append(f"list[{len(obj)}] dict") + inner = AgentProxy._inspect_structure( + first, f"{prefix}[0]" if prefix else "[0]", depth + 1 + ) + if inner: + lines.append(inner) + else: + lines.append(f"list[{len(obj)}] {type(first).__name__}") + else: + lines.append("list (empty)") + else: + lines.append(f"{type(obj).__name__}") + + return "\n".join(lines) + + def get_content_id(self, file_path: str, line_content: str) -> str: + """Resolve a content ID for use as start_line/end_line with EditFile. + + Supports three modes: + - **@L{number}**: e.g., `Agent.get_content_id("foo.py", "@L42")` + returns the content ID of line 42 (1-based). + - **content ID passthrough**: e.g., `Agent.get_content_id("foo.py", "abc::")` + verifies and returns an existing content ID string. + - **text match**: e.g., `Agent.get_content_id("foo.py", "def greet(")` + returns the content ID of the unique line containing that text. + """ + import os + import re + + from cecli.helpers.hashline import resolve_content_to_hashline_ids + from cecli.helpers.hashpos.hashpos import HashPos + from cecli.tools.utils.helpers import resolve_paths + + abs_path, rel_path = resolve_paths(self._coder, file_path) + if not os.path.isfile(abs_path): + raise ValueError(f"File not found: {file_path}") + + content = self._coder.io.read_text(abs_path) + if content is None: + raise ValueError(f"Could not read file: {file_path}") + + lines = content.splitlines() + hp = HashPos(content) + + # @L{number} syntax + m = re.match(r"^@L(\d+)$", line_content.strip()) + if m: + line_num = int(m.group(1)) - 1 + if line_num < 0 or line_num >= len(lines): + raise ValueError(f"Line {m.group(1)} out of range (file has {len(lines)} lines)") + return hp.generate_public_id(lines[line_num], line_num) + "::" + + # Content ID passthrough: already looks like a content ID + if "::" in line_content: + from cecli.helpers.hashline import ContentHashError, normalize_hashline + + try: + normalized = normalize_hashline(line_content) + candidates = hp.resolve_to_lines(normalized) + if candidates: + return line_content + except (ContentHashError, ValueError): + pass + raise ValueError(f"Content ID '{line_content}' not found in {file_path}") + + # Text match via resolve_content_to_hashline_ids + result, _ = resolve_content_to_hashline_ids(content, line_content, None) + if result == line_content or "::" not in str(result): + # Find all matching lines for a helpful error message + matching = [i + 1 for i, line in enumerate(lines) if line_content in line] + if len(matching) > 1: + line_nums = ", ".join(str(n) for n in matching[:10]) + suffix = f" ... ({len(matching)} total)" if len(matching) > 10 else "" + raise ValueError( + f"Pattern '{line_content}' matches {len(matching)} locations " + f"in {file_path} (lines {line_nums}{suffix}). " + f"Use ' @L' to disambiguate (e.g., '{line_content} @L{matching[0]}')." + ) + if matching: + raise ValueError( + f"Could not resolve content ID for '{line_content}' " + f"in {file_path} (line {matching[0]}). " + f"The match may not be unique enough." + ) + raise ValueError(f"Could not resolve content ID for '{line_content}' in {file_path}") + return result + + def resolve_regions( + self, + file_path: str, + regions: list[dict[str, str]], + ): + """ + Store named region patterns for lazy content-ID resolution. + + Content IDs are resolved *on access* via `.get_start(name)` / + `.get_end(name)`, so they are always fresh — even after + intervening edits shift hashline positions. + + Returns an :class:`AgentRegion` instance. + """ + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + return AgentRegion(file_path, self._coder, regions) + + async def edit_region( + self, + file_path: str, + edits: list[dict[str, object]], + change_id: str | None = None, + ): + """ + Thin wrapper around `EditFile` that accepts `{"start": content_id, "end": content_id}` region dicts. + + Use with `Agent.resolve_regions()` and `regions.get(name)`:: + + regions = Agent.resolve_regions("foo.py", [ + {"name": "my_func", "start": "def my_func", "end": "return result"}, + ]) + await Agent.edit_region( + file_path="foo.py", + edits=[ + {"region": regions.get("my_func"), "text": "def my_func():\\n return 42"}, + ], + ) + """ + + edit_tool = self.get_tool("EditFile") + + edit_objects: list[dict[str, object]] = [] + for edit in edits: + region = edit["region"] + + edit_objects.append( + { + "file_path": file_path, + "operation": edit.get("operation", "replace"), + "start_line": region["start"], + "end_line": region["end"], + "text": edit["text"], + } + ) + + return await edit_tool.call( + edits=edit_objects, + change_id=change_id, + ) + + +# --------------------------------------------------------------------------- +# Main execution environment +# --------------------------------------------------------------------------- + + +class _HelpfulBuiltins(dict): + """Custom __builtins__ dict that provides helpful hints for missing functions.""" + + _HINTS: dict[str, str] = { + "open": "Filesystem access is not available. Use the Command tool instead.", + "eval": "eval() is disabled for security.", + "exec": "exec() is disabled for security.", + "__import__": "Imports are disabled. Use only the primitives provided.", + "compile": "compile() is disabled for security.", + "breakpoint": "breakpoint() is disabled in the sandbox.", + "globals": "globals() is disabled. Use state or shared_state for persistence.", + "locals": "locals() is disabled. Use state or shared_state for persistence.", + "vars": "Use vars(obj) instead of vars() — vars(obj) returns non-dunder attrs of obj.", + "getattr": "getattr() is disabled. Access attributes directly.", + "setattr": "setattr() is disabled. Assign attributes directly.", + "delattr": "delattr() is disabled. Use del obj.attr instead.", + } + + def __missing__(self, key: str): + hint = self._HINTS.get(key) + if hint: + raise NameError(f"'{key}' is not available. {hint}") + raise NameError(f"name '{key}' is not defined") + + +# --------------------------------------------------------------------------- diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index caf7ffb79a4..4572b000ea1 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -8,307 +8,97 @@ import ast import asyncio +import collections +import datetime +import itertools +import json import logging +import math +import re import traceback from typing import Any from cecli.helpers import nested, responses +from cecli.helpers.orchestration.agent_proxy import AgentProxy +from cecli.helpers.orchestration.safe_methods import ( + GatherResult, + _escape_newlines_in_strings, + _HelpfulBuiltins, + _safe_dir, + _safe_gather, + _safe_sleep, + _safe_typeof, + _safe_vars, + _SafeJson, + _SafeModuleProxy, + _strip_allowed_imports, +) from cecli.helpers.orchestration.security import ( LoopYieldInjector, SecurityError, SecurityFilter, _cooperative_yield, ) +from cecli.helpers.orchestration.tool_proxy import ToolProxy logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Safe primitives exposed to the sandbox -# --------------------------------------------------------------------------- - - -async def _safe_sleep(seconds: float) -> None: - """Safe sleep wrapper for the orchestration environment.""" - if seconds < 0: - raise ValueError("sleep() requires a non-negative value") - if seconds > 120: - raise ValueError("sleep() is limited to 120 seconds maximum") - await asyncio.sleep(seconds) - - -async def _safe_gather(*awaitables: Any) -> list[Any]: - """ - Safely execute multiple awaitables concurrently. - - Forces ``return_exceptions=True`` so that failures in one task - do not crash the entire batch. - """ - return await asyncio.gather(*awaitables, return_exceptions=True) - - -class _SafeJson: - """Drop-in ``json`` namespace with only ``loads`` and ``dumps``.""" - - @staticmethod - def loads(s: str) -> Any: - import json - - return json.loads(s) - - @staticmethod - def dumps(obj: Any) -> str: - import json - - return json.dumps(obj) - - -# --------------------------------------------------------------------------- -# Tool proxies -# --------------------------------------------------------------------------- - -class ToolProxy: - """ - Proxy for a single tool — local or MCP. +class TrackedDict(dict): + """A dict subclass that records mutations (set/delete/clear/pop) on a parent env. - The LLM code calls ``tool.call(**params)`` and this proxy routes it - through the appropriate execution path. + Tracks both adds (new keys) and modifications (existing key value changes) + so the environment can report only changed variables after execution. """ - def __init__( - self, - tool_name: str, - coder: Any, - *, - tool_module: Any = None, - mcp_server: Any = None, - mcp_tool_name: str = "", - ) -> None: - # Respect the per-coder tool includelist/excludelist filters - incl = getattr(coder, "registered_tools", {}).get("included", set()) - excl = getattr(coder, "registered_tools", {}).get("excluded", set()) - name_lower = tool_name.lower() - if incl and name_lower not in incl: - raise ValueError(f"Tool '{tool_name}' is not in the allowed tools list.") - if name_lower in excl: - raise ValueError(f"Tool '{tool_name}' has been excluded.") - - self._tool_name = tool_name - self._coder = coder - self._tool_module = tool_module - self._mcp_server = mcp_server - self._mcp_tool_name = mcp_tool_name - - async def __call__(self, *args: Any, **kwargs: Any): - """Make the proxy directly callable. - - Supports both ``await tool(key=val)`` and ``await tool("val")``. - Positional arguments are mapped to parameter names using the - tool's schema (when available). - """ - if args and kwargs: - raise TypeError( - f"Tool '{self._tool_name}': cannot mix positional and keyword arguments" - ) - + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._owner: AgentExecutionEnv | None = None + self._is_shared: bool = False + + def _set_owner(self, owner: AgentExecutionEnv, is_shared: bool = False) -> None: + self._owner = owner + self._is_shared = is_shared + + def __setitem__(self, key, value): + is_new = key not in self + super().__setitem__(key, value) + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared, is_new=is_new) + + def __delitem__(self, key): + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared) + super().__delitem__(key) + + def clear(self): + if self._owner is not None: + for key in list(self): + self._owner._record_mutation(key, is_shared=self._is_shared) + super().clear() + else: + super().clear() + + def pop(self, key, *args): + if key in self: + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared) + return super().pop(key, *args) if args: - param_names = self._get_param_names() - if not param_names: - if len(args) == 1: - # Fallback: try common first-param names - for guess in ( - "path", - "read", - "searches", - "edits", - "queries", - "tasks", - "delegations", - "code", - "command_string", - "command", - "summary", - ): - kwargs = {guess: args[0]} - break - else: - raise TypeError( - f"Tool '{self._tool_name}': cannot resolve positional " - f"argument – no schema available" - ) - else: - raise TypeError( - f"Tool '{self._tool_name}': cannot resolve positional " - f"arguments – no schema available" - ) - elif len(args) > len(param_names): - raise TypeError( - f"Tool '{self._tool_name}': too many positional arguments " - f"({len(args)} for {len(param_names)} parameter(s): {param_names})" - ) - else: - kwargs = dict(zip(param_names, args)) - - return await self.call(**kwargs) - - def _get_param_names(self) -> list: - """Extract ordered parameter names from the tool's JSON Schema.""" - if self._tool_module is None: - return [] - try: - props = self._tool_module.SCHEMA["function"]["parameters"]["properties"] - return list(props.keys()) - except (KeyError, TypeError, AttributeError): - return [] - - async def call(self, **kwargs: Any): - """Execute the tool with the given keyword arguments. - - Tool results are normalized to a dict with ``result`` (list), - ``errors`` (list), and ``details`` (list) keys, matching the - documented orchestration contract. - """ - - if self._tool_module is not None: - result = self._tool_module.process_response(self._coder, kwargs) - if asyncio.iscoroutine(result): - result = await result - return self._normalize_result(result) - - if self._mcp_server is not None: - result = await self._coder._execute_mcp_tool( - self._mcp_server, self._mcp_tool_name, kwargs - ) - return self._normalize_result(result) - - raise ValueError(f"No executor for tool '{self._tool_name}'") - - @staticmethod - def _normalize_result(result: Any) -> dict: - """Normalize a tool result into a dict with ``result``, ``errors``, ``details`` keys. - - Handles ``ToolResponse``, error strings, and plain dicts. - """ - from cecli.tools.utils.responses import ToolResponse - - if isinstance(result, ToolResponse): - data = result.to_dict() - out = {} - out["result"] = ( - data["result"] - if isinstance(data["result"], list) - else [data["result"]] if data["result"] else [] - ) - out["errors"] = data.get("errors", []) - out["details"] = data.get("details", []) - return out - - if isinstance(result, str): - # Error strings from handle_tool_error or plain string results - if result.startswith("Error in "): - return { - "result": [], - "errors": [result], - "details": [], - } - return { - "result": [result], - "errors": [], - "details": [], - } - - if isinstance(result, dict): - # Already a dict - ensure it has the expected keys - out = dict(result) - if "result" not in out: - out["result"] = [] - if "errors" not in out: - out["errors"] = [] - if "details" not in out: - out["details"] = [] - if not isinstance(out["result"], list): - out["result"] = [out["result"]] if out["result"] else [] - if not isinstance(out["errors"], list): - out["errors"] = [out["errors"]] - if not isinstance(out["details"], list): - out["details"] = [out["details"]] - return out - - # Fallback: wrap anything else - return { - "result": [str(result)] if result is not None else [], - "errors": [], - "details": [], - } - + return args[0] + raise KeyError(key) -class AgentProxy: - """ - Singleton-like proxy injected into the orchestration environment. - - Usage in LLM-generated code:: - - read_tool = Agent.get_tool("ReadFile") - result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") - - Supports both local tools (from ToolRegistry) and MCP tools (from connected - servers) using ``ServerName--ToolName`` or bare tool-name lookup. - """ - - def __init__(self, coder: Any) -> None: - self._coder = coder - - def get_tool(self, tool_name: str) -> ToolProxy: - from cecli.tools.utils.registry import ToolRegistry - - name_lower = tool_name.lower() - - # 1. Try local tools by exact name (unprefixed) - tool_module = ToolRegistry.get_tool(name_lower) - if tool_module is not None: - return ToolProxy(tool_name, self._coder, tool_module=tool_module) - - # 2. Unprefix: "ServerName--ToolName" → (server_prefix, bare_name) - server_prefix, bare_name = responses.unprefix_tool_name(name_lower) - - # 3. If the prefix is "local", retry ToolRegistry with the bare name - if server_prefix == "local" and bare_name: - tool_module = ToolRegistry.get_tool(bare_name) - if tool_module is not None: - return ToolProxy(tool_name, self._coder, tool_module=tool_module) - - # 4. Search MCP tools for the bare (unprefixed) name - for mcp_server_name, server_tools in self._coder.mcp_tools or []: - for tool_schema in server_tools: - schema_name = nested.getter(tool_schema, "function.name", "") - _schema_prefix, schema_unprefixed = responses.unprefix_tool_name( - schema_name.lower() - ) - if schema_unprefixed == bare_name: - server = self._find_mcp_server(mcp_server_name, server_prefix) - if server is not None: - return ToolProxy( - tool_name, - self._coder, - mcp_server=server, - mcp_tool_name=schema_name, - ) - - raise ValueError(f"Unknown tool: '{tool_name}'") - - def _find_mcp_server(self, server_name: str, server_prefix: str) -> Any: - if not hasattr(self._coder, "mcp_manager") or not self._coder.mcp_manager: - return None - for server in self._coder.mcp_manager: - if server.name == server_name and ( - not server_prefix or server.name.lower() == server_prefix.lower() - ): - return server - return None + def popitem(self): + if not self: + raise KeyError("popitem(): dictionary is empty") + key = next(iter(self)) + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared) + return super().popitem() # --------------------------------------------------------------------------- -# Main execution environment +# Safe primitives exposed to the sandbox # --------------------------------------------------------------------------- @@ -317,12 +107,12 @@ class AgentExecutionEnv: Sandboxed REPL environment for executing LLM-generated orchestration code. Provides: - - ``Agent`` : proxy to look up and call tools - - ``gather`` : safe parallel execution helper - - ``state`` : persistent dict (survives across Orchestrate calls) - - ``sleep`` : safe sleep (0-120 seconds) - - ``print`` : captured output - - ``range``, ``len``, ``int``, ``str``, ``list``, ``dict``, ``bool``, ``Exception`` + - `Agent` : proxy to look up and call tools + - `gather` : safe parallel execution helper + - `state` : persistent dict (survives across Orchestrate calls) + - `sleep` : safe sleep (0-120 seconds) + - `print` : captured output + - `range`, `len`, `int`, `str`, `list`, `dict`, `bool`, `Exception` Security guarantees: - AST security filtering before compilation @@ -332,10 +122,12 @@ class AgentExecutionEnv: """ # Shared across all AgentExecutionEnv instances — any agent can read/write - _shared_state: dict[str, Any] = {} + _shared_state: TrackedDict = TrackedDict() def __init__(self, coder: Any) -> None: - self.state: dict[str, Any] = {} + self.state = TrackedDict() + self.state._set_owner(self) + self._shared_state._set_owner(self, is_shared=True) _safe_builtins: dict[str, Any] = { "print": print, @@ -349,8 +141,13 @@ def __init__(self, coder: Any) -> None: "bool": bool, "tuple": tuple, "set": set, - # "type": type, # excluded for security (can create dynamic classes) + "typeof": _safe_typeof, + "type": _safe_typeof, + "vars": _safe_vars, + "dir": _safe_dir, "isinstance": isinstance, + "hasattr": hasattr, + "repr": repr, "enumerate": enumerate, "zip": zip, "sorted": sorted, @@ -362,8 +159,9 @@ def __init__(self, coder: Any) -> None: "round": round, "any": any, "all": all, - # "filter": filter, # excluded for security - # "map": map, # excluded for security + "filter": filter, + "map": map, + "chr": chr, "Exception": Exception, "ValueError": ValueError, "TypeError": TypeError, @@ -371,10 +169,27 @@ def __init__(self, coder: Any) -> None: "IndexError": IndexError, "AttributeError": AttributeError, "RuntimeError": RuntimeError, + "NameError": NameError, + "re": _SafeModuleProxy(re), + "math": _SafeModuleProxy(math), + "itertools": _SafeModuleProxy(itertools), + "collections": _SafeModuleProxy(collections), + "datetime": _SafeModuleProxy(datetime), + "traceback": _SafeModuleProxy(traceback), } + def _allowed_methods(): + """Return a sorted list of all available functions and objects in the sandbox.""" + builtins = sorted(k for k in _safe_builtins.keys() if not k.startswith("__")) + globals_list = sorted( + k + for k in self.globals.keys() + if not k.startswith("__") and k not in ("__builtins__", "NEWLINE") + ) + return builtins + globals_list + self.globals: dict[str, Any] = { - "__builtins__": _safe_builtins, + "__builtins__": _HelpfulBuiltins(_safe_builtins), "Agent": AgentProxy(coder), "gather": _safe_gather, "sleep": _safe_sleep, @@ -382,10 +197,28 @@ def __init__(self, coder: Any) -> None: "state": self.state, "shared_state": AgentExecutionEnv._shared_state, "__yield": _cooperative_yield, + "NEWLINE": "\n", + "allowed_methods": _allowed_methods, } self.locals: dict[str, Any] = {} - self.globals["reset"] = self.locals.clear + def _make_reset(env_locals, env_state): + def reset_func(local_vars: bool = True, state: bool = False) -> None: + """Clear execution namespaces. + + Args: + local_vars: Clear the ephemeral locals namespace (default True). + state: Also clear per-agent state keys (default False). + Shared state is never affected. + """ + if local_vars: + env_locals.clear() + if state: + env_state.clear() + + return reset_func + + self.globals["reset"] = _make_reset(self.locals, self.state) @staticmethod def _size_of(value: Any) -> int: @@ -396,20 +229,54 @@ def _size_of(value: Any) -> int: return len(value) return len(str(value)) + @staticmethod + def _value_preview(value: Any) -> str: + """Return a short preview of a value for state snapshot display.""" + if value is None: + return "None" + s = str(value) + if len(s) > 80: + return s[:77] + "..." + return s + + def _record_mutation(self, key: object, is_shared: bool = False, is_new: bool = False) -> None: + """Record that a state key was modified during this execution. + + Called automatically by TrackedDict on set/delete/clear/pop. + """ + if not hasattr(self, "_modified_keys"): + self._modified_keys: set = set() + if not hasattr(self, "_modified_shared_keys"): + self._modified_shared_keys: set = set() + if is_shared: + self._modified_shared_keys.add(key) + else: + self._modified_keys.add(key) + def _state_snapshot(self) -> list: - """Build a list of state variable descriptors with modification tracking.""" + """Build a list of state variable descriptors. + + For local state, only modified keys are reported (with `modified: True`). + For shared state, all keys are reported so that cross-agent writes are + visible even when the current agent didn't modify them. Keys modified + by the current execution are marked `modified: True`; all others are + `modified: False`. + """ modified_keys = getattr(self, "_modified_keys", set()) modified_shared_keys = getattr(self, "_modified_shared_keys", set()) entries = [] for key, value in self.state.items(): + if key not in modified_keys: + continue entries.append( { "name": key, "type": type(value).__name__, "size": self._size_of(value), - "modified": key in modified_keys, + "modified": True, "scope": "local", + "preview": self._value_preview(value), } ) @@ -421,6 +288,7 @@ def _state_snapshot(self) -> list: "size": self._size_of(value), "modified": key in modified_shared_keys, "scope": "shared", + "preview": self._value_preview(value), } ) @@ -429,14 +297,17 @@ def _state_snapshot(self) -> list: async def execute(self, code_str: str) -> dict: code_str = code_str.strip() + code_str = _escape_newlines_in_strings(code_str) + code_str, extra_globals = _strip_allowed_imports(code_str) + self.globals.update(extra_globals) if not code_str: return {"results": "", "state_variables": self._state_snapshot()} captured_output: list[str] = [] - # Track which state keys are modified during this execution - _prev_state_keys = set(self.state.keys()) - _prev_shared_keys = set(self._shared_state.keys()) + # Clear mutation tracking for this execution + self._modified_keys = set() + self._modified_shared_keys = set() def _capture_print(*args: Any, **kwargs: Any) -> None: sep = kwargs.pop("sep", " ") @@ -464,12 +335,6 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: tree = LoopYieldInjector().visit(tree) ast.fix_missing_locations(tree) - returns_value = False - if tree.body and isinstance(tree.body[-1], ast.Expr): - last_expr = tree.body[-1].value - tree.body[-1] = ast.Return(value=last_expr) - returns_value = True - wrapper_func = ast.AsyncFunctionDef( name="__agent_async_runner", args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), @@ -504,113 +369,129 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: finally: self.locals.pop("__agent_async_runner", None) self.globals["__builtins__"]["print"] = print - self._modified_keys = set(self.state.keys()) - _prev_state_keys - self._modified_shared_keys = set(self._shared_state.keys()) - _prev_shared_keys + # Mutation tracking is handled automatically by TrackedDict print_output = "".join(captured_output) - if returns_value and result is not None: - if print_output: - code = print_output.rstrip("\n") + "\n" + str(result) - return {"results": code, "state_variables": self._state_snapshot()} - code = str(result) - return {"results": code, "state_variables": self._state_snapshot()} - if print_output: code = print_output.rstrip("\n") return {"results": code, "state_variables": self._state_snapshot()} - if returns_value: - code = str(result) - return {"results": code, "state_variables": self._state_snapshot()} - return {"results": "", "state_variables": self._state_snapshot()} +# flake8: noqa +# fmt: off + def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | None: """ Build the orchestration context block that explains calling conventions. - Only returns content if ``allow_orchestration`` is enabled in agent_config. + Only returns content if `allow_orchestration` is enabled in agent_config. """ if not agent_config.get("allow_orchestration", True): return None return """ -## Programmatic Tool Calling - -The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code. +The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. This is much more efficient than making individual tool calls for loop-heavy workflows. -### Available Primitives +### Primitives | Primitive | Description | |-----------|-------------| -| `Agent.get_tool("ToolName")` | Returns a proxy for any available tool | -| `await tool.call(**params)` | Execute a tool with keyword arguments | -| `gather(*awaitables)` | Run multiple tool calls concurrently | -| `state` | Per-agent persistent dict that survives across Orchestrate calls | -| `shared_state` | Cross-agent shared dict visible to all agents and sub-agents | +| `Agent.get_tool(name)` | Get a tool proxy (case-insensitive, accepts `Local--` or `Server--` prefix) | +| `await tool.call(**params)` | Execute a tool; returns `{"result": [...], "errors": [...], "details": [...]}` — each result item is `{"content": ..., "_": {...}}` | +| `Agent.get_shape(result)` | Inspect a tool result's structure — returns a string; use `print(Agent.get_shape(result))` to see it | + +| `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | +| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | +| `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | + + +| `gather(**named_tasks)` | Run tasks concurrently; returns an iterable with `.key` / `["key"]` access | +| `state` / `shared_state` | `state` persists across *all* `Orchestrate` calls within the same agent session (not just one call). `shared_state` persists across *all* agent sessions globally | +| `json.loads(s)` / `json.dumps(obj, indent=..., sort_keys=...)` | Parse / serialize JSON with optional formatting | | `sleep(seconds)` | Pause execution (0-120s max) | -| `print(...)` | Output messages; captured and returned in the result | -| `json.loads(s)` | Parse a JSON string into a Python dict/list | -| `json.dumps(obj)` | Serialize a Python object to a JSON string | -| `reset()` | Clear all local variables (does not touch `state`/`shared_state`) | +| `print(...)` / `reset(local_vars=True, state=False)` | Output messages; clear local namespace (and optionally state) | +| `typeof(x)` / `isinstance(x, t)` / `hasattr(x, n)` / `repr(x)` / `vars(obj)` | Type inspection and debugging | +| `allowed_methods()` | List all available builtin function names | + +### Available Modules -### Common Patterns +Pre-imported, read-only standard library modules: + +| Module | Common uses | +|--------|------------| +| `re` | Regular expressions: `re.search(r"pat", s)`, `re.findall(...)` | +| `math` | Math functions: `math.ceil(n)`, `math.sqrt(n)` | +| `itertools` | Combinatorics: `itertools.chain(a, b)`, `itertools.product(...)` | +| `collections` | Container helpers: `collections.Counter(...)`, `collections.defaultdict(...)` | +| `datetime` | Date/time: `datetime.datetime.now()`, `datetime.timedelta(...)` | +| `traceback` | Traceback formatting: `traceback.format_exc()`, `traceback.format_tb(...)` | + +### Usage -**Sequential calls:** ```python -delegate = Agent.get_tool("Delegate") -a = await delegate.call(delegations=[{"name": "worker", "prompt": "Analyze foo.py"}]) -b = await delegate.call(delegations=[{"name": "worker", "prompt": "Analyze bar.py"}]) -f"Results: {a} and {b}" +tool = Agent.get_tool("delegate") +tool_outputs = await gather( + a=tool.call(prompt="A"), + b=tool.call(prompt="B"), +) +print(tool_outputs.a) # attribute access +print(tool_outputs["b"]) # key access ``` -**Parallel calls:** +### Editing with Regions + +Use `Agent.resolve_regions()` to convert text patterns into content IDs, then `Agent.edit_region()` to apply edits using the resolved IDs. + +#### Step 1 — resolve region boundaries once + ```python -delegate = Agent.get_tool("Delegate") -tasks = [ - delegate.call(delegations=[{"name": "worker", "prompt": "Analyze a.py"}]), - delegate.call(delegations=[{"name": "worker", "prompt": "Analyze b.py"}]), - delegate.call(delegations=[{"name": "worker", "prompt": "Analyze c.py"}]), -] -results = await gather(*tasks) -f"Got {len(results)} results" +regions = Agent.resolve_regions("foo.py", [ + {"name": "my_func", "start": "def my_func", "end": "return result"}, + {"name": "init", "start": "def __init__", "end": "self.x = x"}, +]) ``` -**Accumulating state across calls:** +#### Step 2 — Use `regions.get(name)` with `Agent.edit_region()` (recommended shorthand) + ```python -state["count"] = state.get("count", 0) + len(some_result) -f"Total so far: {state['count']}" +await Agent.edit_region( + file_path="foo.py", + edits=[ + {"region": regions.get("my_func"), "text": "def my_func():\\n return 42"}, + ], +) ``` -**Structured tool responses:** -All tool calls return a dict has three keys: -- `result` — a list of result entries from the tool -- `errors` — a list of error strings (empty when successful) -- `details` — a list of extra contextual detail strings +#### Alternative: Call `EditFile` directly with `regions.get_start()` / `regions.get_end()` ```python -grep = Agent.get_tool("Grep") -response = await grep.call(searches=[{"pattern": "TODO", "directory": "cecli/tools"}]) - -for entry in response["result"]: - print(f"Match: {entry}") -for err in response["errors"]: - print(f"Error: {err}") -f"Found {len(response['result'])} matching files, {len(response['errors'])} errors" +edit_tool = Agent.get_tool("EditFile") +await edit_tool.call(edits=[{ + "file_path": "foo.py", + "operation": "replace", + "start_line": regions.get_start("my_func"), + "end_line": regions.get_end("my_func"), + "text": "def my_func():\\n return 42", +}]) ``` -### Tool Parameters - -Use the top-level parameters from each tool's schema as keyword arguments for `.call()`. -Refer to the tool descriptions for exact parameter names and types. +### Gotchas +- **Types**: compare with `typeof(x) == dict` or `isinstance(x, dict)` — NOT `typeof(x) == "dict"` +- **Args**: use keyword args only — `tool.call(file_path="f", ...)` +- **gather**: always use named `gather(x=a, y=b)` — positional args are not supported ### Rules -1. No imports - use only the primitives above +1. No imports - use only the primitives and modules above 2. Do not access attributes starting with `_` (private/dunder) 3. All tool calls must be awaited -4. The last expression's value is returned as the tool result +4. Use `print(...)` to output results — only printed output is returned """ + + +# flake8: noqa +# fmt: on diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py new file mode 100644 index 00000000000..8b5720dc04a --- /dev/null +++ b/cecli/helpers/orchestration/region_resolver.py @@ -0,0 +1,390 @@ +""" +Region resolution helpers for the orchestration sandbox. + +Provides AgentRegion for storing named region boundary *patterns* that are +resolved to content IDs on-demand at access time. This ensures content IDs +are always fresh — important after intervening edits shift hashline positions. +""" + +from __future__ import annotations + +from typing import Any + + +class AgentRegion: + """ + Stores named region boundary *patterns* resolved to content IDs on access. + + Content IDs can shift after edits, so ``get_start(name)`` and + ``get_end(name)`` re-read the file and re-resolve patterns at each call. + Use these directly in ``EditFile`` calls for always-fresh IDs. + + When a region spec uses a **content ID** (e.g. ``"abc::"``) instead of + text, the referenced line content is snapshotted on first resolution. + If the ID goes stale after intervening edits, subsequent resolutions + fall back to content matching against the snapshotted line text. + + Example usage in orchestration code:: + + regions = Agent.resolve_regions("foo.py", [ + {"name": "helper", "start": "def helper", "end": "return x"}, + ]) + edit_tool = Agent.get_tool("EditFile") + await edit_tool.call(edits=[{ + "file_path": "foo.py", + "operation": "replace", + "start_line": regions.get_start("helper"), + "end_line": regions.get_end("helper"), + "text": "def helper():\\n return 42", + }]) + """ + + def __init__( + self, + file_path: str, + coder: Any, + region_specs: list[dict[str, str]], + ) -> None: + self._file_path = file_path + self._coder = coder + self._specs: dict[str, dict[str, str]] = {} + + for spec in region_specs: + name = spec["name"] + entry: dict[str, object] = { + "start": spec["start"], + "end": spec["end"], + } + if "start_line_hint" in spec: + entry["start_line_hint"] = spec["start_line_hint"] + if "end_line_hint" in spec: + entry["end_line_hint"] = spec["end_line_hint"] + self._specs[name] = entry + + # Eagerly validate uniqueness for all regions at creation time. + # This catches ambiguous patterns immediately with clear error messages. + self._eager_validate() + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def get_start(self, name: str) -> str: + """Re-read file and return the current content ID for the start of *name*.""" + + return self._resolve(name)[0] + + def get_end(self, name: str) -> str: + """Re-read file and return the current content ID for the end of *name*.""" + + return self._resolve(name)[1] + + def get_start_line(self, name: str) -> int: + """Re-read file and return the current 1-based start line for *name*.""" + + return self._resolve(name)[2] + + def get_end_line(self, name: str) -> int: + """Re-read file and return the current 1-based end line for *name*.""" + + return self._resolve(name)[3] + + def __contains__(self, name: str) -> bool: + return name in self._specs + + def __len__(self) -> int: + return len(self._specs) + + def names(self) -> list[str]: + """Return the list of region names.""" + + return sorted(self._specs.keys()) + + def get(self, name: str) -> dict[str, str]: + """Return ``{"start": content_id, "end": content_id}`` for *name*. + + The returned dict can be passed directly as the ``region`` value + in ``Agent.edit_region()`` edits. + """ + + return {"start": self.get_start(name), "end": self.get_end(name)} + + def __repr__(self) -> str: + names = ", ".join(sorted(self._specs.keys())) + return f"AgentRegion({len(self._specs)} regions on " f"{self._file_path!r}: {names})" + + # ------------------------------------------------------------------ + # Resolution internals + # ------------------------------------------------------------------ + + def _eager_validate(self) -> None: + """Eagerly resolve and validate all region patterns at init time. + + Raises ValueError immediately for ambiguous patterns so the LLM + gets clear feedback without waiting for the first access. + """ + + for name in list(self._specs.keys()): + self._resolve(name) + + def _resolve(self, name: str) -> tuple[str, str, int, int]: + """ + Re-read file and resolve *name* to + (start_id, end_id, start_line, end_line). + + When a pattern is a content ID the referenced line is snapshotted + so future resolutions can fall back to content matching if the + original ID goes stale. + """ + + import os + + from cecli.helpers.hashline import ( + ContentHashError, + normalize_hashline, + resolve_content_to_hashline_ids, + ) + from cecli.helpers.hashpos.hashpos import HashPos + from cecli.tools.utils.helpers import resolve_paths + + spec = self._specs[name] + + # Read explicit line hints from spec (preferred over @L in patterns). + # 1-based in the spec, converted to 0-based internally. + explicit_start = spec.get("start_line_hint") + explicit_end = spec.get("end_line_hint") + + abs_path, rel_path = resolve_paths(self._coder, self._file_path) + + if not os.path.isfile(abs_path): + raise ValueError(f"File not found: {self._file_path}") + + content = self._coder.io.read_text(abs_path) + + if content is None: + raise ValueError(f"Could not read file: {self._file_path}") + + lines = content.splitlines() + hp = HashPos(content) + + start_pattern = self._resolve_pattern( + hp, lines, spec, "start", normalize_hashline, ContentHashError + ) + end_pattern = self._resolve_pattern( + hp, lines, spec, "end", normalize_hashline, ContentHashError + ) + + # Always strip @L hints from patterns — they are metadata, not literal text. + # Explicit hints (start_line_hint / end_line_hint) override any @L in patterns. + start_pattern, extracted_start = self._extract_l_hint(start_pattern) + end_pattern, extracted_end = self._extract_l_hint(end_pattern) + start_hint = (explicit_start - 1) if explicit_start is not None else extracted_start + end_hint = (explicit_end - 1) if explicit_end is not None else extracted_end + + # Validate uniqueness for text-based patterns (not content IDs or special markers). + self._validate_pattern_uniqueness(start_pattern, start_hint, "start", name, lines) + self._validate_pattern_uniqueness(end_pattern, end_hint, "end", name, lines) + + start_id, end_id = resolve_content_to_hashline_ids(content, start_pattern, end_pattern) + + # Resolve line numbers from content IDs + def _line_from_id(content_id: str, default_if_not_found: int) -> int: + if content_id == "@000": + return 1 + + if content_id == "000@": + return len(lines) + + try: + normalized = normalize_hashline(content_id) + candidates = hp.resolve_to_lines(normalized) + + if candidates: + return candidates[0] + 1 + except (ContentHashError, ValueError): + pass + + return default_if_not_found + + start_line = _line_from_id(start_id, -1) + end_line = _line_from_id(end_id, -1) + + if start_line < 0 or end_line < 0: + parts = [] + if start_line >= 0: + parts.append(f" Start pattern resolved to: {start_pattern!r} (line {start_line})") + else: + parts.append(f" Start pattern NOT FOUND: {start_pattern!r}") + if end_line >= 0: + parts.append(f" End pattern resolved to: {end_pattern!r} (line {end_line})") + else: + parts.append(f" End pattern NOT FOUND: {end_pattern!r}") + raise ValueError( + f"Could not resolve line numbers for region " + f"'{name}' in {self._file_path}\n" + "\n".join(parts) + ) + + return start_id, end_id, start_line, end_line + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + @staticmethod + def _search_in_lines(lines: list[str], pattern: str) -> list[int]: + """Return 0-based indices of all lines where *pattern* matches. + + Supports multiline patterns (each line of *pattern* must be + a substring of the corresponding line in *lines*). + """ + + pattern_lines = pattern.split("\n") + indices = [] + for i in range(len(lines) - len(pattern_lines) + 1): + if all(p_line in lines[i + j] for j, p_line in enumerate(pattern_lines)): + indices.append(i) + return indices + + @staticmethod + def _extract_l_hint(pattern: str) -> tuple[str, int | None]: + """Extract an @L line hint suffix from a pattern. + + Returns (stripped_pattern, 0_based_line_number_or_None). + """ + + import re + + m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) + if m: + return pattern[: m.start()], int(m.group(1)) - 1 + return pattern, None + + @staticmethod + def _narrow_by_proximity(indices: list[int], target: int, max_results: int = 5) -> list[int]: + """Return up to *max_results* indices closest to *target* (0-based).""" + + if not indices: + return [] + scored = [(abs(i - target), i) for i in indices] + scored.sort(key=lambda x: x[0]) + return [idx for _, idx in scored[:max_results]] + + def _validate_pattern_uniqueness( + self, + pattern: str, + hint: int | None, + boundary: str, + name: str, + lines: list[str], + ) -> None: + """Raise ValueError if *pattern* matches multiple locations. + + Only validates text patterns — content IDs and special markers + (@000 / 000@) are inherently unique and are skipped. + """ + + if self._looks_like_content_id(pattern): + return + if pattern in ("@000", "000@"): + return + + matches = self._search_in_lines(lines, pattern) + if len(matches) <= 1: + return + + # Try @L hint narrowing — find the unique closest match + if hint is not None: + best = min(matches, key=lambda i: abs(i - hint)) + best_dist = abs(best - hint) + conflicts = [i for i in matches if i != best and abs(i - hint) == best_dist] + if not conflicts: + return # Unique closest match found + matches_for_display = [best] + conflicts + else: + matches_for_display = matches + + line_nums = [str(i + 1) for i in sorted(matches_for_display)] + display = ", ".join(line_nums[:10]) + if len(line_nums) > 10: + display += f", ... ({len(line_nums)} total)" + + if hint is not None: + raise ValueError( + f"{boundary.capitalize()} pattern '{pattern}' for region " + f"'{name}' has {len(matches)} matches; @L{hint + 1} hint ties " + f"between {len(matches_for_display)} equally-close locations " + f"(lines {display}). Use a more specific pattern." + ) + + raise ValueError( + f"{boundary.capitalize()} pattern '{pattern}' for region " + f"'{name}' matches {len(matches)} locations " + f"(lines {display}). " + f"Use a more specific pattern or append ' @L' to " + f"disambiguate (e.g., '{pattern} @L{line_nums[0]}')." + ) + + @staticmethod + def _looks_like_content_id(value: str) -> bool: + """Return True if *value* appears to be a content ID rather than text.""" + + from cecli.helpers.hashline import ContentHashError, normalize_hashline + + if value in ("@000", "000@"): + return True + + try: + normalize_hashline(value) + + return True + except (ContentHashError, ValueError): + return False + + def _resolve_pattern( + self, + hp, + lines: list[str], + spec: dict[str, str], + key: str, + normalize_hashline, + ContentHashError, + ) -> str: + """ + Resolve a single boundary pattern. + + Content-ID patterns have their referenced line content snapshotted + so stale IDs can be recovered via content matching. + """ + + pattern = spec[key] + + # Special markers never go stale + if pattern in ("@000", "000@"): + return pattern + + if not self._looks_like_content_id(pattern): + return pattern + + # Content ID — try to resolve and snapshot the line content + content_key = f"_{key}_content" + + try: + normalized = normalize_hashline(pattern) + candidates = hp.resolve_to_lines(normalized) + + if candidates and candidates[0] < len(lines): + spec[content_key] = lines[candidates[0]] + + return pattern + except (ContentHashError, ValueError): + pass + + # Content ID may be stale — fall back to snapshotted line content + cached = spec.get(content_key) + + if cached: + return cached + + # No fallback available; return the (stale) ID and let the + # caller's resolution logic handle the error + return pattern diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py new file mode 100644 index 00000000000..2910c56490b --- /dev/null +++ b/cecli/helpers/orchestration/safe_methods.py @@ -0,0 +1,611 @@ +""" +Safe primitives and shim classes for the orchestration sandbox. + +Provides helper functions, container classes, and module proxies +that are injected into the sandbox globals. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +# --------------------------------------------------------------------------- +# Safe primitives exposed to the sandbox +# --------------------------------------------------------------------------- + + +async def _safe_sleep(seconds: float) -> None: + """Safe sleep wrapper for the orchestration environment.""" + if seconds < 0: + raise ValueError("sleep() requires a non-negative value") + if seconds > 120: + raise ValueError("sleep() is limited to 120 seconds maximum") + await asyncio.sleep(seconds) + + +class GatherResult: + """Result container for named ``gather()`` calls. + + Supports both attribute access (``results.my_task``) and + key access (``results["my_task"]``), plus ``len()`` and + iteration for unpacking. + """ + + def __init__(self, results: dict[str, Any]) -> None: + object.__setattr__(self, "_results", results) + + def __getattr__(self, name: str) -> Any: + if name.startswith("_"): + raise AttributeError( + "Cannot access private attribute " + repr(name) + " on GatherResult" + ) + results = object.__getattribute__(self, "_results") + if name not in results: + raise AttributeError( + "GatherResult has no key " + + repr(name) + + ". Available keys: " + + ", ".join(sorted(results.keys())) + ) + return results[name] + + def __getitem__(self, key: str) -> Any: + results = object.__getattribute__(self, "_results") + if not isinstance(key, str): + raise TypeError( + f"GatherResult keys must be strings, got {type(key).__name__}. " + f"Use attribute access: results.my_key, " + f"or string keys: results['my_key']. " + f"Available keys: {', '.join(sorted(results.keys()))}" + ) + if key not in results: + raise KeyError( + f"GatherResult has no key {key!r}. " + f"Available keys: {', '.join(sorted(results.keys()))}" + ) + return results[key] + + def __setattr__(self, name: str, value: Any) -> None: + if not name.startswith("_"): + raise AttributeError("GatherResult is read-only: cannot set " + repr(name)) + object.__setattr__(self, name, value) + + def __len__(self) -> int: + results = object.__getattribute__(self, "_results") + return len(results) + + def __iter__(self): + results = object.__getattribute__(self, "_results") + return iter(results.keys()) + + def keys(self) -> Any: + results = object.__getattribute__(self, "_results") + return results.keys() + + def values(self) -> Any: + results = object.__getattribute__(self, "_results") + return results.values() + + def items(self) -> Any: + results = object.__getattribute__(self, "_results") + return results.items() + + def __dir__(self): + results = object.__getattribute__(self, "_results") + return sorted(set(super().__dir__()) | set(results.keys())) + + def __contains__(self, key: str) -> bool: + results = object.__getattribute__(self, "_results") + return key in results + + def __repr__(self) -> str: + results = object.__getattribute__(self, "_results") + inner = ", ".join(f"{k}={type(v).__name__}" for k, v in results.items()) + return f"GatherResult({inner})" + + +async def _safe_gather(**named_awaitables: Any): + """ + Safely execute multiple awaitables concurrently. + + All awaitables must be passed as keyword arguments. Results are returned + as a ``GatherResult`` with attribute and key access:: + + results = await gather(read_a=task_a, grep_b=task_b) + print(results.read_a) # attribute access + print(results["grep_b"]) # key access + len(results) # number of results + + Forces ``return_exceptions=True`` so that failures in one task + do not crash the entire batch. Exceptions are converted to + structured error dicts. + """ + if not named_awaitables: + return GatherResult({}) + + keys = list(named_awaitables.keys()) + coros = list(named_awaitables.values()) + results = await asyncio.gather(*coros, return_exceptions=True) + processed = {} + for key, r in zip(keys, results): + if isinstance(r, BaseException): + processed[key] = { + "result": [], + "errors": [f"{type(r).__name__}: {r}"], + "details": [], + } + else: + processed[key] = r + return GatherResult(processed) + + +def _safe_typeof(obj: Any) -> type: + """Safe type inspection — only accepts 1 argument, cannot create classes. + + Unlike ``type(name, bases, dict)`` which can create new classes at runtime, + ``_safe_typeof(obj)`` only returns the type of an object for inspection. + """ + return type(obj) + + +def _safe_vars(obj: Any) -> dict: + """Safe vars() - only accepts 1 argument, returns non-dunder attributes. + + Unlike ``vars()`` with no arguments (which returns the local namespace), + ``vars(obj)`` returns the non-dunder instance attributes of a single object. + For objects without ``__dict__`` (e.g. basic types, ``__slots__``-only objects), + returns an empty dict. + + Private and dunder attributes are silently excluded from the result. + Objects with problematic ``__dict__`` or ``__getattr__`` implementations + are handled gracefully (empty dict returned) rather than crashing. + """ + try: + if hasattr(obj, "__dict__"): + d = obj.__dict__ + if isinstance(d, dict): + return {k: v for k, v in d.items() if not k.startswith("_")} + except Exception: + pass + + # Handle __slots__-only objects + try: + for cls in type(obj).__mro__: + slots = getattr(cls, "__slots__", ()) + if slots: + result = {} + for slot in slots: + if not slot.startswith("_") and hasattr(obj, slot): + try: + result[slot] = getattr(obj, slot) + except Exception: + pass + if result: + return result + except Exception: + pass + + return {} + + +def _safe_dir(obj: Any) -> list: + """Safe dir() - returns public attributes only, without builtins or dunders. + + Filters out names starting with ``_`` and common builtin attributes + inherited from ``object`` (e.g., ``__class__``, ``__dict__``, etc.). + + Unlike the builtin ``dir()`` which includes everything, this returns + only the user-facing attributes and methods of the object. + """ + all_attrs = dir(obj) + + return [ + a + for a in all_attrs + if not a.startswith("_") + and a + not in ( + "__builtins__", + "__cached__", + "__doc__", + "__file__", + "__loader__", + "__name__", + "__package__", + "__spec__", + ) + ] + + +def _naive_escape_newlines(code: str, NL: str, BSN: str) -> str: + """Replace literal newlines with ``\\n`` inside all string literals. + + Simple quote-matching — does not understand f-string expression boundaries. + Callers should validate the result with ``compile()`` and fall back to the + f-string-aware variant when this produces invalid code.""" + + result = [] + idx = 0 + while idx < len(code): + ch = code[idx] + if ch == chr(34) or ch == chr(39): + quote = ch + if idx + 2 < len(code) and code[idx : idx + 3] == quote * 3: + quote = quote * 3 + result.append(quote) + idx += len(quote) + end = code.find(quote, idx) + if end == -1: + result.append(code[idx:]) + idx = len(code) + break + inner = code[idx:end] + inner = inner.replace(NL, BSN) + result.append(inner) + result.append(quote) + idx = end + len(quote) + elif ch == chr(35): + nl = code.find(NL, idx) + if nl == -1: + result.append(code[idx:]) + idx = len(code) + else: + result.append(code[idx:nl]) + idx = nl + else: + result.append(ch) + idx += 1 + return "".join(result) + + +def _process_fstring_body(code: str, idx: int, quote: str, NL: str, BSN: str, result: list) -> int: + """Process the body of an f-string, escaping literal newlines only in + string-literal portions (brace-depth 0). Expression portions (brace-depth + >= 1) are appended verbatim so that multi-line expressions survive.""" + + literal_start = idx + pos = idx + brace_depth = 0 + + while pos < len(code): + c = code[pos] + + # Closing quote when not inside an expression + if brace_depth == 0 and c == quote: + if len(quote) == 1: + break + # triple-quoted — need three consecutive quote chars + if pos + 2 < len(code) and code[pos : pos + 3] == quote: + break + pos += 1 + continue + + # Backslash — skip the escaped character + if c == chr(92): + pos += 2 + continue + + # Opening brace + if c == chr(123): + if pos + 1 < len(code) and code[pos + 1] == chr(123): + pos += 2 # {{ — literal brace + continue + if brace_depth == 0: + # Flush the string-literal portion up to { + inner = code[literal_start:pos] + inner = inner.replace(NL, BSN) + result.append(inner) + result.append(c) + literal_start = pos + 1 + brace_depth += 1 + pos += 1 + continue + + # Closing brace + if c == chr(125): + if pos + 1 < len(code) and code[pos + 1] == chr(125): + pos += 2 # }} — literal brace + continue + if brace_depth > 0: + brace_depth -= 1 + if brace_depth == 0: + # Expression block ended — append verbatim + result.append(code[literal_start:pos]) + result.append(c) + literal_start = pos + 1 + pos += 1 + continue + + # Nested string literal inside an expression — skip over it + if brace_depth > 0 and (c == chr(34) or c == chr(39)): + nq = c + if pos + 2 < len(code) and code[pos : pos + 3] == nq * 3: + nq = nq * 3 + pos += len(nq) + ne = code.find(nq, pos) + pos = (ne + len(nq)) if ne != -1 else len(code) + continue + + pos += 1 + + # Flush the final string-literal portion + if literal_start < pos: + inner = code[literal_start:pos] + inner = inner.replace(NL, BSN) + result.append(inner) + + if pos < len(code): + result.append(quote) + pos += len(quote) + + return pos + + +def _fstring_aware_escape_newlines(code: str, NL: str, BSN: str) -> str: + """Like ``_naive_escape_newlines``, but f-string expression blocks + + (``{...}``) have their newlines preserved verbatim.""" + + result = [] + idx = 0 + while idx < len(code): + ch = code[idx] + if ch == chr(34) or ch == chr(39): + quote = ch + if idx + 2 < len(code) and code[idx : idx + 3] == quote * 3: + quote = quote * 3 + + # Detect f-string prefix + prev = idx - 1 + is_fstring = prev >= 0 and code[prev] in "fF" + if not is_fstring and prev >= 1: + if code[prev] in "rR" and code[prev - 1] in "fF": + is_fstring = True + + result.append(quote) + idx += len(quote) + + if is_fstring: + idx = _process_fstring_body(code, idx, quote, NL, BSN, result) + else: + end = code.find(quote, idx) + if end == -1: + result.append(code[idx:]) + idx = len(code) + break + inner = code[idx:end] + inner = inner.replace(NL, BSN) + result.append(inner) + result.append(quote) + idx = end + len(quote) + elif ch == chr(35): + nl = code.find(NL, idx) + if nl == -1: + result.append(code[idx:]) + idx = len(code) + else: + result.append(code[idx:nl]) + idx = nl + else: + result.append(ch) + idx += 1 + return "".join(result) + + +def _escape_newlines_in_strings(code: str) -> str: + """Pre-process code to escape literal newlines inside string literals. + + So that LLMs can write backslash-n literally in strings without + Python interpreting it as an actual newline causing SyntaxError. + + Uses a two-pass strategy: + 1. Try naive quote-matching (fast, handles regular strings). + 2. If ``compile`` rejects the result, fall back to an f-string-aware + pass that preserves newlines inside ``{...}`` expression blocks. + """ + NL = chr(10) + + # Fast path — no literal newlines, nothing to do + if NL not in code: + return code + + # Fast path — code is already valid Python + try: + compile(code, "", "exec") + return code + except SyntaxError: + pass + + BSN = chr(92) + chr(110) + + # Pass 1 — naive quote-matching (works for regular strings) + escaped = _naive_escape_newlines(code, NL, BSN) + try: + compile(escaped, "", "exec") + return escaped + except SyntaxError: + pass + + # Pass 2 — f-string aware (preserves newlines in expression blocks) + return _fstring_aware_escape_newlines(code, NL, BSN) + + +# Modules that are pre-imported in the sandbox. Import statements for +# these can be safely commented out instead of triggering SecurityError. +_PREIMPORTED_MODULES: frozenset[str] = frozenset( + { + "re", + "math", + "itertools", + "collections", + "datetime", + "traceback", + } +) + + +def _strip_allowed_imports(code: str) -> tuple[str, dict[str, object]]: + """Strip import lines for modules already pre-imported in the sandbox. + + The sandbox provides ``re``, ``math``, ``itertools``, ``collections``, + ``datetime``, and ``traceback`` as read-only proxies. + + Returns ``(code, extra_globals)`` where *extra_globals* maps imported + names to their resolved values. The caller must inject these into the + execution namespace before running the code. + + ``import module`` lines are commented out — the module name is already + available as a global. + + ``from module import name [as alias]`` lines are commented out and the + corresponding entries are added to *extra_globals* so the names are + available without Python scoping issues. + """ + + import re as _re + + lines = code.splitlines() + result = [] + extra_globals: dict[str, object] = {} + + for line in lines: + stripped = line.lstrip() + indent = line[: len(line) - len(stripped)] + + # "import module" or "import module as alias" + m = _re.match( + r"import\s+(\w+)(?:\s+as\s+(\w+))?\s*$", + stripped, + ) + if m and m.group(1) in _PREIMPORTED_MODULES: + mod_name = m.group(1) + alias = m.group(2) or mod_name + + result.append(f"{indent}# {stripped} " f"# auto-removed: {mod_name} is pre-imported") + # Only inject if aliased (otherwise the module name is already + # available as-is) + if alias != mod_name: + import importlib + + extra_globals[alias] = importlib.import_module(mod_name) + continue + + # "from module import name1 [as alias1], ..." + m = _re.match( + r"from\s+(\w+)\s+import\s+(.*)$", + stripped, + ) + if m and m.group(1) in _PREIMPORTED_MODULES: + mod_name = m.group(1) + names_clause = m.group(2) + + result.append(f"{indent}# {stripped} " f"# auto-removed: {mod_name} is pre-imported") + + for name_item in names_clause.split(","): + name_item = name_item.strip() + as_match = _re.match( + r"(\w+)\s+as\s+(\w+)\s*$", + name_item, + ) + if as_match: + extra_globals[as_match.group(2)] = _resolve_module_attr( + mod_name, as_match.group(1) + ) + else: + extra_globals[name_item] = _resolve_module_attr(mod_name, name_item) + + continue + + result.append(line) + + return "\n".join(result), extra_globals + + +def _resolve_module_attr(mod_name: str, attr: str) -> object: + """Return ``mod_name.attr`` by importing the real module.""" + + import importlib + + module = importlib.import_module(mod_name) + return getattr(module, attr) + + +class _SafeJson: + """Drop-in ``json`` namespace with ``loads``, ``dumps``, and ``JSONDecodeError``.""" + + JSONDecodeError = json.JSONDecodeError + + @staticmethod + def loads(s: str) -> Any: + import json + + return json.loads(s) + + @staticmethod + def dumps(obj: Any, **kwargs) -> str: + import json + + # Whitelist of safe kwargs for formatting + allowed = {"indent", "sort_keys", "default", "separators", "ensure_ascii", "allow_nan"} + safe_kwargs = {k: v for k, v in kwargs.items() if k in allowed} + return json.dumps(obj, **safe_kwargs) + + +class _SafeModuleProxy: + """Proxy that forwards attribute reads to a real module but prevents mutation. + + Setting an attribute on the proxy only affects the proxy, not the real module. + This prevents sandbox code from monkey-patching standard library modules. + """ + + def __init__(self, module: Any) -> None: + object.__setattr__(self, "_module", module) + + def __getattr__(self, name: str) -> Any: + if name.startswith("_"): + raise AttributeError( + "Cannot access private attribute " + repr(name) + " on module proxy" + ) + module = object.__getattribute__(self, "_module") + return getattr(module, name) + + def __setattr__(self, name: str, value: Any) -> None: + if not name.startswith("_"): + raise AttributeError("Module proxy is read-only — cannot set " + repr(name)) + object.__setattr__(self, name, value) + + def __dir__(self) -> list: + module = object.__getattribute__(self, "_module") + return [x for x in dir(module) if not x.startswith("_")] + + +# --------------------------------------------------------------------------- +# Helpful builtins +# --------------------------------------------------------------------------- + + +class _HelpfulBuiltins(dict): + """Custom __builtins__ dict that provides helpful hints for missing functions.""" + + _HINTS: dict[str, str] = { + "open": "Filesystem access is not available. Use the Command tool instead.", + "eval": "eval() is disabled for security.", + "exec": "exec() is disabled for security.", + "__import__": "Imports are disabled. Use only the primitives provided.", + "compile": "compile() is disabled for security.", + "breakpoint": "breakpoint() is disabled in the sandbox.", + "globals": "globals() is disabled. Use state or shared_state for persistence.", + "locals": "locals() is disabled. Use state or shared_state for persistence.", + "vars": "Use vars(obj) instead of vars() — vars(obj) returns non-dunder attrs of obj.", + "getattr": "getattr() is disabled. Access attributes directly.", + "setattr": "setattr() is disabled. Assign attributes directly.", + "delattr": "delattr() is disabled. Use del obj.attr instead.", + } + + def __missing__(self, key: str): + hint = self._HINTS.get(key) + if hint: + raise NameError(f"'{key}' is not available. {hint}") + raise NameError(f"name '{key}' is not defined") diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py index 28bdd0c5e3a..9067c522cbf 100644 --- a/cecli/helpers/orchestration/security.py +++ b/cecli/helpers/orchestration/security.py @@ -34,7 +34,6 @@ class SecurityFilter(ast.NodeVisitor): "breakpoint", "globals", "locals", - "vars", "getattr", "setattr", "delattr", @@ -46,8 +45,13 @@ def visit_Import(self, node: ast.Import) -> None: def visit_ImportFrom(self, node: ast.ImportFrom) -> None: raise SecurityError("Imports are disabled in the agent orchestration environment.") + _SAFE_DUNDER: set[str] = {"__name__", "__doc__"} + def visit_Attribute(self, node: ast.Attribute) -> None: if node.attr.startswith("_"): + if node.attr in self._SAFE_DUNDER: + self.generic_visit(node) + return raise SecurityError(f"Access to private/dunder attribute '{node.attr}' is forbidden.") self.generic_visit(node) diff --git a/cecli/helpers/orchestration/tool_proxy.py b/cecli/helpers/orchestration/tool_proxy.py new file mode 100644 index 00000000000..f38bf44bf82 --- /dev/null +++ b/cecli/helpers/orchestration/tool_proxy.py @@ -0,0 +1,223 @@ +""" +Proxy for a single tool — local or MCP. + +The LLM code calls ``tool.call(**params)`` and this proxy routes it +through the appropriate execution path. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + + +class ToolProxy: + """ + Proxy for a single tool — local or MCP. + + The LLM code calls ``tool.call(**params)`` and this proxy routes it + through the appropriate execution path. + """ + + def __init__( + self, + tool_name: str, + coder: Any, + *, + tool_module: Any = None, + mcp_server: Any = None, + mcp_tool_name: str = "", + ) -> None: + # Respect the per-coder tool includelist/excludelist filters + incl = getattr(coder, "registered_tools", {}).get("included", set()) + excl = getattr(coder, "registered_tools", {}).get("excluded", set()) + name_lower = tool_name.lower() + if incl and name_lower not in incl: + raise ValueError(f"Tool '{tool_name}' is not in the allowed tools list.") + if name_lower in excl: + raise ValueError(f"Tool '{tool_name}' has been excluded.") + + self._tool_name = tool_name + self._coder = coder + self._tool_module = tool_module + self._mcp_server = mcp_server + self._mcp_tool_name = mcp_tool_name + + async def __call__(self, *args: Any, **kwargs: Any): + """Make the proxy directly callable. + + Supports both ``await tool(key=val)`` and ``await tool("val")``. + Positional arguments are mapped to parameter names using the + tool's schema (when available). + """ + if args and kwargs: + raise TypeError( + f"Tool '{self._tool_name}': cannot mix positional and keyword arguments" + ) + + if args: + param_names = self._get_param_names() + if not param_names: + if len(args) == 1: + # Fallback: try common first-param names + for guess in ( + "path", + "read", + "searches", + "edits", + "queries", + "tasks", + "delegations", + "code", + "command_string", + "command", + "summary", + ): + kwargs = {guess: args[0]} + break + else: + raise TypeError( + f"Tool '{self._tool_name}': cannot resolve positional " + f"argument – no schema available" + ) + else: + raise TypeError( + f"Tool '{self._tool_name}': cannot resolve positional " + f"arguments – no schema available" + ) + elif len(args) > len(param_names): + raise TypeError( + f"Tool '{self._tool_name}': too many positional arguments " + f"({len(args)} for {len(param_names)} parameter(s): {param_names})" + ) + else: + kwargs = dict(zip(param_names, args)) + + return await self.call(**kwargs) + + def _get_param_names(self) -> list: + """Extract ordered parameter names from the tool's JSON Schema.""" + if self._tool_module is None: + return [] + try: + props = self._tool_module.SCHEMA["function"]["parameters"]["properties"] + return list(props.keys()) + except (KeyError, TypeError, AttributeError): + return [] + + async def call(self, **kwargs: Any): + """Execute the tool with the given keyword arguments. + + Tool results are normalized to a dict with ``result`` (list), + ``errors`` (list), and ``details`` (list) keys, matching the + documented orchestration contract. + """ + + if self._tool_module is not None: + result = self._tool_module.process_response(self._coder, kwargs, _stringify=False) + if asyncio.iscoroutine(result): + result = await result + result = self._tool_module.ptc_format(result) + return self._normalize_result(result) + + if self._mcp_server is not None: + result = await self._coder._execute_mcp_tool( + self._mcp_server, self._mcp_tool_name, kwargs + ) + return self._normalize_result(result) + + raise ValueError(f"No executor for tool '{self._tool_name}'") + + @staticmethod + def _normalize_result(result: Any) -> dict: + """Normalize a tool result into a unified dict with ``result``, ``errors``, ``details`` keys. + + Each item in the ``result`` list has the shape ``{"content": ..., "_": {...}}``. + """ + + from cecli.tools.utils.responses import ToolResponse + + if isinstance(result, ToolResponse): + data = result.to_dict() + return { + "result": data.get("result", []), + "errors": data.get("errors", []), + "details": data.get("details", []), + } + + if isinstance(result, str): + # Attempt to auto-parse a JSON-stringified ToolResponse + try: + parsed = json.loads(result) + if isinstance(parsed, dict): + if "result" in parsed: + result_list = parsed["result"] + if not isinstance(result_list, list): + result_list = [result_list] if result_list else [] + return { + "result": [ + ( + item + if isinstance(item, dict) and "content" in item + else {"content": item, "_": {}} + ) + for item in result_list + ], + "errors": parsed.get("errors", []), + "details": parsed.get("details", []), + } + return { + "result": [{"content": parsed, "_": {}}], + "errors": [], + "details": [], + } + except (json.JSONDecodeError, TypeError): + pass + + # Error strings from handle_tool_error + if result.startswith("Error in "): + return { + "result": [], + "errors": [result], + "details": [], + } + + # Plain string result + return { + "result": [{"content": result, "_": {}}], + "errors": [], + "details": [], + } + + if isinstance(result, dict): + if "result" in result: + result_list = result["result"] + if not isinstance(result_list, list): + result_list = [result_list] if result_list else [] + + out = dict(result) + out["result"] = [ + ( + item + if isinstance(item, dict) and "content" in item + else {"content": item, "_": {}} + ) + for item in result_list + ] + out.setdefault("errors", []) + out.setdefault("details", []) + return out + + return { + "result": [{"content": result, "_": {}}], + "errors": [], + "details": [], + } + + # Fallback: wrap anything else + return { + "result": [{"content": str(result), "_": {}}] if result is not None else [], + "errors": [], + "details": [], + } diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index c28f837adc4..133b10671c1 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -412,6 +412,8 @@ def execute( # 5. Format and return result + cls.clear_invocation_cache() + if files_processed == 1: # Single file case result = all_results[0] @@ -420,26 +422,37 @@ def execute( ) if result["failed_edits"]: success_message += f" ({len(result['failed_edits'])} failed)" - # Include failed edit details in message to LLM success_message += "\nFailed edits:\n" + "\n".join(result["failed_edits"]) - change_id_to_return = result.get("change_id") - else: - # Multiple files case - success_message = ( - f"Applied {total_successful_edits} edits across {files_processed} files" - ) - if all_failed_edits: - success_message += f" ({len(all_failed_edits)} failed)" - # Include failed edit details in message to LLM - success_message += "\nFailed edits:\n" + "\n".join(all_failed_edits) - change_id_to_return = None # Multiple change IDs, can't return single one - cls.clear_invocation_cache() + response.append_result( + content=f"\u2713 {success_message}", + metadata={ + "change_id": result.get("change_id"), + "file_path": result.get("file_path"), + "successful_edits": result.get("successful_edits"), + "failed_edits": result.get("failed_edits", []), + }, + ) + else: + # Multiple files case — append per-file structured results + for result in all_results: + per_file_message = ( + f"Applied {result['successful_edits']} edits in {result['file_path']}" + ) + if result["failed_edits"]: + per_file_message += f" ({len(result['failed_edits'])} failed)" + per_file_message += "\nFailed edits:\n" + "\n".join(result["failed_edits"]) + + response.append_result( + content=f"\u2713 {per_file_message}", + metadata={ + "change_id": result.get("change_id"), + "file_path": result.get("file_path"), + "successful_edits": result.get("successful_edits"), + "failed_edits": result.get("failed_edits", []), + }, + ) - response.append_result( - f"\u2713 {success_message}" - + (f" (change_id: {change_id_to_return})" if change_id_to_return else "") - ) return response except ToolError as e: diff --git a/cecli/tools/explore_code.py b/cecli/tools/explore_code.py index cdf875942a2..5a9df0ff5e3 100644 --- a/cecli/tools/explore_code.py +++ b/cecli/tools/explore_code.py @@ -128,7 +128,8 @@ def execute(cls, coder, queries, **kwargs): # Replace hyphens with underscores (common in code) and strip special chars. safe_symbol = symbol.replace("-", "_") if symbol else symbol results = c.search(safe_symbol, limit=limit) - response.append_result(cls._format_search_results(results, symbol)) + results = cls._filter_gitignored(results, coder) + response.append_result(content=cls._format_search_results(results, symbol)) elif action == "investigate": symbol_name = symbol file_hint = "" @@ -143,27 +144,41 @@ def execute(cls, coder, queries, **kwargs): try: investigation = c.investigate(safe_name, file_hint) + investigation = cls._filter_investigation_gitignored( + investigation, coder + ) response.append_result( - cls._format_investigation_results(investigation, symbol) + content=cls._format_investigation_results(investigation, symbol) ) except Exception as e: if "multiple matches" in str(e).lower(): results = c.search(symbol_name, limit=10) - locations = "\n".join( - [f"- {r['file']}:{r['start_line']}" for r in results] - ) - msg = ( - f"Error: Multiple matches found for '{symbol}'.\nPlease use a" - " more specific name or check the locations" - f" below:\n{locations}" + response.append_result( + content={ + "action": "investigate", + "symbol": symbol, + "error": "Multiple matches found", + "hint": ( + "Please use a more specific name or filename:symbol format" + ), + "locations": [ + { + "file": r.get("rel_path") or r.get("file", ""), + "start_line": r.get("start_line", 0), + } + for r in results + ], + } ) - response.append_result(msg) else: raise e elif action == "find_references": safe_symbol = symbol.replace("-", "_") if symbol else symbol references = c.find_references(safe_symbol, limit=limit) - response.append_result(cls._format_reference_results(references, symbol)) + references = cls._filter_gitignored(references, coder) + response.append_result( + content=cls._format_reference_results(references, symbol) + ) else: all_failed_queries.append( f"Error for symbol '{symbol}': Unknown action '{action}'" @@ -198,115 +213,142 @@ def execute(cls, coder, queries, **kwargs): c.close() @classmethod - def _format_search_results(cls, results, symbol): - """Format search results for display.""" - if not results: - return f"No symbols found matching '{symbol}'" + def _filter_gitignored(cls, results, coder): + """Filter out results whose file path is git-ignored.""" + + if not results or not hasattr(coder, "repo") or not coder.repo: + return results - formatted = [f"Found {len(results)} symbols matching '{symbol}':"] - for i, result in enumerate(results[:15], 1): - name = result.get("name", "Unknown") - kind = result.get("kind", "unknown") - file = result.get("rel_path") or result.get("file", "Unknown") - start_line = result.get("start_line", 0) - signature = result.get("signature", "") - parent = result.get("parent") + filtered = [] + for r in results: + file_path = r.get("rel_path") or r.get("file", "") + if not file_path: + filtered.append(r) + continue + if not coder.repo.git_ignored_file(file_path): + filtered.append(r) - location = f"{file}:{start_line}" - if parent: - location = f"{location} (in {parent})" + return filtered - formatted.append(f"{i}. {name}{signature} ({kind}) at {location}") + @classmethod + def _filter_investigation_gitignored(cls, investigation, coder): + """Filter git-ignored entries from an investigation result.""" + + if not investigation or not hasattr(coder, "repo") or not coder.repo: + return investigation + + # Filter references + if "refs" in investigation: + investigation["refs"] = cls._filter_gitignored(investigation["refs"], coder) - if len(results) > 15: - formatted.append(f"... and {len(results) - 15} more results") + # Filter impact/callers + if "impact" in investigation: + investigation["impact"] = cls._filter_gitignored(investigation["impact"], coder) + + return investigation + + @classmethod + def _format_search_results(cls, results, symbol): + """Format search results as structured data.""" - return "\n".join(formatted) + if not results: + return {"action": "search", "symbol": symbol, "count": 0, "results": []} + + return { + "action": "search", + "symbol": symbol, + "count": len(results), + "results": [ + { + "name": r.get("name", ""), + "kind": r.get("kind", ""), + "file": r.get("rel_path") or r.get("file", ""), + "start_line": r.get("start_line", 0), + "signature": r.get("signature", ""), + "parent": r.get("parent"), + } + for r in results + ], + } @classmethod def _format_investigation_results(cls, investigation, symbol): - """Format investigation results for display.""" + """Format investigation results as structured data.""" + if not investigation: - return f"No information found for symbol '{symbol}'" + return {"action": "investigate", "symbol": symbol, "error": "No information found"} # Handle nested structure if present if "results" in investigation and "result" in investigation["results"]: investigation = investigation["results"]["result"] - formatted = [f"Investigation of symbol '{symbol}':"] + result = { + "action": "investigate", + "symbol": symbol, + } # Extract definition information definition = investigation.get("symbol") if definition: - def_name = definition.get("name", symbol) - def_file = definition.get("rel_path") or definition.get("file", "Unknown") - def_line = definition.get("start_line", 0) - def_kind = definition.get("kind", "unknown") - def_sig = definition.get("signature", "") - formatted.append( - f"Definition: {def_name}{def_sig} ({def_kind}) at {def_file}:{def_line}" - ) + result["definition"] = { + "name": definition.get("name", symbol), + "file": definition.get("rel_path") or definition.get("file", ""), + "line": definition.get("start_line", 0), + "kind": definition.get("kind", ""), + "signature": definition.get("signature", ""), + } # Source code snippet source = investigation.get("source") if source: - formatted.append("\nSource Code:") - formatted.append("```python") - formatted.append(source.strip()) - formatted.append("```") + result["source"] = source.strip() # References references = investigation.get("refs", []) - ref_count = len(references) if references else 0 - formatted.append(f"\nReferences found: {ref_count}") - - if references and ref_count > 0: - formatted.append("Top references:") - for i, ref in enumerate(references[:10], 1): - ref_file = ref.get("rel_path") or ref.get("file", "Unknown") - ref_line = ref.get("line", 0) - formatted.append(f"{i}. {ref_file}:{ref_line}") - - if ref_count > 10: - formatted.append(f"... and {ref_count - 10} more references") + result["ref_count"] = len(references) if references else 0 + if references: + result["references"] = [ + { + "file": ref.get("rel_path") or ref.get("file", ""), + "line": ref.get("line", 0), + } + for ref in references + ] # Impact / Callers impact = investigation.get("impact", []) if impact: - formatted.append("\nImpact (Callers):") - for i, imp in enumerate(impact[:10], 1): - imp_file = imp.get("rel_path") or imp.get("file", "Unknown") - imp_line = imp.get("line", 0) - imp_caller = imp.get("caller", "unknown") - formatted.append(f"{i}. {imp_caller} at {imp_file}:{imp_line}") + result["impact"] = [ + { + "caller": imp.get("caller", ""), + "file": imp.get("rel_path") or imp.get("file", ""), + "line": imp.get("line", 0), + } + for imp in impact + ] - if len(impact) > 10: - formatted.append(f"... and {len(impact) - 10} more callers") - - return "\n".join(formatted) + return result @classmethod def _format_reference_results(cls, references, symbol): - """Format reference finding results for display.""" - if not references: - return f"No references found for symbol '{symbol}'" - - formatted = [f"Found {len(references)} references to '{symbol}':"] - for i, ref in enumerate(references[:15], 1): - file = ref.get("rel_path") or ref.get("file", "Unknown") - line = ref.get("line", 0) - context = ref.get("context", []) + """Format reference finding results as structured data.""" - formatted.append(f"{i}. {file}:{line}") - if context: - formatted.append(" Context:") - for line_text in context: - formatted.append(f" {line_text.strip()}") - - if len(references) > 15: - formatted.append(f"... and {len(references) - 15} more references") - - return "\n".join(formatted) + if not references: + return {"action": "find_references", "symbol": symbol, "count": 0, "results": []} + + return { + "action": "find_references", + "symbol": symbol, + "count": len(references), + "results": [ + { + "file": ref.get("rel_path") or ref.get("file", ""), + "line": ref.get("line", 0), + "context": ref.get("context", []), + } + for ref in references + ], + } @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index 3b658ac0556..f650ebe5680 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -507,7 +507,7 @@ def execute( op_result["has_more_files"] = has_more op_result["files"] = [ { - "path": pf["path"], + "file": pf["path"], "match_count": pf.get("count_from_pass", 0), "truncated": pf["path"] in truncated_files, "content": pf["content"], @@ -545,7 +545,26 @@ def execute( response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) for op_result in all_operation_results: - response.append_result(op_result) + files = op_result.get("files", []) + metadata = {k: v for k, v in op_result.items() if k != "files"} + + # Build a human-readable string summary as content + pattern = op_result.get("pattern", "") + if op_result.get("error"): + summary = f"[{pattern}]\nError: {op_result['error']}" + elif op_result.get("total_matches", 0) == 0: + summary = f"[{pattern}]\nNo matches found." + else: + lines = [f"[{pattern}]"] + for f in files: + truncated_mark = " (truncated)" if f.get("truncated") else "" + lines.append(f"{f['file']}: {f['match_count']} match(es){truncated_mark}") + if op_result.get("has_more_files"): + lines.append(f"... ({op_result['total_files']} files total)") + summary = "\n".join(lines) + + response.append_result(content=summary, metadata=metadata) + return response @classmethod diff --git a/cecli/tools/orchestrate.py b/cecli/tools/orchestrate.py index 4b0405f8186..9a5b80bceb6 100644 --- a/cecli/tools/orchestrate.py +++ b/cecli/tools/orchestrate.py @@ -46,12 +46,15 @@ async def execute(cls, coder, code, **kwargs): BaseTool.clear_invocation_cache() env = OrchestrationService.get_instance(coder) result = await env.execute(code) - response = ToolResponse(cls.NORM_NAME) - response.append_result(result) + response = ToolResponse(cls.NORM_NAME, result_type="list") + response.append_result( + content=result["results"], metadata={"state_variables": result["state_variables"]} + ) return response @classmethod def format_output(cls, coder, mcp_server, tool_response): + color_start, color_end = color_markers(coder) tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index 5da142db691..ff59c2a3ee2 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -238,16 +238,18 @@ def execute(cls, coder, read, **kwargs): if num_lines == 0: new_context_details.append( - "\n".join( - [ - f"File {rel_path} is empty.", - ( - "Next: use EditFile with start_line @000 and end_line @000 to" - " write content, or ResourceManager to scaffold — do not call" - " ReadFile again on this empty file." - ), - ] - ) + { + "file_path": rel_path, + "status": "full", + "total_lines": 0, + "prefixed_contents": "", + "outline": "", + "note": ( + f"File {rel_path} is empty. Next: use EditFile with start_line @000 and" + " end_line @000 to write content, or ResourceManager to scaffold —" + " do not call ReadFile again on this empty file." + ), + } ) new_context_retrieved.append(rel_path) cls._last_read_turn[abs_path] = coder.turn_count @@ -357,15 +359,20 @@ def execute(cls, coder, read, **kwargs): ) if cls._special_marker_count[abs_path] > 1: coder.abs_fnames.add(abs_path) - preview = f"Full contents of {rel_path} will be added to context in future message." + preview = { + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "prefixed_contents": "", + "outline": "", + "note": ( + f"Full contents of {rel_path} will be added to context in future message." + ), + } if abs_path in coder.abs_read_only_fnames: coder.abs_read_only_fnames.remove(abs_path) - preview_key = ( - json.dumps(preview, sort_keys=True) - if isinstance(preview, dict) - else preview - ) + preview_key = json.dumps(preview, sort_keys=True) if preview_key not in all_outputs_set: all_outputs_set.add(preview_key) if len(all_outputs): @@ -375,11 +382,20 @@ def execute(cls, coder, read, **kwargs): # If the range was large and we're showing a preview, add explicit guidance range_lines = e_idx - s_idx + 1 if not has_stub: - response.append_result( + note_text = ( f"read operation {read_index + 1}: {rel_path} range " f"({range_lines} lines) is large. " f"Use @L ranges (e.g., @L{s_idx + 1}, @L{e_idx + 1}) for precise reads." ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) continue @@ -529,13 +545,23 @@ def execute(cls, coder, read, **kwargs): type="tool-result", ) - response.append_result( + note_text = ( f"Retrieved context for {len(new_context_details)} operation(s). " "Full results for these reads will be given in a follow up message." - "Note: Full contents contain aggregated content across multiple reads." + " Note: Full contents contain aggregated content across multiple reads." + ) + response.append_result( + content=note_text, + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, ) for d in new_context_details: - response.append_result(d) + content_text = d.pop("prefixed_contents", "") + response.append_result(content=content_text, metadata=d) if already_up_to_details: coder.io.tool_output( ( @@ -545,23 +571,56 @@ def execute(cls, coder, read, **kwargs): type="tool-result", ) - response.append_result( + note_text = ( "Earlier contents still valid from previous read for " f"{len(already_up_to_details)} operation(s). " "Relevant contents for these reads available in previous message." ) + response.append_result( + content=note_text, + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) for d in already_up_to_details: - response.append_result(d) + content_text = d.pop("prefixed_contents", "") + response.append_result(content=content_text, metadata=d) if already_up_to_date and not new_context_retrieved: response.append_result( - "Do not call `ReadFile` again with these parameters again unless you edit" - " the relevant files." + content=( + "Do not call `ReadFile` again with these parameters again" + " unless you edit the relevant files." + ), + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, ) if all_outputs: for output in all_outputs: - response.append_result(output) - response.append_result("Use these outlines to refine your search.") + if output: + if isinstance(output, dict): + outline_content = output.pop("outline", "") + prefixed_content = output.pop("prefixed_contents", "") + preview_content = prefixed_content or outline_content + response.append_result(content=preview_content, metadata=output) + else: + response.append_result(output) + response.append_result( + content="Use these outlines to refine your search.", + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) if error_outputs: coder.io.tool_error( @@ -571,7 +630,15 @@ def execute(cls, coder, read, **kwargs): for err in error_outputs: response.append_error(err) - response.append_result(f"File Context Turn {coder.turn_count}") + response.append_result( + content=f"File Context Turn {coder.turn_count}", + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) return response except ToolError as e: @@ -603,10 +670,13 @@ def format_model_response( result = { "file_path": rel_path, + "status": "full", "start_line": s_idx + 1, "end_line": e_idx + 1, "total_lines": len(hashed_lines), "prefixed_contents": prefixed, + "outline": "", + "note": "", } return result @@ -854,6 +924,21 @@ def format_error(cls, coder, error_text, file_path, range_start, range_end, oper def on_duplicate_request(cls, coder, **kwargs): coder.edit_allowed = True + @classmethod + def ptc_format(cls, result): + """Strip placeholder entries from the result before sandbox exposure.""" + if isinstance(result, ToolResponse) and result.result_type == "list": + result._result = [ + item + for item in result._result + if not ( + isinstance(item, dict) + and isinstance(item.get("_"), dict) + and item["_"].get("status") == "placeholder" + ) + ] + return result + @classmethod def _extend_range_with_stub(cls, coder, abs_path, s_idx, e_idx, num_lines): """ @@ -1153,7 +1238,7 @@ def _disambiguate_start_indices( proximity_narrowed = cls._narrow_by_proximity(start_indices, start_hint, max_results=5) if proximity_narrowed and len(proximity_narrowed) <= 5: line_nums = [str(i + 1) for i in sorted(proximity_narrowed)] - response.append_result( + note_text = ( f"read operation {read_index + 1}: start pattern " f"'{range_start}' matched many locations in {rel_path}; " f"narrowed to {len(proximity_narrowed)} closest to @L hint " @@ -1161,6 +1246,15 @@ def _disambiguate_start_indices( f"Tip: append ' @L' to any pattern (e.g., " f"'{range_start} @L{start_hint + 1}') to target a specific match." ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) return proximity_narrowed narrowed = cls._try_fuzzy_narrow_indices( @@ -1173,7 +1267,7 @@ def _disambiguate_start_indices( if narrowed and len(narrowed) <= 5: line_nums = [str(i + 1) for i in sorted(narrowed)] - response.append_result( + note_text = ( f"read operation {read_index + 1}: start pattern " f"'{range_start}' matched many locations in {rel_path}; " f"narrowed to {len(narrowed)} structural match(es) " @@ -1181,15 +1275,33 @@ def _disambiguate_start_indices( f"Tip: append ' @L' to a pattern (e.g., " f"'{range_start} @L{line_nums[0]}') to target a specific match." ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) return narrowed line_nums = [str(i + 1) for i in sorted(start_indices[:5])] - response.append_result( + note_text = ( f"read operation {read_index + 1}: start pattern " f"'{range_start}' too broad ({len(start_indices)} matches) " f"in {rel_path}. Using first 5 (lines {', '.join(line_nums)}). " f"Refine with specific names or @L ranges for precision." ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) return start_indices[:5] @classmethod @@ -1366,6 +1478,8 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr io = coder.io abs_path, rel_path = resolve_paths(coder, abs_path) + content = io.read_text(abs_path) + stub = RepoMap.get_file_stub( abs_path, io, start_line=start_idx, end_line=end_idx, line_numbers=line_numbers ) @@ -1374,6 +1488,9 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr if stub and stub != "# No outline available": return { "file_path": rel_path, + "status": "outline", + "total_lines": len(content.splitlines()), + "prefixed_contents": "", "outline": stub, "note": ( f"Large File. Tip: use @L ranges for precise reads" @@ -1383,7 +1500,14 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr content = io.read_text(abs_path) if not content: - return "" + return { + "file_path": rel_path, + "status": "outline", + "total_lines": 0, + "prefixed_contents": "", + "outline": "", + "note": "Empty file.", + }, False lines = content.splitlines() num_file_lines = len(lines) @@ -1393,7 +1517,14 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr total_lines = actual_end - actual_start + 1 if total_lines <= 0: - return "", False + return { + "file_path": rel_path, + "status": "outline", + "total_lines": 0, + "prefixed_contents": "", + "outline": "", + "note": "Invalid range.", + }, False if total_lines <= 20: # Return all lines @@ -1433,4 +1564,14 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr parts.append("truncated:") parts.append("\n".join(file_contents)) - return "\n".join(parts), False + return { + "file_path": rel_path, + "status": "outline", + "total_lines": num_file_lines, + "prefixed_contents": "", + "outline": "\n".join(parts), + "note": ( + f"Tip: use @L ranges for precise reads" + f" (e.g., @L{actual_start + 1}, @L{actual_end + 1})." + ), + }, False diff --git a/cecli/tools/resource_manager.py b/cecli/tools/resource_manager.py index 14bbdfc47f3..a507601cc1b 100644 --- a/cecli/tools/resource_manager.py +++ b/cecli/tools/resource_manager.py @@ -17,6 +17,7 @@ class Tool(BaseTool): NORM_NAME = "resourcemanager" + RESULT_TYPE = "list" SCHEMA = { "type": "function", "function": { @@ -169,7 +170,7 @@ async def execute( ) coder.io.tool_output("⛭ Modifying Context", type="tool-result") - response = ToolResponse(cls.NORM_NAME) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) # Expand wildcards for MCP operations if "*" in load_mcp_servers and coder.mcp_manager: diff --git a/cecli/tools/utils/base_tool.py b/cecli/tools/utils/base_tool.py index cc612ed7399..bbe4ad75a91 100644 --- a/cecli/tools/utils/base_tool.py +++ b/cecli/tools/utils/base_tool.py @@ -41,16 +41,18 @@ def execute(cls, coder, **params): pass @classmethod - def process_response(cls, coder, params): + def process_response(cls, coder, params, _stringify=True): """ Process the tool response by creating an instance and calling execute. Args: coder: The Coder instance params: Dictionary of parameters + _stringify: If True (default), ToolResponse results are converted to str. + If False, ToolResponse is returned as-is for sandbox use. Returns: - str: Result message + str or ToolResponse: Result message (or ToolResponse when _stringify=False) """ # Validate required parameters from SCHEMA @@ -140,7 +142,7 @@ def process_response(cls, coder, params): result = cls.execute(coder, **params) if isinstance(result, ToolResponse): - return str(result) + return result if not _stringify else str(result) return result except Exception as e: @@ -163,3 +165,20 @@ def on_duplicate_request(cls, coder, **kwargs): def clear_invocation_cache(cls): cls._invocations.clear() cls._invocation_summary.clear() + + @classmethod + def ptc_format(cls, result): + """Post-tool-call formatting hook for orchestration sandbox exposure. + + By default, passes through the result unchanged. Tools can override + this to reshape the output that gets exposed to the orchestration + sandbox (e.g., stripping placeholder entries). + + Args: + result: The result from ``execute()`` (typically a ``ToolResponse`` + or string). + + Returns: + The (possibly modified) result. + """ + return result diff --git a/cecli/tools/utils/responses.py b/cecli/tools/utils/responses.py index e74897a8830..8b92252bfaa 100644 --- a/cecli/tools/utils/responses.py +++ b/cecli/tools/utils/responses.py @@ -4,15 +4,17 @@ class ToolResponse: """Assists in formatting all tool call responses as JSON for the LLM. - By default, every tool response is wrapped in JSON: + Every tool response is wrapped in JSON: { - "tool_name": str, - "result": str | list[str], - "errors": list[str], - "details": list[str] + "result": [{"content": ..., "_": {...}}], + "errors": [...], + "details": [...] } + Each result item has a ``content`` key (the primary output) and an + ``_`` key (metadata). Use ``append_result(content, metadata=None)`` + Usage inside a tool's ``execute`` method:: response = ToolResponse("my_tool", result_type="str") @@ -32,21 +34,29 @@ def __init__(self, tool_name, result_type="str"): self._errors = [] self._details = [] - def append_result(self, result): + def append_result(self, content, metadata=None): """Append a result entry. If ``result_type`` is ``"str"`` the text is concatenated (with a newline separator for subsequent calls). If ``result_type`` is ``"list"`` each call adds a new item to the results list. + + When metadata is provided (``result_type="list"`` only), it is + stored under the ``_`` key alongside ``content``. + + Backward compat: if ``content`` is a dict *without* a ``content`` + key it is treated as the content value and metadata defaults to + an empty dict. """ if self.result_type == "str": if self._result: - self._result += "\n" + str(result) + self._result += "\n" + str(content) else: - self._result = str(result) + self._result = str(content) else: - self._result.append(result) + item = {"content": content, "_": metadata or {}} + self._result.append(item) def append_error(self, error): """Collect an error message.""" @@ -61,9 +71,13 @@ def append_detail(self, detail): def to_dict(self): """Return the response as a plain Python dictionary.""" + if self.result_type == "str": + results = [{"content": self._result, "_": {}}] if self._result else [] + else: + results = self._result + return { - "tool_name": self.tool_name, - "result": self._result, + "result": results, "errors": self._errors, "details": self._details, } diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 5885daa6eb7..0ced95dfdfd 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -155,9 +155,31 @@ def test_security_filter_blocks_locals(): assert not _run_security_filter_safe("locals()"), "SecurityFilter should block 'locals'" -def test_security_filter_blocks_vars(): - """SecurityFilter blocks ``vars()``.""" - assert not _run_security_filter_safe("vars()"), "SecurityFilter should block 'vars'" +def test_security_filter_allows_vars(): + """SecurityFilter allows ``vars(obj)`` — safe at runtime via _safe_vars.""" + assert _run_security_filter_safe( + "vars(obj)" + ), "SecurityFilter should allow 'vars(obj)' as safe version is provided" + + +@pytest.mark.asyncio +async def test_env_vars_requires_one_arg(): + """vars() with no arguments raises TypeError (no local namespace leak).""" + env = _make_env() + result = await env.execute("vars()") + assert ( + "TypeError" in result["results"] + ), f"Expected TypeError for vars() no-args, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_vars_works_with_obj(): + """vars(obj) returns non-dunder attributes of obj.""" + env = _make_env() + result = await env.execute("def f(): pass\nf.x = 42\nprint(vars(f))") + assert ( + '"x": 42' in result["results"] or "'x': 42" in result["results"] + ), f"Expected 'x' attr in vars result, got: {result!r}" def test_security_filter_blocks_getattr(): @@ -359,7 +381,7 @@ async def test_env_rejects_import(): async def test_env_json_dumps(): """AgentExecutionEnv.execute() provides ``json.dumps`` in globals.""" env = _make_env() - result = await env.execute("json.dumps({'key': 'value'})") + result = await env.execute("print(json.dumps({'key': 'value'}))") assert result["results"] == '{"key": "value"}', f"Expected JSON output, got: {result!r}" @@ -407,7 +429,7 @@ async def test_env_rejects_global_stmt(): async def test_env_runs_simple_expression(): """AgentExecutionEnv.execute() runs a simple arithmetic expression.""" env = _make_env() - result = await env.execute("42") + result = await env.execute("print(42)") assert result["results"] == "42", f"Expected '42', got: {result!r}" @@ -424,10 +446,10 @@ async def test_env_state_persistence(): """AgentExecutionEnv.state persists across execute() calls.""" env = _make_env() - result1 = await env.execute("state['key'] = 'value1'\nstate['key']") + result1 = await env.execute("state['key'] = 'value1'\nprint(state['key'])") assert result1["results"] == "value1", f"Expected 'value1', got: {result1!r}" - result2 = await env.execute("state['key']") + result2 = await env.execute("print(state['key'])") assert result2["results"] == "value1", f"Expected 'value1' (persisted), got: {result2!r}" @@ -435,7 +457,7 @@ async def test_env_state_persistence(): async def test_env_runs_list_comprehension(): """AgentExecutionEnv.execute() runs a list comprehension.""" env = _make_env() - result = await env.execute("[i * 2 for i in range(5)]") + result = await env.execute("print([i * 2 for i in range(5)])") assert ( result["results"] == "[0, 2, 4, 6, 8]" ), f"Expected list comprehension result, got: {result!r}" @@ -443,17 +465,17 @@ async def test_env_runs_list_comprehension(): @pytest.mark.asyncio async def test_env_returns_last_expression(): - """AgentExecutionEnv returns the value of the last expression.""" + """AgentExecutionEnv only returns printed output — last expression values must be printed.""" env = _make_env() - result = await env.execute("x = 10\ny = 20\nx + y") + result = await env.execute("x = 10\ny = 20\nprint(x + y)") assert result["results"] == "30", f"Expected '30' from last expression, got: {result!r}" @pytest.mark.asyncio async def test_env_print_and_expression(): - """AgentExecutionEnv returns both print output and last expression.""" + """AgentExecutionEnv returns print output only.""" env = _make_env() - result = await env.execute("print('computed')\n42") + result = await env.execute("print('computed')\nprint(42)") assert "computed" in result["results"] assert "42" in result["results"] @@ -488,8 +510,10 @@ async def test_env_sleep_works(): async def test_env_gather_works(): """AgentExecutionEnv's gather primitive works.""" env = _make_env() - result = await env.execute("await gather()") - assert result["results"] == "[]", f"Expected '[]' from gather(), got: {result!r}" + result = await env.execute("print(await gather())") + assert ( + result["results"] == "GatherResult()" + ), f"Expected 'GatherResult()' from gather(), got: {result!r}" @pytest.mark.asyncio @@ -505,13 +529,13 @@ async def test_env_safe_builtins_available(): """AgentExecutionEnv provides safe builtins (len, range, int, str, etc.).""" env = _make_env() - result = await env.execute("len([1, 2, 3])") + result = await env.execute("print(len([1, 2, 3]))") assert result["results"] == "3", f"Expected '3', got: {result!r}" - result = await env.execute("str(42)") + result = await env.execute("print(str(42))") assert result["results"] == "42", f"Expected '42', got: {result!r}" - result = await env.execute("list(range(3))") + result = await env.execute("print(list(range(3)))") assert result["results"] == "[0, 1, 2]", f"Expected '[0, 1, 2]', got: {result!r}" @@ -532,19 +556,19 @@ async def test_env_runs_function_def(): result = await env.execute(""" def add(a, b): return a + b -add(2, 3) +print(add(2, 3)) """) assert result["results"] == "5", f"Expected '5' from function call, got: {result!r}" @pytest.mark.asyncio async def test_env_mixed_print_and_return(): - """AgentExecutionEnv returns print output followed by expression value.""" + """AgentExecutionEnv returns only printed output.""" env = _make_env() result = await env.execute(""" print("start") result = 99 -result +print(result) """) assert ( "start" in result["results"] and "99" in result["results"] @@ -657,9 +681,9 @@ async def test_tool_proxy_mcp_dispatch(): tool = proxy.get_tool("MockServer--MockTool") result = await tool.call(param1="value1") assert "result" in result - assert "mcp-result" in result["result"][0] - assert "MockTool" in result["result"][0] - assert "param1" in result["result"][0] + assert "mcp-result" in result["result"][0]["content"] + assert "MockTool" in result["result"][0]["content"] + assert "param1" in result["result"][0]["content"] def test_agent_proxy_includelist_filters(): @@ -716,3 +740,612 @@ def test_agent_proxy_local_prefix_tool(): tool = proxy.get_tool("Local--ReadFile") assert tool._tool_module is not None, "Local--ReadFile should resolve via ToolRegistry" assert tool._mcp_server is None + + +# =================================================================== +# AgentRegion +# =================================================================== + + +def test_agent_region_basic(tmp_path): + """AgentRegion stores patterns and resolves lazily via get_start/get_end.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + # Create a real file so lazy resolution works + source = "def foo():\n return 42\n" + test_file = os.path.join(str(tmp_path), "basic.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + regions = AgentRegion( + "basic.py", + coder, + [ + {"name": "foo", "start": "def foo", "end": "return 42"}, + ], + ) + + assert "foo" in regions + assert "missing" not in regions + assert len(regions) == 1 + assert "AgentRegion(1" in repr(regions) + + # Lazy resolution on access + start_id = regions.get_start("foo") + end_id = regions.get_end("foo") + + assert "::" in start_id + assert "::" in end_id + + start_line = regions.get_start_line("foo") + end_line = regions.get_end_line("foo") + + assert start_line > 0 + assert end_line > 0 + assert start_line <= end_line + + +# =================================================================== +# AgentProxy.resolve_regions +# =================================================================== + + +def _make_coder_with_io(tmp_path): + """Build a coder mock that supports resolve_paths + file I/O.""" + + import os + + class _MockIO: + def read_text(self, abs_path): + if os.path.isfile(abs_path): + with open(abs_path, "r") as f: + return f.read() + + return None + + class _MockCoder: + root = tmp_path + io = _MockIO() + registered_tools = {"included": set(), "excluded": set()} + mcp_tools = [] + + def abs_root_path(self, file_path): + import os + + return os.path.join(self.root, file_path) + + def get_rel_fname(self, abs_path): + import os + + return os.path.relpath(abs_path, self.root) + + return _MockCoder() + + +def test_resolve_regions_basic(tmp_path): + """AgentProxy.resolve_regions resolves text patterns to content IDs.""" + + import os + + from cecli.helpers.orchestration.environment import AgentProxy + + # Create a test file with known regions + source = """\ +def helper_one(): + \"\"\"Returns 'one'.\"\"\" + return "one" + + +def helper_two(): + \"\"\"Returns 'two'.\"\"\" + return "two" + + +def main(): + result = helper_one() + helper_two() + return result +""" + + test_file = os.path.join(str(tmp_path), "test_regions.py") + + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + proxy = AgentProxy(coder) + + regions = proxy.resolve_regions( + "test_regions.py", + [ + {"name": "helper_one", "start": "def helper_one", "end": 'return "one"'}, + {"name": "helper_two", "start": "def helper_two", "end": 'return "two"'}, + {"name": "main", "start": "def main", "end": "return result"}, + ], + ) + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + assert isinstance(regions, AgentRegion) + assert len(regions) == 3 + + # Each region should have valid content IDs + for name in ("helper_one", "helper_two", "main"): + start_id = regions.get_start(name) + end_id = regions.get_end(name) + + assert "::" in start_id, f"{name} start_id should be a content ID: {start_id}" + assert "::" in end_id, f"{name} end_id should be a content ID: {end_id}" + + assert regions.get_start_line(name) > 0 + assert regions.get_end_line(name) > 0 + assert regions.get_start_line(name) <= regions.get_end_line(name) + + +def test_resolve_regions_file_not_found(tmp_path): + """Lazy resolution raises ValueError when file doesn't exist.""" + + import pytest + + from cecli.helpers.orchestration.environment import AgentProxy + + coder = _make_coder_with_io(str(tmp_path)) + + proxy = AgentProxy(coder) + regions = proxy.resolve_regions( + "nonexistent.py", + [{"name": "x", "start": "def x", "end": "return"}], + ) + + # resolve_regions is now lazy — error surfaces on access + with pytest.raises(ValueError, match="File not found"): + regions.get_start("x") + + +def test_resolve_regions_empty_regions_list(tmp_path): + """AgentProxy.resolve_regions handles empty region list.""" + + import os + + from cecli.helpers.orchestration.environment import AgentProxy + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = "x = 1\n" + + test_file = os.path.join(str(tmp_path), "empty_test.py") + + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + proxy = AgentProxy(coder) + + regions = proxy.resolve_regions("empty_test.py", []) + + assert isinstance(regions, AgentRegion) + assert len(regions) == 0 + + +def test_agent_region_content_id_fallback(tmp_path): + """AgentRegion snapshots line content from content-ID patterns for stale-ID fallback.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def greet(name): + return f"hello {name}" + + +def farewell(name): + return f"goodbye {name}" +""" + + test_file = os.path.join(str(tmp_path), "fallback_test.py") + + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # First, resolve using the Agent to get a real content ID + from cecli.helpers.orchestration.environment import AgentProxy + + proxy = AgentProxy(coder) + regions = proxy.resolve_regions( + "fallback_test.py", + [{"name": "greet", "start": "def greet", "end": 'return f"hello {name}"'}], + ) + + # Get the content ID for the start of "greet" + start_id = regions.get_start("greet") + + # Now create a NEW AgentRegion that uses the raw content ID as the pattern. + # This simulates an agent that captured content IDs from a previous turn. + regions2 = AgentRegion( + "fallback_test.py", + coder, + [{"name": "greet", "start": start_id, "end": start_id}], + ) + + # First resolution should snapshot the line content + result_id = regions2.get_start("greet") + assert "::" in result_id + + # Now modify the file (simulating an edit that shifts hashlines) + new_source = """\ +# header comment added +def greet(name): + return f"hello {name}" + + +def farewell(name): + return f"goodbye {name}" +""" + + with open(test_file, "w") as f: + f.write(new_source) + + # The original content ID is now stale, but the snapshot should let + # us resolve via content matching + result_id2 = regions2.get_start("greet") + assert "::" in result_id2, f"Should resolve via fallback content match, got: {result_id2!r}" + + +def test_agent_region_rejects_ambiguous_pattern(tmp_path): + """AgentRegion raises ValueError when a text pattern matches multiple locations.""" + + import os + + import pytest + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + # Create a file where "return x" appears multiple times + source = """\ +def foo(): + return x + +def bar(): + return x + +def baz(): + return x +""" + + test_file = os.path.join(str(tmp_path), "ambiguous.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + regions = AgentRegion( + "ambiguous.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return x"}, + ], + ) + + # start is unique, but end matches 3 locations + with pytest.raises(ValueError, match="End pattern"): + regions.get_end("bar") + + +def test_agent_region_disambiguates_with_l_hint(tmp_path): + """AgentRegion uses @L hint to disambiguate when a pattern matches multiple locations.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def foo(): + return x + +def bar(): + return y + +def baz(): + return z +""" + + test_file = os.path.join(str(tmp_path), "hint_test.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # "return" appears on lines 2, 5, 8 — use @L hint to target bar's return + regions = AgentRegion( + "hint_test.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return @L6"}, + ], + ) + + start_id = regions.get_start("bar") + end_id = regions.get_end("bar") + + assert "::" in start_id + assert "::" in end_id + assert regions.get_start_line("bar") == 4 + assert regions.get_end_line("bar") == 5 + + +def test_agent_region_rejects_l_hint_still_ambiguous(tmp_path): + """AgentRegion raises ValueError when @L hint has equally-close matches (tie).""" + + import os + + import pytest + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +return x + +return y +middle +return x +return z +""" + + test_file = os.path.join(str(tmp_path), "bad_hint.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # "return x" at lines 1 and 4 (0-based: 0 and 3), hint @L3 (0-based 2) + # Both are distance 1 away from the hint + regions = AgentRegion( + "bad_hint.py", + coder, + [ + {"name": "top", "start": "return x @L1", "end": "return x @L3"}, + ], + ) + + # "return x" at lines 1 and 4 both distance 1 from @L3 (0-based 2) + with pytest.raises(ValueError, match="End pattern.*@L3 hint ties"): + regions.get_end("top") + + +def test_agent_region_explicit_line_hints(tmp_path): + """Explicit start_line_hint / end_line_hint fields disambiguate patterns.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def foo(): + return x + +def bar(): + return y + +def baz(): + return z +""" + + test_file = os.path.join(str(tmp_path), "explicit_hint.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # Use explicit end_line_hint instead of @L in pattern + regions = AgentRegion( + "explicit_hint.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return", "end_line_hint": 6}, + ], + ) + + start_id = regions.get_start("bar") + end_id = regions.get_end("bar") + + assert "::" in start_id + assert "::" in end_id + assert regions.get_start_line("bar") == 4 + assert regions.get_end_line("bar") == 5 + + +def test_agent_region_explicit_hint_overrides_at_syntax(tmp_path): + """Explicit line_hint fields override @L hints embedded in patterns.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def foo(): + return x + +def bar(): + return y + +def baz(): + return z +""" + + test_file = os.path.join(str(tmp_path), "override_hint.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # @L2 points to foo's return, but explicit end_line_hint=3 points to bar's + regions = AgentRegion( + "override_hint.py", + coder, + [ + { + "name": "bar", + "start": "def bar", + "end": "return @L3", + "end_line_hint": 6, + }, + ], + ) + + # Explicit hint (6→line 5) should win over @L3 (→line 2) + assert regions.get_end_line("bar") == 5 + + +def test_resolve_regions_rejects_ambiguous_via_proxy(tmp_path): + """resolve_regions() through AgentProxy surfaces ambiguity errors at access time.""" + + import os + + import pytest + + from cecli.helpers.orchestration.environment import AgentProxy + + source = """\ +def one(): + pass + +def two(): + pass + +def three(): + pass +""" + + test_file = os.path.join(str(tmp_path), "proxy_ambig.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + proxy = AgentProxy(coder) + + regions = proxy.resolve_regions( + "proxy_ambig.py", + [{"name": "two", "start": "def two", "end": "pass"}], + ) + + # "pass" appears 3 times — should raise at access time + with pytest.raises(ValueError, match="End pattern.*pass.*matches 3"): + regions.get_end("two") + + +def test_gather_result_attribute_access(): + """GatherResult supports attribute access for named results.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"read_a": {"result": ["hello"]}, "grep_b": {"result": ["world"]}}) + assert gr.read_a == {"result": ["hello"]} + assert gr.grep_b == {"result": ["world"]} + + +def test_gather_result_key_access(): + """GatherResult supports key/index access for named results.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"x": 1, "y": 2}) + assert gr["x"] == 1 + assert gr["y"] == 2 + + +def test_gather_result_len(): + """GatherResult supports len().""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 1, "b": 2, "c": 3}) + assert len(gr) == 3 + + +def test_gather_result_contains(): + """GatherResult supports 'in' operator.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 1}) + assert "a" in gr + assert "b" not in gr + + +def test_gather_result_iteration(): + """GatherResult is iterable (yields values).""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 10, "b": 20}) + values = list(gr) + assert ("a", 10) in values + assert ("b", 20) in values + assert len(values) == 2 + + +def test_gather_result_read_only(): + """GatherResult prevents mutation via attribute assignment.""" + import pytest + + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 1}) + with pytest.raises(AttributeError, match="read-only"): + gr.a = 2 + + +def test_gather_result_repr(): + """GatherResult repr shows types.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 42, "b": "hello"}) + r = repr(gr) + assert "GatherResult" in r + assert "int" in r or "str" in r + + +def test_gather_result_missing_key(): + """GatherResult raises AttributeError with helpful message for missing keys.""" + import pytest + + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"foo": 1, "bar": 2}) + with pytest.raises(AttributeError, match="no key"): + gr.baz + + +@pytest.mark.asyncio +async def test_env_named_gather(): + """AgentExecutionEnv supports named gather returning GatherResult.""" + env = _make_env() + result = await env.execute( + "results = await gather(a=tool.call(), b=tool.call())\n" + "print(results.a)\n" + "print(results['b'])\n" + "print(len(results))\n" + ) + # Just verify it doesn't crash and produces output + assert result["results"] # should have some output + + +@pytest.mark.asyncio +async def test_env_gather_mixed_rejected(): + """Mixing positional and keyword args in gather raises TypeError.""" + env = _make_env() + result = await env.execute("await gather(1, a=2)") + assert "TypeError" in result["results"] or "cannot mix" in result["results"].lower() + + +@pytest.mark.asyncio +async def test_env_named_gather_exception_handling(): + """Named gather converts exceptions to error dicts per-key.""" + env = _make_env() + result = await env.execute( + "results = await gather(good=tool.call(), bad=tool.call())\n" + "print('errors' in results.bad)\n" + ) + # Just verify it doesn't crash + assert result["results"] is not None diff --git a/tests/tools/test_grep.py b/tests/tools/test_grep.py index 04e6dd1b2fa..9030f883468 100644 --- a/tests/tools/test_grep.py +++ b/tests/tools/test_grep.py @@ -55,7 +55,8 @@ def test_dash_prefixed_pattern_is_searched_literally(search_term, tmp_path, monk operations = response_dict["result"] assert len(operations) == 1 op = operations[0] - assert op["pattern"] == search_term - assert op["error"] is None - assert "total_files" in op + assert op["_"]["pattern"] == search_term + assert op["_"]["error"] is None + assert "total_files" in op["_"] + assert isinstance(op["_"]["total_files"], int) coder.io.tool_error.assert_not_called() diff --git a/tests/tools/test_read_range_execute.py b/tests/tools/test_read_range_execute.py index 3f7ce011f35..461206c0063 100644 --- a/tests/tools/test_read_range_execute.py +++ b/tests/tools/test_read_range_execute.py @@ -171,7 +171,6 @@ def test_both_digits_valid_range( try: show = [{"file_path": self.test_file, "range_start": "5", "range_end": "10"}] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in str(result) assert "line5" in str(result) assert "line10" in str(result) finally: @@ -305,7 +304,6 @@ def test_both_text_patterns(self, mock_coder, mock_file_context, mock_chunks, mo } ] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in str(result) assert "def foo()" in str(result) assert "def bar()" in str(result) finally: @@ -335,7 +333,7 @@ def test_text_pattern_multiline(self, mock_coder, mock_file_context, mock_chunks try: show = [{"file_path": self.test_file, "range_start": "def foo", "range_end": "def bar"}] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in str(result) + assert "return 1" in str(result) finally: self._teardown() @@ -524,7 +522,7 @@ def func_f(): } ] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in str(result) + assert "def func_a" in str(result) finally: self._teardown() From 27428e15d85ace2eb5dedf464f54055045b463f8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 00:31:23 -0400 Subject: [PATCH 22/68] Resolve agent pain points --- cecli/helpers/orchestration/agent_proxy.py | 68 ++++++++- cecli/helpers/orchestration/environment.py | 135 ++++++++++++++---- .../helpers/orchestration/region_resolver.py | 15 +- cecli/helpers/orchestration/safe_methods.py | 11 +- cecli/helpers/orchestration/security.py | 1 - tests/helpers/test_orchestration.py | 6 +- 6 files changed, 196 insertions(+), 40 deletions(-) diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index 732d8b5d63a..b2092a7e8e9 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -52,8 +52,21 @@ def get_tool(self, tool_name: str) -> ToolProxy: tool_module = ToolRegistry.get_tool(bare_name) if tool_module is not None: return ToolProxy(tool_name, self._coder, tool_module=tool_module) + # Also try with hyphen normalization on the bare name + bare_underscored = bare_name.replace("-", "_") + if bare_underscored != bare_name: + tool_module = ToolRegistry.get_tool(bare_underscored) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 3b. Replace hyphens with underscores and retry (kebab-case -> snake_case) + underscored = name_lower.replace("-", "_") + if underscored != name_lower: + tool_module = ToolRegistry.get_tool(underscored) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) - # 3b. Strip underscores from the tool name and retry (snake_case fallback) + # 3c. Strip underscores from the tool name and retry (snake_case fallback) no_underscore = name_lower.replace("_", "") if no_underscore != name_lower: tool_module = ToolRegistry.get_tool(no_underscore) @@ -301,11 +314,64 @@ async def edit_region( } ) + self._validate_edit_regions(edits, edit_objects) + return await edit_tool.call( edits=edit_objects, change_id=change_id, ) + @staticmethod + def _validate_edit_regions( + edits: list[dict[str, object]], + edit_objects: list[dict[str, object]], + ) -> None: + """Reject batches where any two edit regions overlap or are adjacent. + + Adjacent edits (within 8 lines of each other) corrupt content IDs + because the first edit regenerates IDs in the surrounding ~8-line + window, making the second edit's target IDs stale. + """ + _ADJACENCY_THRESHOLD = 8 # lines of ID-regeneration buffer + + if len(edits) <= 1: + return + + # Collect (start_line, end_line, idx) for each edit + indexed: list[tuple[int, int, int]] = [] + for i, (edit, eo) in enumerate(zip(edits, edit_objects)): + region = edit["region"] + sl = region.get("start_line") + el = region.get("end_line") + if sl is None or el is None: + # No line numbers available — skip validation (best-effort) + return + indexed.append((int(sl), int(el), i)) + + # Sort by start_line + indexed.sort(key=lambda x: x[0]) + + for j in range(len(indexed) - 1): + sl_a, el_a, idx_a = indexed[j] + sl_b, el_b, idx_b = indexed[j + 1] + + if sl_b <= el_a: + raise ValueError( + f"Overlapping edit regions detected: " + f"edit {idx_a + 1} (lines {sl_a}-{el_a}) overlaps with " + f"edit {idx_b + 1} (lines {sl_b}-{el_b}). " + f"Split edits into separate calls." + ) + if sl_b <= el_a + _ADJACENCY_THRESHOLD: + raise ValueError( + f"Adjacent edit regions detected: " + f"edit {idx_a + 1} (lines {sl_a}-{el_a}) is within " + f"{_ADJACENCY_THRESHOLD} lines of edit {idx_b + 1} (lines {sl_b}-{el_b}). " + f"Content IDs within ~8 lines of an edit are regenerated, " + f"so adjacent edits corrupt each other's targets. " + f"Split edits into separate calls." + ) + # --------------------------------------------------------------------------- # Main execution environment diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 4572b000ea1..319b25d9e31 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -60,6 +60,40 @@ def _set_owner(self, owner: AgentExecutionEnv, is_shared: bool = False) -> None: self._owner = owner self._is_shared = is_shared + def __getattr__(self, key: str): + """Route attribute reads through dict lookup. + + Internal attrs (_owner, _is_shared) are accessed via object.__getattribute__ + to avoid recursion. All other names are looked up in the dict. + """ + if key.startswith("_"): + raise AttributeError(f"Cannot access private attribute {key!r} on TrackedDict") + try: + return self[key] + except KeyError: + raise AttributeError( + f"TrackedDict has no key {key!r}. " + f"Available keys: {', '.join(sorted(self.keys()))}" + ) from None + + def __setattr__(self, key: str, value) -> None: + """Route attribute writes through dict __setitem__. + + Internal attrs (_owner, _is_shared) are set directly on the instance + to avoid polluting dict state. + """ + if key.startswith("_"): + object.__setattr__(self, key, value) + else: + self[key] = value + + def __delattr__(self, key: str) -> None: + """Route attribute deletes through dict __delitem__.""" + if key.startswith("_"): + object.__delattr__(self, key) + else: + del self[key] + def __setitem__(self, key, value): is_new = key not in self super().__setitem__(key, value) @@ -99,7 +133,29 @@ def popitem(self): # --------------------------------------------------------------------------- # Safe primitives exposed to the sandbox -# --------------------------------------------------------------------------- + + +def _make_sandbox_dir(globals_dict, locals_dict): + """Create a safe dir() callable that handles both no-arg and obj modes. + + When called without arguments (like the builtin ``dir()``), returns a sorted + list of public (non-dunder) names from the sandbox globals, locals, and + builtins (including pre-imported modules like ``re``, ``math``, etc.). + When called with an object, delegates to ``_safe_dir(obj)``. + """ + _SENTINEL = object() + + def sandbox_dir(obj=_SENTINEL): + if obj is _SENTINEL: + names = set(globals_dict.keys()) | set(locals_dict.keys()) + builtins = globals_dict.get("__builtins__", {}) + if isinstance(builtins, dict): + names |= set(builtins.keys()) + return sorted(n for n in names if not n.startswith("_")) + + return _safe_dir(obj) + + return sandbox_dir class AgentExecutionEnv: @@ -129,6 +185,10 @@ def __init__(self, coder: Any) -> None: self.state._set_owner(self) self._shared_state._set_owner(self, is_shared=True) + # Initialize early so _safe_builtins can reference them + self.globals: dict[str, Any] = {} + self.locals: dict[str, Any] = {} + _safe_builtins: dict[str, Any] = { "print": print, "range": range, @@ -144,9 +204,10 @@ def __init__(self, coder: Any) -> None: "typeof": _safe_typeof, "type": _safe_typeof, "vars": _safe_vars, - "dir": _safe_dir, + "dir": _make_sandbox_dir(self.globals, self.locals), "isinstance": isinstance, "hasattr": hasattr, + "getattr": getattr, "repr": repr, "enumerate": enumerate, "zip": zip, @@ -188,19 +249,35 @@ def _allowed_methods(): ) return builtins + globals_list - self.globals: dict[str, Any] = { - "__builtins__": _HelpfulBuiltins(_safe_builtins), - "Agent": AgentProxy(coder), - "gather": _safe_gather, - "sleep": _safe_sleep, - "json": _SafeJson, - "state": self.state, - "shared_state": AgentExecutionEnv._shared_state, - "__yield": _cooperative_yield, - "NEWLINE": "\n", - "allowed_methods": _allowed_methods, - } - self.locals: dict[str, Any] = {} + def _allowed_tools(): + """Return a sorted list of available tool names for use with Agent.get_tool().""" + from cecli.helpers import nested + + tool_names = [] + tool_list = coder.get_tool_list() + for tool in tool_list: + name = nested.getter(tool, "function.name", "") + if name: + tool_names.append(name) + return sorted(tool_names) + + self.globals.clear() + self.globals.update( + { + "__builtins__": _HelpfulBuiltins(_safe_builtins), + "Agent": AgentProxy(coder), + "gather": _safe_gather, + "sleep": _safe_sleep, + "json": _SafeJson, + "state": self.state, + "shared_state": AgentExecutionEnv._shared_state, + "__yield": _cooperative_yield, + "NEWLINE": "\n", + "allowed_methods": _allowed_methods, + "allowed_tools": _allowed_tools, + } + ) + self.locals.clear() def _make_reset(env_locals, env_state): def reset_func(local_vars: bool = True, state: bool = False) -> None: @@ -351,33 +428,34 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: code = f"Compilation Error: {e}" return {"results": code, "state_variables": self._state_snapshot()} + def _build_result(msg: str = "") -> dict: + print_output = "".join(captured_output) + parts = [] + if print_output: + parts.append(print_output.rstrip("\n")) + if msg: + parts.append(msg) + code = "\n".join(parts) if parts else "" + return {"results": code, "state_variables": self._state_snapshot()} + try: exec(compiled_code, self.globals, self.locals) runner_coro = self.locals["__agent_async_runner"]() result = await runner_coro except asyncio.CancelledError: - code = "Execution Error: Script was cancelled." - return {"results": code, "state_variables": self._state_snapshot()} + return _build_result("Execution Error: Script was cancelled.") except SecurityError as e: - code = f"Security Error: {e}" - return {"results": code, "state_variables": self._state_snapshot()} + return _build_result(f"Security Error: {e}") except Exception as e: tb = traceback.format_exc() logger.warning("Orchestration execution error: %s\n%s", e, tb) - code = f"Execution Error: {type(e).__name__}: {e}" - return {"results": code, "state_variables": self._state_snapshot()} + return _build_result(f"Execution Error: {type(e).__name__}: {e}") finally: self.locals.pop("__agent_async_runner", None) self.globals["__builtins__"]["print"] = print # Mutation tracking is handled automatically by TrackedDict - print_output = "".join(captured_output) - - if print_output: - code = print_output.rstrip("\n") - return {"results": code, "state_variables": self._state_snapshot()} - - return {"results": "", "state_variables": self._state_snapshot()} + return _build_result() # flake8: noqa @@ -416,6 +494,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `print(...)` / `reset(local_vars=True, state=False)` | Output messages; clear local namespace (and optionally state) | | `typeof(x)` / `isinstance(x, t)` / `hasattr(x, n)` / `repr(x)` / `vars(obj)` | Type inspection and debugging | | `allowed_methods()` | List all available builtin function names | +| `allowed_tools()` | List all available tool names for use with ``Agent.get_tool()`` | ### Available Modules diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index 8b5720dc04a..e6caf57ef24 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -100,14 +100,21 @@ def names(self) -> list[str]: return sorted(self._specs.keys()) - def get(self, name: str) -> dict[str, str]: - """Return ``{"start": content_id, "end": content_id}`` for *name*. + def get(self, name: str) -> dict[str, object]: + """Return ``{"start": ..., "end": ..., "start_line": N, "end_line": N}`` for *name*. The returned dict can be passed directly as the ``region`` value - in ``Agent.edit_region()`` edits. + in ``Agent.edit_region()`` edits. ``start_line`` / ``end_line`` are + 1-based for readability and enable adjacent-edit detection. """ - return {"start": self.get_start(name), "end": self.get_end(name)} + start_id, end_id, start_line, end_line = self._resolve(name) + return { + "start": start_id, + "end": end_id, + "start_line": start_line + 1, + "end_line": end_line + 1, + } def __repr__(self) -> str: names = ", ".join(sorted(self._specs.keys())) diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index 2910c56490b..e2afcda3b99 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -78,7 +78,7 @@ def __len__(self) -> int: def __iter__(self): results = object.__getattribute__(self, "_results") - return iter(results.keys()) + return iter(results.items()) def keys(self) -> Any: results = object.__getattribute__(self, "_results") @@ -106,7 +106,7 @@ def __repr__(self) -> str: return f"GatherResult({inner})" -async def _safe_gather(**named_awaitables: Any): +async def _safe_gather(*args: Any, **named_awaitables: Any): """ Safely execute multiple awaitables concurrently. @@ -122,6 +122,12 @@ async def _safe_gather(**named_awaitables: Any): do not crash the entire batch. Exceptions are converted to structured error dicts. """ + if args: + raise TypeError( + "gather() requires keyword arguments. " + "Use named arguments like gather(a=task1, b=task2), " + "not positional arguments like gather(task1, task2)." + ) if not named_awaitables: return GatherResult({}) @@ -599,7 +605,6 @@ class _HelpfulBuiltins(dict): "globals": "globals() is disabled. Use state or shared_state for persistence.", "locals": "locals() is disabled. Use state or shared_state for persistence.", "vars": "Use vars(obj) instead of vars() — vars(obj) returns non-dunder attrs of obj.", - "getattr": "getattr() is disabled. Access attributes directly.", "setattr": "setattr() is disabled. Assign attributes directly.", "delattr": "delattr() is disabled. Use del obj.attr instead.", } diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py index 9067c522cbf..caadab2cc4c 100644 --- a/cecli/helpers/orchestration/security.py +++ b/cecli/helpers/orchestration/security.py @@ -34,7 +34,6 @@ class SecurityFilter(ast.NodeVisitor): "breakpoint", "globals", "locals", - "getattr", "setattr", "delattr", } diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 0ced95dfdfd..672943b7a7b 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -182,9 +182,9 @@ async def test_env_vars_works_with_obj(): ), f"Expected 'x' attr in vars result, got: {result!r}" -def test_security_filter_blocks_getattr(): - """SecurityFilter blocks ``getattr()``.""" - assert not _run_security_filter_safe("getattr(x, 'y')"), "SecurityFilter should block 'getattr'" +def test_security_filter_allows_getattr(): + """SecurityFilter allows ``getattr()``.""" + assert _run_security_filter_safe("getattr(x, 'y')"), "SecurityFilter should allow 'getattr'" def test_security_filter_blocks_setattr(): From 0e144abd75bea4980623ea4f6da644a09770a40c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 00:39:10 -0400 Subject: [PATCH 23/68] ExploreCode should return strings like other ToolResponses --- cecli/tools/explore_code.py | 170 ++++++++++++++++++------------------ 1 file changed, 86 insertions(+), 84 deletions(-) diff --git a/cecli/tools/explore_code.py b/cecli/tools/explore_code.py index 5a9df0ff5e3..81552db6b84 100644 --- a/cecli/tools/explore_code.py +++ b/cecli/tools/explore_code.py @@ -154,21 +154,15 @@ def execute(cls, coder, queries, **kwargs): if "multiple matches" in str(e).lower(): results = c.search(symbol_name, limit=10) response.append_result( - content={ - "action": "investigate", - "symbol": symbol, - "error": "Multiple matches found", - "hint": ( - "Please use a more specific name or filename:symbol format" - ), - "locations": [ - { - "file": r.get("rel_path") or r.get("file", ""), - "start_line": r.get("start_line", 0), - } + content=( + f"Error: Multiple matches found for '{symbol}'.\n" + f"Please use a more specific name or check the locations" + f" below:\n" + + "\n".join( + f"- {r.get('rel_path') or r.get('file', '')}:{r.get('start_line', 0)}" for r in results - ], - } + ) + ) ) else: raise e @@ -249,106 +243,114 @@ def _filter_investigation_gitignored(cls, investigation, coder): @classmethod def _format_search_results(cls, results, symbol): - """Format search results as structured data.""" - + """Format search results for display.""" if not results: - return {"action": "search", "symbol": symbol, "count": 0, "results": []} - - return { - "action": "search", - "symbol": symbol, - "count": len(results), - "results": [ - { - "name": r.get("name", ""), - "kind": r.get("kind", ""), - "file": r.get("rel_path") or r.get("file", ""), - "start_line": r.get("start_line", 0), - "signature": r.get("signature", ""), - "parent": r.get("parent"), - } - for r in results - ], - } + return f"No symbols found matching '{symbol}'" + + formatted = [f"Found {len(results)} symbols matching '{symbol}':"] + for i, result in enumerate(results[:15], 1): + name = result.get("name", "Unknown") + kind = result.get("kind", "unknown") + file = result.get("rel_path") or result.get("file", "Unknown") + start_line = result.get("start_line", 0) + signature = result.get("signature", "") + parent = result.get("parent") + + location = f"{file}:{start_line}" + if parent: + location = f"{location} (in {parent})" + + formatted.append(f"{i}. {name}{signature} ({kind}) at {location}") + + if len(results) > 15: + formatted.append(f"... and {len(results) - 15} more results") + + return "\n".join(formatted) @classmethod def _format_investigation_results(cls, investigation, symbol): - """Format investigation results as structured data.""" - + """Format investigation results for display.""" if not investigation: - return {"action": "investigate", "symbol": symbol, "error": "No information found"} + return f"No information found for symbol '{symbol}'" # Handle nested structure if present if "results" in investigation and "result" in investigation["results"]: investigation = investigation["results"]["result"] - result = { - "action": "investigate", - "symbol": symbol, - } + formatted = [f"Investigation of symbol '{symbol}':"] # Extract definition information definition = investigation.get("symbol") if definition: - result["definition"] = { - "name": definition.get("name", symbol), - "file": definition.get("rel_path") or definition.get("file", ""), - "line": definition.get("start_line", 0), - "kind": definition.get("kind", ""), - "signature": definition.get("signature", ""), - } + def_name = definition.get("name", symbol) + def_file = definition.get("rel_path") or definition.get("file", "Unknown") + def_line = definition.get("start_line", 0) + def_kind = definition.get("kind", "unknown") + def_sig = definition.get("signature", "") + formatted.append( + f"Definition: {def_name}{def_sig} ({def_kind}) at {def_file}:{def_line}" + ) # Source code snippet source = investigation.get("source") if source: - result["source"] = source.strip() + formatted.append("\nSource Code:") + formatted.append("```python") + formatted.append(source.strip()) + formatted.append("```") # References references = investigation.get("refs", []) - result["ref_count"] = len(references) if references else 0 - if references: - result["references"] = [ - { - "file": ref.get("rel_path") or ref.get("file", ""), - "line": ref.get("line", 0), - } - for ref in references - ] + ref_count = len(references) if references else 0 + formatted.append(f"\nReferences found: {ref_count}") + + if references and ref_count > 0: + formatted.append("Top references:") + for i, ref in enumerate(references[:10], 1): + ref_file = ref.get("rel_path") or ref.get("file", "Unknown") + ref_line = ref.get("line", 0) + formatted.append(f"{i}. {ref_file}:{ref_line}") + + if ref_count > 10: + formatted.append(f"... and {ref_count - 10} more references") # Impact / Callers impact = investigation.get("impact", []) if impact: - result["impact"] = [ - { - "caller": imp.get("caller", ""), - "file": imp.get("rel_path") or imp.get("file", ""), - "line": imp.get("line", 0), - } - for imp in impact - ] + formatted.append("\nImpact (Callers):") + for i, imp in enumerate(impact[:10], 1): + imp_file = imp.get("rel_path") or imp.get("file", "Unknown") + imp_line = imp.get("line", 0) + imp_caller = imp.get("caller", "unknown") + formatted.append(f"{i}. {imp_caller} at {imp_file}:{imp_line}") - return result + if len(impact) > 10: + formatted.append(f"... and {len(impact) - 10} more callers") + + return "\n".join(formatted) @classmethod def _format_reference_results(cls, references, symbol): - """Format reference finding results as structured data.""" - + """Format reference finding results for display.""" if not references: - return {"action": "find_references", "symbol": symbol, "count": 0, "results": []} - - return { - "action": "find_references", - "symbol": symbol, - "count": len(references), - "results": [ - { - "file": ref.get("rel_path") or ref.get("file", ""), - "line": ref.get("line", 0), - "context": ref.get("context", []), - } - for ref in references - ], - } + return f"No references found for symbol '{symbol}'" + + formatted = [f"Found {len(references)} references to '{symbol}':"] + for i, ref in enumerate(references[:15], 1): + file = ref.get("rel_path") or ref.get("file", "Unknown") + line = ref.get("line", 0) + context = ref.get("context", []) + + formatted.append(f"{i}. {file}:{line}") + if context: + formatted.append(" Context:") + for line_text in context: + formatted.append(f" {line_text.strip()}") + + if len(references) > 15: + formatted.append(f"... and {len(references) - 15} more references") + + return "\n".join(formatted) @classmethod def format_output(cls, coder, mcp_server, tool_response): From 1aab5fc95386d600544ad4af11c0d9637c537d91 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 09:27:07 -0400 Subject: [PATCH 24/68] Allow delegation to run inline for orchestration --- cecli/tools/delegate.py | 105 ++++++++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/cecli/tools/delegate.py b/cecli/tools/delegate.py index be770578daf..80ede54cb79 100644 --- a/cecli/tools/delegate.py +++ b/cecli/tools/delegate.py @@ -41,6 +41,14 @@ class Tool(BaseTool): "type": "string", "description": "Task description to give the sub-agent.", }, + "async": { + "type": "boolean", + "default": True, + "description": ( + "If true (default), delegate asynchronously (fire-and-forget)." + " If false, wait for the result." + ), + }, }, "required": ["name", "prompt"], }, @@ -55,12 +63,13 @@ class Tool(BaseTool): @classmethod async def execute(cls, coder, **kwargs): """Delegate one or more sub-agents to work on sub-tasks in parallel.""" + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) delegations = kwargs.get("delegations", []) if not delegations or not isinstance(delegations, list): response.append_error( - "'delegations' parameter must be a non-empty array of {name, prompt} objects." + "'delegations' parameter must be a non-empty array of {name, prompt, async} objects." ) return response @@ -80,33 +89,77 @@ async def execute(cls, coder, **kwargs): agent_service = AgentService.get_instance(coder) - async def _spawn_one(name: str, prompt: str) -> tuple[str, str]: - """Spawn a single sub-agent and return (name, uuid_or_error).""" + # Separate async (fire-and-forget) and sync (blocking) delegations + async_delegations = [(d["name"], d["prompt"]) for d in delegations if d.get("async", True)] + sync_delegations = [ + (d["name"], d["prompt"]) for d in delegations if not d.get("async", True) + ] + + async def _spawn_one(name: str, prompt: str) -> tuple: + """Spawn a single sub-agent (fire-and-forget). Returns (name, uuid_or_error, error).""" try: new_coder, info = await agent_service.spawn(name, prompt, parent=coder) - return name, info.coder.uuid + return name, info.coder.uuid, None + except Exception as e: + return name, None, f"failed: {e}" + + async def _invoke_one(name: str, prompt: str) -> tuple: + """Invoke a single sub-agent (blocking). Returns (name, summary_or_error, error).""" + try: + summary = await agent_service.invoke(name, prompt, parent=coder) + return name, summary or "(no summary)", None except Exception as e: - return name, f"failed: {e}" - - # Dispatch all delegations in parallel (spawn is fire-and-forget, but - # _create_sub_agent_coder is async so we gather for concurrency) - tasks = [_spawn_one(d["name"], d["prompt"]) for d in delegations] - raw_results = await asyncio.gather(*tasks) - - started_agents: list[tuple[str, str]] = list(raw_results) - - # Build a consolidated report - lines = [] - for name, result in started_agents: - if result.startswith("failed:"): - lines.append(f"✗ **{name}**: {result}") - else: - lines.append(f"✓ **{name}** agent started with id `{result}`") - - n_total = len(started_agents) - n_ok = sum(1 for _, r in started_agents if not r.startswith("failed:")) - combined = "\n".join(lines) - response.append_result(f"📋 Delegation results ({n_ok}/{n_total} dispatched):\n{combined}") + return name, None, f"failed: {e}" + + # Process async delegations (fire-and-forget spawn) + async_results = [] + if async_delegations: + tasks = [_spawn_one(n, p) for n, p in async_delegations] + async_results = list(await asyncio.gather(*tasks)) + + # Process sync delegations (blocking invoke) + sync_results = [] + if sync_delegations: + tasks = [_invoke_one(n, p) for n, p in sync_delegations] + sync_results = list(await asyncio.gather(*tasks)) + + # Build response + if not sync_delegations: + # All async: single combined result (current behavior) + lines = [] + for name, result, error in async_results: + if error: + lines.append(f"✗ **{name}**: {error}") + else: + lines.append(f"✓ **{name}** agent started with id `{result}`") + + n_total = len(async_results) + n_ok = sum(1 for _, _, e in async_results if not e) + combined = "\n".join(lines) + response.append_result( + f"📋 Delegation results ({n_ok}/{n_total} dispatched):\n{combined}" + ) + else: + # Mixed or all sync: individual results per non-async agent + if async_delegations: + lines = [] + for name, result, error in async_results: + if error: + lines.append(f"✗ **{name}**: {error}") + else: + lines.append(f"✓ **{name}** agent started with id `{result}`") + combined = "\n".join(lines) + n_ok = sum(1 for _, _, e in async_results if not e) + response.append_result( + f"📋 Async delegation results ({n_ok}/{len(async_results)} dispatched):\n{combined}" + ) + + for name, summary, error in sync_results: + if error: + response.append_result(f"✗ **{name}** agent failed: {error}") + else: + response.append_result(f"✓ **{name}** agent completed:\n{summary}") + return response @classmethod @@ -134,6 +187,8 @@ def format_output(cls, coder, mcp_server, tool_response): coder.io.tool_output(f"{color_start}delegation_{i + 1}:{color_end}") coder.io.tool_output(f"agent: {name}") coder.io.tool_output(f"task: {prompt}") + is_async = d.get("async", True) + coder.io.tool_output(f"mode: {'async' if is_async else 'sync'}") if i < len(delegations) - 1: coder.io.tool_output("") From 00d7cb0c86b2f6d23bb7b767e96f7bc7178b5c29 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 09:41:06 -0400 Subject: [PATCH 25/68] Updates per agent feedback on ease of use --- cecli/helpers/orchestration/environment.py | 4 +- tests/helpers/test_orchestration.py | 72 ++++++++++------------ 2 files changed, 36 insertions(+), 40 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 319b25d9e31..52af524c103 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -540,7 +540,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non await Agent.edit_region( file_path="foo.py", edits=[ - {"region": regions.get("my_func"), "text": "def my_func():\\n return 42"}, + {"region": regions.get("my_func"), "text": "def my_func():\n return 42"}, ], ) ``` @@ -554,7 +554,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non "operation": "replace", "start_line": regions.get_start("my_func"), "end_line": regions.get_end("my_func"), - "text": "def my_func():\\n return 42", + "text": "def my_func():\n return 42", }]) ``` diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 672943b7a7b..4acf3d611da 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -887,7 +887,7 @@ def main(): def test_resolve_regions_file_not_found(tmp_path): - """Lazy resolution raises ValueError when file doesn't exist.""" + """Eager validation raises ValueError at resolve_regions time when file doesn't exist.""" import pytest @@ -896,14 +896,13 @@ def test_resolve_regions_file_not_found(tmp_path): coder = _make_coder_with_io(str(tmp_path)) proxy = AgentProxy(coder) - regions = proxy.resolve_regions( - "nonexistent.py", - [{"name": "x", "start": "def x", "end": "return"}], - ) - # resolve_regions is now lazy — error surfaces on access + # Eager validation surfaces errors at construction time with pytest.raises(ValueError, match="File not found"): - regions.get_start("x") + proxy.resolve_regions( + "nonexistent.py", + [{"name": "x", "start": "def x", "end": "return"}], + ) def test_resolve_regions_empty_regions_list(tmp_path): @@ -998,7 +997,7 @@ def farewell(name): def test_agent_region_rejects_ambiguous_pattern(tmp_path): - """AgentRegion raises ValueError when a text pattern matches multiple locations.""" + """AgentRegion raises ValueError eagerly when a text pattern matches multiple locations.""" import os @@ -1011,9 +1010,11 @@ def test_agent_region_rejects_ambiguous_pattern(tmp_path): def foo(): return x + def bar(): return x + def baz(): return x """ @@ -1024,17 +1025,15 @@ def baz(): coder = _make_coder_with_io(str(tmp_path)) - regions = AgentRegion( - "ambiguous.py", - coder, - [ - {"name": "bar", "start": "def bar", "end": "return x"}, - ], - ) - - # start is unique, but end matches 3 locations + # start is unique, but end matches 3 locations — error surfaces eagerly with pytest.raises(ValueError, match="End pattern"): - regions.get_end("bar") + AgentRegion( + "ambiguous.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return x"}, + ], + ) def test_agent_region_disambiguates_with_l_hint(tmp_path): @@ -1080,7 +1079,7 @@ def baz(): def test_agent_region_rejects_l_hint_still_ambiguous(tmp_path): - """AgentRegion raises ValueError when @L hint has equally-close matches (tie).""" + """AgentRegion raises ValueError eagerly when @L hint has equally-close matches (tie).""" import os @@ -1104,18 +1103,15 @@ def test_agent_region_rejects_l_hint_still_ambiguous(tmp_path): coder = _make_coder_with_io(str(tmp_path)) # "return x" at lines 1 and 4 (0-based: 0 and 3), hint @L3 (0-based 2) - # Both are distance 1 away from the hint - regions = AgentRegion( - "bad_hint.py", - coder, - [ - {"name": "top", "start": "return x @L1", "end": "return x @L3"}, - ], - ) - - # "return x" at lines 1 and 4 both distance 1 from @L3 (0-based 2) + # Both are distance 1 away from the hint — error surfaces eagerly with pytest.raises(ValueError, match="End pattern.*@L3 hint ties"): - regions.get_end("top") + AgentRegion( + "bad_hint.py", + coder, + [ + {"name": "top", "start": "return x @L1", "end": "return x @L3"}, + ], + ) def test_agent_region_explicit_line_hints(tmp_path): @@ -1203,7 +1199,7 @@ def baz(): def test_resolve_regions_rejects_ambiguous_via_proxy(tmp_path): - """resolve_regions() through AgentProxy surfaces ambiguity errors at access time.""" + """resolve_regions() through AgentProxy surfaces ambiguity errors eagerly at construction time.""" import os @@ -1215,9 +1211,11 @@ def test_resolve_regions_rejects_ambiguous_via_proxy(tmp_path): def one(): pass + def two(): pass + def three(): pass """ @@ -1229,14 +1227,12 @@ def three(): coder = _make_coder_with_io(str(tmp_path)) proxy = AgentProxy(coder) - regions = proxy.resolve_regions( - "proxy_ambig.py", - [{"name": "two", "start": "def two", "end": "pass"}], - ) - - # "pass" appears 3 times — should raise at access time + # "pass" appears 3 times — error surfaces eagerly at resolve_regions time with pytest.raises(ValueError, match="End pattern.*pass.*matches 3"): - regions.get_end("two") + proxy.resolve_regions( + "proxy_ambig.py", + [{"name": "two", "start": "def two", "end": "pass"}], + ) def test_gather_result_attribute_access(): From d7b257eb903a858cb73fd40d96a62a7065b26929 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 10:02:04 -0400 Subject: [PATCH 26/68] Update requirements for later versions of ltiellm --- requirements.txt | 4 ++++ requirements/common-constraints.txt | 2 ++ requirements/requirements.in | 3 +++ 3 files changed, 9 insertions(+) diff --git a/requirements.txt b/requirements.txt index 80fce896211..7fcb2b594dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -240,6 +240,10 @@ openai==2.8.1 # via # -c requirements/common-constraints.txt # litellm +orjson==3.11.9 + # via + # -c requirements/common-constraints.txt + # -r requirements/requirements.in oslex==0.1.3 # via # -c requirements/common-constraints.txt diff --git a/requirements/common-constraints.txt b/requirements/common-constraints.txt index ffc8da5939f..54cd4ebd296 100644 --- a/requirements/common-constraints.txt +++ b/requirements/common-constraints.txt @@ -302,6 +302,8 @@ objgraph==3.6.2 # via -r requirements/requirements-dev.in openai==2.8.1 # via litellm +orjson==3.11.9 + # via -r requirements/requirements.in oslex==0.1.3 # via -r requirements/requirements.in packaging==25.0 diff --git a/requirements/requirements.in b/requirements/requirements.in index a6575c2168f..40e50fba133 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -46,6 +46,9 @@ rustworkx>=0.15.0 # scipy is still needed for other parts of the codebase scipy>=1.15.3 +# dependencies added because litellm is so mid and doesn't declare all of its deps properly +orjson>=3.11.6 + # GitHub Release action failing on "KeyError: 'home-page'" # https://github.com/pypa/twine/blob/6fbf880ee60915cf1666348c4bdd78a10415f2ac/twine/__init__.py#L40 # Uses importlib-metadata From afc2cce9880f05682a3258a4dc6be1c48e222e78 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 10:03:43 -0400 Subject: [PATCH 27/68] Change get_shape() to peek --- cecli/helpers/orchestration/agent_proxy.py | 32 ++++++++++++++++------ cecli/helpers/orchestration/environment.py | 2 +- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index b2092a7e8e9..527c82a5f0c 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -102,7 +102,7 @@ def _find_mcp_server(self, server_name: str, server_prefix: str) -> Any: return server return None - def get_shape(self, result: Any) -> str: + def peek(self, result: Any) -> str: """Inspect the structure of a tool result and return a readable summary. Tool results have a standard shape:: @@ -114,25 +114,40 @@ def get_shape(self, result: Any) -> str: } This method unwraps the structure to show what keys are available, - helping the LLM navigate deeply nested tool outputs. + helping the LLM navigate deeply nested tool outputs. Leaf values + include a short content snippet so you can see actual data. Example:: output = await grep_tool.call(pattern="TODO", file_glob="*.py") - print(Agent.get_shape(output)) - # Shows: result[0].content, result[0]._.file, result[0]._.match_count, ... + print(Agent.peek(output)) + # Shows: result[0].content: str = 'def foo():...' + # result[0]._.file_path: str = 'src/main.py' # Now the LLM can confidently access: for item in output["result"]: - print(item["content"]) # or item["_"]["file"] + print(item["content"]) # or item["_"]["file_path"] - Returns a multi-line string describing the available access paths. + Returns a multi-line string describing the available access paths + with short content previews for leaf values. """ + return self._inspect_structure(result) + @staticmethod + def _content_preview(value: Any, max_chars: int = 20) -> str: + """Return a short preview of a scalar value's stringified content.""" + + s = str(value) + if len(s) <= max_chars: + return repr(s) + + return repr(s[:max_chars]) + "..." + @staticmethod def _inspect_structure(obj: Any, prefix: str = "", depth: int = 0) -> str: - """Recursively inspect the structure of a tool result — paths and types only.""" + """Recursively inspect the structure of a tool result — paths, types, and content previews.""" + max_depth = 3 max_keys = 5 lines: list[str] = [] @@ -164,7 +179,8 @@ def _inspect_structure(obj: Any, prefix: str = "", depth: int = 0) -> str: else: lines.append(f"{path}: list (empty)") else: - lines.append(f"{path}: {type(value).__name__}") + preview = AgentProxy._content_preview(value) + lines.append(f"{path}: {type(value).__name__} = {preview}") elif isinstance(obj, list): if obj: diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 52af524c103..f1aeb1a2e22 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -480,7 +480,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non |-----------|-------------| | `Agent.get_tool(name)` | Get a tool proxy (case-insensitive, accepts `Local--` or `Server--` prefix) | | `await tool.call(**params)` | Execute a tool; returns `{"result": [...], "errors": [...], "details": [...]}` — each result item is `{"content": ..., "_": {...}}` | -| `Agent.get_shape(result)` | Inspect a tool result's structure — returns a string; use `print(Agent.get_shape(result))` to see it | +| `Agent.peek(result)` | Inspect a tool result's structure and leaf content — returns a string; use `print(Agent.peek(result))` to see it | | `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | | `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | From 5038ebd0f283df623c65f2bb090e4e2cc4fd2154 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 10:26:05 -0400 Subject: [PATCH 28/68] Don't restrict based on adjacency, add notes on state persistence --- cecli/helpers/orchestration/agent_proxy.py | 2 +- cecli/helpers/orchestration/environment.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index 527c82a5f0c..73a88186506 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -330,7 +330,7 @@ async def edit_region( } ) - self._validate_edit_regions(edits, edit_objects) + # self._validate_edit_regions(edits, edit_objects) return await edit_tool.call( edits=edit_objects, diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index f1aeb1a2e22..1adb7ac6f53 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -473,6 +473,8 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non return """ The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. This is much more efficient than making individual tool calls for loop-heavy workflows. +Variables and methods defined in a script are persisted in subsequent turns. +As such, results from previous calls can be reused and helper methods can be defined to enhance usage of the environment. ### Primitives From f5a4b7d09c279d01be7faa932cdc6eb880fbc77b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 14:13:08 -0400 Subject: [PATCH 29/68] Genericize _stringify and exempt orchestrate from invocation tracking --- cecli/helpers/orchestration/tool_proxy.py | 2 +- cecli/tools/utils/base_tool.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cecli/helpers/orchestration/tool_proxy.py b/cecli/helpers/orchestration/tool_proxy.py index f38bf44bf82..10743f0ebcb 100644 --- a/cecli/helpers/orchestration/tool_proxy.py +++ b/cecli/helpers/orchestration/tool_proxy.py @@ -115,7 +115,7 @@ async def call(self, **kwargs: Any): """ if self._tool_module is not None: - result = self._tool_module.process_response(self._coder, kwargs, _stringify=False) + result = self._tool_module.process_response(self._coder, kwargs, _convert=False) if asyncio.iscoroutine(result): result = await result result = self._tool_module.ptc_format(result) diff --git a/cecli/tools/utils/base_tool.py b/cecli/tools/utils/base_tool.py index bbe4ad75a91..b749c148ad6 100644 --- a/cecli/tools/utils/base_tool.py +++ b/cecli/tools/utils/base_tool.py @@ -41,18 +41,18 @@ def execute(cls, coder, **params): pass @classmethod - def process_response(cls, coder, params, _stringify=True): + def process_response(cls, coder, params, _convert=True): """ Process the tool response by creating an instance and calling execute. Args: coder: The Coder instance params: Dictionary of parameters - _stringify: If True (default), ToolResponse results are converted to str. + _convert: If True (default), ToolResponse results are converted to str. If False, ToolResponse is returned as-is for sandbox use. Returns: - str or ToolResponse: Result message (or ToolResponse when _stringify=False) + str or ToolResponse: Result message (or ToolResponse when _convert=False) """ # Validate required parameters from SCHEMA @@ -90,7 +90,7 @@ def process_response(cls, coder, params, _stringify=True): return handle_tool_error(coder, tool_name, ValueError(error_msg)) # Check for repeated invocations if TRACK_INVOCATIONS is enabled - if cls.TRACK_INVOCATIONS: + if cls.TRACK_INVOCATIONS and _convert: tool_name = None if cls.SCHEMA and "function" in cls.SCHEMA: tool_name = cls.SCHEMA["function"].get("name", "Unknown Tool") @@ -142,7 +142,7 @@ def process_response(cls, coder, params, _stringify=True): result = cls.execute(coder, **params) if isinstance(result, ToolResponse): - return result if not _stringify else str(result) + return result if not _convert else str(result) return result except Exception as e: From abaa961147f53c18e5a9ceca913f2b11156584d6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 15:11:06 -0400 Subject: [PATCH 30/68] Return session for http based servers --- cecli/mcp/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cecli/mcp/server.py b/cecli/mcp/server.py index bb92c1473dd..a6bc691b9c3 100644 --- a/cecli/mcp/server.py +++ b/cecli/mcp/server.py @@ -272,6 +272,8 @@ async def connect(self): server_info["client_info"]["token_endpoint"] = token_endpoint save_mcp_oauth_token(self.name, server_info) + + return session except Exception as e: logging.error(f"Error initializing {self.name}: {e}") await self.disconnect() From bdd3ad77413ef2aa30881e81a50c21ac7b4de6f3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 22:44:21 -0400 Subject: [PATCH 31/68] Update hashpos to be more conservative on when it identifies a line with a full hash --- cecli/helpers/agents/service.py | 2 +- cecli/helpers/hashline.py | 123 ++++++++++- cecli/helpers/hashpos/hashpos.py | 193 +++++++++--------- cecli/helpers/orchestration/agent_proxy.py | 31 ++- .../helpers/orchestration/region_resolver.py | 4 +- cecli/helpers/orchestration/safe_methods.py | 2 +- cecli/hooks/helpers.py | 2 +- cecli/prompts/agent.yml | 28 ++- cecli/prompts/subagent.yml | 28 ++- cecli/tools/edit_file.py | 32 +-- cecli/tools/read_file.py | 44 ++-- cecli/tools/utils/responses.py | 2 +- cecli/tools/validations/validations.py | 2 +- 13 files changed, 302 insertions(+), 191 deletions(-) diff --git a/cecli/helpers/agents/service.py b/cecli/helpers/agents/service.py index ed7a517b192..f229573a156 100644 --- a/cecli/helpers/agents/service.py +++ b/cecli/helpers/agents/service.py @@ -575,7 +575,7 @@ def start_generate_task(self, info: SubAgentInfo, user_message: str) -> asyncio. Args: - .. note:: + .. note: **Ordering dependency with mark_sub_agent_finished()** diff --git a/cecli/helpers/hashline.py b/cecli/helpers/hashline.py index 4c980ec5c67..55d0815b98b 100644 --- a/cecli/helpers/hashline.py +++ b/cecli/helpers/hashline.py @@ -3,7 +3,11 @@ import re from collections import Counter -from cecli.helpers.hashpos.hashpos import HashPos +from cecli.helpers.hashpos.hashpos import ( # noqa + HASH_DELIMITER, + UNIQUE_HASH_DELIMITER, + HashPos, +) HASHLINE_PREFIX_RE = HashPos.HASH_PREFIX_RE @@ -72,14 +76,14 @@ def strip_hashline(text: str) -> str: return HashPos.strip_prefix(text) -def normalize_hashline(hashline_str: str) -> str: +def normalize_hashline(hashline_str: str, throw=True) -> str: """ Normalize a hashline string to the content id hash fragment. """ if hashline_str in ("@000", "000@"): return hashline_str try: - return HashPos.normalize(hashpos_str=hashline_str) + return HashPos.normalize(hashpos_str=hashline_str, throw=throw) except ValueError as e: raise ContentHashError(str(e)) @@ -331,8 +335,11 @@ def _find_multiline_match(lines, value): def _resolve_to_hash_id(lines, idx, hp): """Generate a hash ID for the line at the given index.""" - hash_id = hp.generate_public_id(lines[idx], idx) - return hash_id + "::" + line_text = lines[idx] + # Compute the occurrence number (1-based) of this line instance + occurrence = 1 + sum(1 for i in range(idx) if lines[i] == line_text) + hash_id = hp.generate_public_id(line_text, idx, occurrence) + return HashPos.get_wrapped_id(hash_id) lines = original_content.splitlines() hp = HashPos(original_content) @@ -363,7 +370,15 @@ def _resolve_to_hash_id(lines, idx, hp): if candidates: resolved_start_idx = candidates[0] except (ContentHashError, ValueError): - pass + # The value looks like a content ID but normalized to UNIQUE_HASH_DELIMITER which can't + # be spatially resolved. Fall back: strip the UNIQUE_HASH_DELIMITER prefix and treat the + # remaining content as line content to match. + content = HashPos.strip_prefix(start_value) + if content != start_value and content.strip(): + containing_indices = _find_substring_matches(lines, content) + if len(containing_indices) == 1: + resolved_start_idx = containing_indices[0] + resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) # Resolve end_value based on proximity to start position resolved_end = end_value @@ -389,6 +404,30 @@ def _resolve_to_hash_id(lines, idx, hp): key=lambda idx: abs(idx - resolved_start_idx), ) resolved_end = _resolve_to_hash_id(lines, closest_idx, hp) + elif end_value is not None and _looks_like_content_id(end_value): + # Already a content ID - try to resolve it + try: + normalized = normalize_hashline(end_value) + candidates = hp.resolve_to_lines(normalized) + if candidates and resolved_start_idx is not None: + # Pick candidate closest to start position + resolved_end = _resolve_to_hash_id(lines, candidates[0], hp) + except (ContentHashError, ValueError): + # The value looks like a content ID but normalized to UNIQUE_HASH_DELIMITER which can't + # be spatially resolved. Fall back: strip the UNIQUE_HASH_DELIMITER prefix and treat the + # remaining content as line content to match. + content = HashPos.strip_prefix(end_value) + if content != end_value and content.strip(): + containing_indices = _find_substring_matches(lines, content) + if len(containing_indices) == 1: + idx = containing_indices[0] + resolved_end = _resolve_to_hash_id(lines, idx, hp) + elif len(containing_indices) > 1 and resolved_start_idx is not None: + closest_idx = min( + containing_indices, + key=lambda idx: abs(idx - resolved_start_idx), + ) + resolved_end = _resolve_to_hash_id(lines, closest_idx, hp) return resolved_start, resolved_end @@ -1705,6 +1744,52 @@ def _has_indent_jump(test_lines: list, unit: int) -> bool: return safe_resolved_ops, rejected_ops +def _try_resolve_as_unique_line( + hp: HashPos, + value: str, +) -> str | None: + """ + Try to resolve a value by matching it against unique lines in the source content. + + If the value (after stripping) matches exactly one line in the source + AND that line appears only once (unique), returns a tilde-wrapped hash ID + that can be resolved to line indices by the HashPos engine. + + This serves as a preliminary resolution step that can salvage operations + whose hash references could not be normalized, by checking if the value + corresponds to unique line content in the file. + + Args: + hp: HashPos instance for the source content + value: The value to resolve (typically line content rather than a hash) + + Returns: + A tilde-wrapped hash ID if resolution succeeds, None otherwise + """ + if not hp or not value: + return None + + value_stripped = value.strip() + if not value_stripped: + return None + + # Find all lines that exactly match this value + matching_lines = [] + for i, line in enumerate(hp.lines): + if line.strip() == value_stripped: + matching_lines.append((i, line)) + + # Only resolve if it matches exactly one unique line + if len(matching_lines) == 1: + idx, matched_line = matching_lines[0] + # Verify the line is actually unique in the source (appears only once) + if hp.line_counts.get(matched_line, 0) == 1: + hash_id = hp.generate_public_id(matched_line, idx, 1) + return HashPos.get_wrapped_id(hash_id) + + return None + + def apply_hashline_operations( original_content: str, operations: list, @@ -1730,15 +1815,35 @@ def apply_hashline_operations( # Normalize hashline inputs in operations normalized_operations = [] failed_ops = [] + # Create a single HashPos instance for preliminary resolution and content hashing + hp = HashPos(original_content) if original_content else HashPos("") # Loop through each operation to normalize hashline strings for i, op in enumerate(operations): try: normalized_op = op.copy() # Normalize start line hash to ensure consistent format - normalized_op["start_line_hash"] = normalize_hashline(op["start_line_hash"]) + try: + normalized_op["start_line_hash"] = normalize_hashline(op["start_line_hash"]) + except (ContentHashError, ValueError): + # Preliminary resolution: check if the value matches a unique line + resolved = _try_resolve_as_unique_line(hp, op["start_line_hash"]) + if resolved is not None: + normalized_op["start_line_hash"] = resolved + else: + raise + if "end_line_hash" in op: # Normalize end line hash if present - normalized_op["end_line_hash"] = normalize_hashline(op["end_line_hash"]) + try: + normalized_op["end_line_hash"] = normalize_hashline(op["end_line_hash"]) + except (ContentHashError, ValueError): + # Preliminary resolution: check if the value matches a unique line + resolved = _try_resolve_as_unique_line(hp, op["end_line_hash"]) + if resolved is not None: + normalized_op["end_line_hash"] = resolved + else: + raise + normalized_operations.append(normalized_op) except Exception as e: failed_ops.append({"index": i, "error": str(e), "operation": op}) @@ -1755,7 +1860,7 @@ def apply_hashline_operations( # Apply hashline to original content once # This converts content to hashed lines for line tracking - hashed_content = hashline(original_content) + hashed_content = hp.format_content() hashed_lines = hashed_content.splitlines(keepends=True) # Resolve all operations to indices first diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index a6e528e198e..81622256d66 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -2,6 +2,10 @@ import xxhash +# Delimiter used to wrap public hash IDs +HASH_DELIMITER = "~" +UNIQUE_HASH_DELIMITER = "~~" + class HashPos: # 1024-character Base1024 corpus @@ -60,131 +64,127 @@ class HashPos: "ÂÃÄÇ" ) - # We escape every individual character just to be completely safe from regex metacharacters _B1024_REGEX_SET = "".join(re.escape(c) for c in B1024) - # Regex pattern for HashPos format: {3-char-hash}:: - HASH_PREFIX_RE = re.compile(rf"^([{_B1024_REGEX_SET}]{{3}})::") - # Regex for normalization: 3 hash chars optionally followed by '::' - NORMALIZE_RE = re.compile(rf"^([{_B1024_REGEX_SET}]{{3}})(?:)?::") - # Regex for a raw 3-character fragment - FRAGMENT_RE = re.compile(rf"^[{_B1024_REGEX_SET}]{{3}}$") + # Regex matches EITHER the exact string '~~' OR a tilde-wrapped 4-character Base1024 hash + HASH_PREFIX_RE = re.compile( + rf"^({UNIQUE_HASH_DELIMITER}|{HASH_DELIMITER}[{_B1024_REGEX_SET}]{{4}}{HASH_DELIMITER})" + ) + FRAGMENT_RE = re.compile( + rf"^({UNIQUE_HASH_DELIMITER}|{HASH_DELIMITER}[{_B1024_REGEX_SET}]{{4}}{HASH_DELIMITER})$" + ) - # Looser pattern: any 3 chars with at least one non-ASCII followed by :: - _LOOSE_PREFIX_RE = re.compile(r"^(?=.{0,2}[^\x00-\x7f]).{3}::") + # Loose prefix for robust stripping: Matches a tilde-wrapped 4-char string containing non-ASCII + _LOOSE_PREFIX_RE = re.compile( + rf"^{HASH_DELIMITER}(?=.{{0,3}}[^\x00-\x7f]).{{4}}{HASH_DELIMITER}" + ) def __init__(self, source_text: str = ""): self.lines = source_text.splitlines() self.total = len(self.lines) + self.line_counts = {} + for line in self.lines: + if line.strip(): + self.line_counts[line] = self.line_counts.get(line, 0) + 1 + def _get_line_hash(self, text: str) -> int: - """ - Creates a 20-bit digest of the current line's text. - """ return xxhash.xxh3_64_intdigest(text.encode("utf-8")) & 0xFFFFF - def _get_adjacent_hash(self, line_idx: int) -> int: - """ - Creates a 10-bit digest of specific surrounding lines at offsets - -7, -5, -3, -2, +2, +3, +5, +7 to provide local context. - """ - offsets = [-7, -5, -3, -2, 2, 3, 5, 7] - adjacent_lines = [] - - for offset in offsets: - target_idx = line_idx + offset - if 0 <= target_idx < self.total: - adjacent_lines.append(self.lines[target_idx]) - - context = "\n".join(adjacent_lines) - return xxhash.xxh3_64_intdigest(context.encode("utf-8")) & 0x3FF - - def generate_private_id(self, text: str) -> str: - """ - Generates a fast 12-bit (3 hex chars) hash based purely on the line text. - """ - bits = xxhash.xxh3_64_intdigest(text.encode("utf-8")) & 0xFFF - return f"{bits:03x}" - - def generate_public_id(self, text: str, line_idx: int) -> str: - """ - Generates a 3-character Base1024 ID. - Layout: [20-bit Line Hash] [10-bit Adjacent Hash] = 30 bits total. - Each Base1024 character holds 10 bits. - """ + def generate_public_id(self, text: str, line_idx: int, occurrence: int) -> str: line_hash = self._get_line_hash(text) - adj_hash = self._get_adjacent_hash(line_idx) - # Pack the 30-bit integer - packed = (line_hash << 10) | adj_hash + # Explicit modulo for bounds wrapping + idx_bits = line_idx % 16384 + occ_bits = occurrence % 64 + + packed = (line_hash << 20) | (idx_bits << 6) | occ_bits res = "" - for _ in range(3): - # Extract 10 bits at a time using modulo 1024 + for _ in range(4): res += self.B1024[packed % 1024] packed //= 1024 return res - def unpack_public_id(self, public_id: str) -> tuple[int, int]: - """ - Reverses the Public ID back into its (Line Hash, Adjacent Hash) values. - """ + def unpack_public_id(self, public_id: str) -> tuple[int, int, int]: packed = 0 for i, char in enumerate(public_id): - # Each character restores 10 bits packed |= self.B1024.index(char) << (10 * i) - # Extract bits based on layout - line_hash = (packed >> 10) & 0xFFFFF - adj_hash = packed & 0x3FF + occ_bits = packed & 0x3F + idx_bits = (packed >> 6) & 0x3FFF + line_hash = (packed >> 20) & 0xFFFFF - return line_hash, adj_hash + return line_hash, idx_bits, occ_bits - def format_content(self, use_private_ids: bool = False, start_line: int = 1) -> str: + def format_content(self, start_line: int = 1) -> str: formatted_lines = [] + seen = {} + for i, line in enumerate(self.lines): - prefix = ( - self.generate_private_id(line) - if use_private_ids - else self.generate_public_id(line, i) - ) - if line.strip(): - formatted_lines.append(f"{prefix}::{line}") - else: + if not line.strip(): formatted_lines.append(f"{line}") + continue + + count = self.line_counts[line] + + if count == 1: + # Flush directly against code using the unique token + formatted_lines.append(f"{UNIQUE_HASH_DELIMITER}{line}") + else: + occ = seen.get(line, 0) + 1 + seen[line] = occ + prefix = self.generate_public_id(line, i, occ) + # Wrap the generated Base1024 hash in tildes + formatted_lines.append(f"{self.get_wrapped_id(prefix)}{line}") return "\n".join(formatted_lines) def resolve_to_lines(self, public_id: str, start_line: int = 1) -> list[int]: - target_line_hash, target_adj_hash = self.unpack_public_id(public_id) - matches = [] + if public_id == UNIQUE_HASH_DELIMITER: + raise ValueError( + f"Cannot spatially resolve the unique '{UNIQUE_HASH_DELIMITER}' identifier without line text." + ) + + # Strip the surrounding tildes to unpack the core 4 characters + clean_id = public_id.strip(HASH_DELIMITER) + if len(clean_id) != 4: + raise ValueError(f"Invalid public ID string for unpacking: {public_id}") + + target_line_hash, target_idx, target_occ = self.unpack_public_id(clean_id) - # 1. Primary Filter: Find all lines whose 20-bit line content hash matches + matches = [] for i, line in enumerate(self.lines): if self._get_line_hash(line) == target_line_hash: - matches.append(i) + matches.append((i, line)) if not matches: return [] - # If perfectly unique (highly likely given 20 bits of line entropy), return immediately if len(matches) == 1: - return matches + return [matches[0][0]] + + current_seen = {} + scored_matches = [] + + for i, line in matches: + current_seen[line] = current_seen.get(line, 0) + 1 + current_occ = current_seen[line] - # 2. Tie-Breaking Heuristic: - # If multiple identical lines exist, score them based on adjacency match. - def score_match(idx: int) -> int: - # Adjacency match: 0 means exact match, 1 means mismatch (we want lower scores) - return 0 if self._get_adjacent_hash(idx) == target_adj_hash else 1 + # Apply identical modulo to current spatial data before comparing + current_idx_mod = i % 16384 + current_occ_mod = current_occ % 64 - matches.sort(key=score_match) + # Cartesian distance squared + distance_sq = ((current_idx_mod - target_idx) ** 2) + ( + (current_occ_mod - target_occ) ** 2 + ) + scored_matches.append((distance_sq, i)) - return matches + scored_matches.sort(key=lambda x: x[0]) + return [m[1] for m in scored_matches] def resolve_range(self, start_id: str, end_id: str) -> tuple[int, int]: - """ - Resolves a block range from two Public IDs. - """ starts = self.resolve_to_lines(start_id) ends = self.resolve_to_lines(end_id) @@ -197,15 +197,16 @@ def resolve_range(self, start_id: str, end_id: str) -> tuple[int, int]: return s, e raise ValueError( - f"Found matches for {start_id} and {end_id}, but no logically ordered range or unique matches." + f"Found matches for {start_id} and {end_id}, but no logically ordered range." ) + @staticmethod + def get_wrapped_id(public_id: str) -> str: + """Wrap a public ID with the HashPos delimiters for use in hashline content.""" + return f"{HASH_DELIMITER}{public_id}{HASH_DELIMITER}" + @staticmethod def strip_prefix(text: str) -> str: - """ - Remove HashPos prefixes from the start of every line. - Also strips any 3-char sequence with at least one non-ASCII char followed by ::. - """ lines = text.splitlines(keepends=True) result_lines = [] for line in lines: @@ -213,35 +214,25 @@ def strip_prefix(text: str) -> str: if stripped_line == line: stripped_line = HashPos._LOOSE_PREFIX_RE.sub("", line, count=1) result_lines.append(stripped_line) - return "".join(result_lines) @staticmethod def extract_prefix(line: str) -> str: - """ - Extract the hash prefix from a line if it has a HashPos prefix. - """ match = HashPos.HASH_PREFIX_RE.match(line) if match: return match.group(1) return "" @staticmethod - def normalize(hashpos_str: str) -> str: - """ - Normalize a HashPos string to the 3-character hash fragment. - """ + def normalize(hashpos_str: str, throw=True) -> str: if hashpos_str is None: raise ValueError("HashPos string cannot be None") - if HashPos.FRAGMENT_RE.match(hashpos_str): - return hashpos_str - - match = HashPos.NORMALIZE_RE.match(hashpos_str) + match = HashPos.HASH_PREFIX_RE.match(hashpos_str) if match: return match.group(1) - raise ValueError( - f"Invalid HashPos format '{hashpos_str}'. " - r"Expected a 3-character string from the Base1024 character set." - ) + if throw: + raise ValueError(f"Invalid HashPos format '{hashpos_str}'.") + else: + return False diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index 73a88186506..36747105424 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -1,7 +1,7 @@ """ Singleton-like proxy injected into the orchestration environment. -Usage in LLM-generated code:: +Usage in LLM-generated code: read_tool = Agent.get_tool("ReadFile") result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") @@ -22,7 +22,7 @@ class AgentProxy: """ Singleton-like proxy injected into the orchestration environment. - Usage in LLM-generated code:: + Usage in LLM-generated code: read_tool = Agent.get_tool("ReadFile") result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") @@ -105,7 +105,7 @@ def _find_mcp_server(self, server_name: str, server_prefix: str) -> Any: def peek(self, result: Any) -> str: """Inspect the structure of a tool result and return a readable summary. - Tool results have a standard shape:: + Tool results have a standard shape: { "result": [{"content": ..., "_": {"file_path": ..., ...}}], @@ -117,7 +117,7 @@ def peek(self, result: Any) -> str: helping the LLM navigate deeply nested tool outputs. Leaf values include a short content snippet so you can see actual data. - Example:: + Example: output = await grep_tool.call(pattern="TODO", file_glob="*.py") print(Agent.peek(output)) @@ -207,7 +207,7 @@ def get_content_id(self, file_path: str, line_content: str) -> str: Supports three modes: - **@L{number}**: e.g., `Agent.get_content_id("foo.py", "@L42")` returns the content ID of line 42 (1-based). - - **content ID passthrough**: e.g., `Agent.get_content_id("foo.py", "abc::")` + - **content ID passthrough**: e.g., `Agent.get_content_id("foo.py", "~abcd~")` verifies and returns an existing content ID string. - **text match**: e.g., `Agent.get_content_id("foo.py", "def greet(")` returns the content ID of the unique line containing that text. @@ -216,7 +216,7 @@ def get_content_id(self, file_path: str, line_content: str) -> str: import re from cecli.helpers.hashline import resolve_content_to_hashline_ids - from cecli.helpers.hashpos.hashpos import HashPos + from cecli.helpers.hashpos.hashpos import HASH_DELIMITER, HashPos from cecli.tools.utils.helpers import resolve_paths abs_path, rel_path = resolve_paths(self._coder, file_path) @@ -236,10 +236,12 @@ def get_content_id(self, file_path: str, line_content: str) -> str: line_num = int(m.group(1)) - 1 if line_num < 0 or line_num >= len(lines): raise ValueError(f"Line {m.group(1)} out of range (file has {len(lines)} lines)") - return hp.generate_public_id(lines[line_num], line_num) + "::" + line_text = lines[line_num] + occurrence = 1 + sum(1 for i in range(line_num) if lines[i] == line_text) + return hp.get_wrapped_id(hp.generate_public_id(line_text, line_num, occurrence)) # Content ID passthrough: already looks like a content ID - if "::" in line_content: + if HashPos.FRAGMENT_RE.match(line_content): from cecli.helpers.hashline import ContentHashError, normalize_hashline try: @@ -249,11 +251,20 @@ def get_content_id(self, file_path: str, line_content: str) -> str: return line_content except (ContentHashError, ValueError): pass + + # Fall back: value looks like a content ID (contains "~") but couldn't be + # resolved. Strip the prefix and try to match the remaining content. + stripped = HashPos.strip_prefix(line_content) + if stripped != line_content and stripped.strip(): + result, _ = resolve_content_to_hashline_ids(content, stripped, None) + if result != stripped and HASH_DELIMITER in str(result): + return result + raise ValueError(f"Content ID '{line_content}' not found in {file_path}") # Text match via resolve_content_to_hashline_ids result, _ = resolve_content_to_hashline_ids(content, line_content, None) - if result == line_content or "::" not in str(result): + if result == line_content or HASH_DELIMITER not in str(result): # Find all matching lines for a helpful error message matching = [i + 1 for i, line in enumerate(lines) if line_content in line] if len(matching) > 1: @@ -301,7 +312,7 @@ async def edit_region( """ Thin wrapper around `EditFile` that accepts `{"start": content_id, "end": content_id}` region dicts. - Use with `Agent.resolve_regions()` and `regions.get(name)`:: + Use with `Agent.resolve_regions()` and `regions.get(name)`: regions = Agent.resolve_regions("foo.py", [ {"name": "my_func", "start": "def my_func", "end": "return result"}, diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index e6caf57ef24..4aa20c69d65 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -19,12 +19,12 @@ class AgentRegion: ``get_end(name)`` re-read the file and re-resolve patterns at each call. Use these directly in ``EditFile`` calls for always-fresh IDs. - When a region spec uses a **content ID** (e.g. ``"abc::"``) instead of + When a region spec uses a **content ID** (e.g. `"~abcd~"`) instead of text, the referenced line content is snapshotted on first resolution. If the ID goes stale after intervening edits, subsequent resolutions fall back to content matching against the snapshotted line text. - Example usage in orchestration code:: + Example usage in orchestration code: regions = Agent.resolve_regions("foo.py", [ {"name": "helper", "start": "def helper", "end": "return x"}, diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index e2afcda3b99..0665a150190 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -111,7 +111,7 @@ async def _safe_gather(*args: Any, **named_awaitables: Any): Safely execute multiple awaitables concurrently. All awaitables must be passed as keyword arguments. Results are returned - as a ``GatherResult`` with attribute and key access:: + as a ``GatherResult`` with attribute and key access: results = await gather(read_a=task_a, grep_b=task_b) print(results.read_a) # attribute access diff --git a/cecli/hooks/helpers.py b/cecli/hooks/helpers.py index 7b8ce539c1a..0d0677c2c11 100644 --- a/cecli/hooks/helpers.py +++ b/cecli/hooks/helpers.py @@ -3,7 +3,7 @@ Provides a higher-level API for accessing conversation history, making model calls, and working with sub-agents from within hook implementations. -Typical usage:: +Typical usage: from cecli.hooks import HookHelpers diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index 93c958b9e23..8ca4718cf63 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -26,22 +26,30 @@ main_system: | **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT - File contents will be prefixed with identifiers. Each line starts with a case-sensitive content ID followed by `::`. These are used to target where editing tools will perform edits. - They are virtual, deterministic identifiers generated on-the-fly when representing files and never change from reading alone. Only IDs within ~5 lines of an edit site are regenerated — IDs farther away remain valid across edits. - Do not search for these content IDs directly. You will not be able to generate them. Use content IDs from a recent ReadFile to target edits. + File contents are presented with virtual prefix identifiers to help you target edits accurately. These are generated on-the-fly when reading the file. + + - **Unique Lines (`~~`):** Lines that appear only once in the file are prefixed with `~~`. To target these lines for edits, you can simply reference the exact literal text of the line. + - **Duplicate Lines (e.g., `~“0车加~`):** Lines that appear multiple times are prefixed with an identifier containing an occurrence index and a content hash. You MUST include this exact prefix when targeting these lines to disambiguate which specific instance you want to edit. + + Do not attempt to generate, guess, or calculate these duplicate IDs yourself. Always use the exact line contents and prefixes provided from the most recent file read. **Example File** ``` - R号定::#!/usr/bin/env python3 + ~~#!/usr/bin/env python3 + + ~~def example_process_data(data): + ~~ if not data: + ~“0车加~ return None - ،即游::def example_method(): - ป止나:: return "example" + ~~ data.clean() - 么으q::def foo(): - 内능索:: return "bar" + ~~ if data.is_stale(): + ~引0车加~ return None - 增せе::def bar(): - 打Б2:: return example_method() + " " + foo() + ~~ try: + ~~ data.save() + ~~ except Exception: + ~니0车加~ return None ``` ## Core Workflow diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index f157df21532..191bffd66e6 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -11,22 +11,30 @@ main_system: | **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT - File contents will be prefixed with identifiers. Each line starts with a case-sensitive content ID followed by `::`. These are used to target where editing tools will perform edits. - They are virtual, deterministic identifiers generated on-the-fly when representing files and never change from reading alone. Only IDs within ~5 lines of an edit site are regenerated — IDs farther away remain valid across edits. - Do not search for these content IDs directly. You will not be able to generate them. Use content IDs from a recent ReadFile to target edits. + File contents are presented with virtual prefix identifiers to help you target edits accurately. These are generated on-the-fly when reading the file. + + - **Unique Lines (`~~`):** Lines that appear only once in the file are prefixed with `~~`. To target these lines for edits, you can simply reference the exact literal text of the line. + - **Duplicate Lines (e.g., `~“0车加~`):** Lines that appear multiple times are prefixed with an identifier containing an occurrence index and a content hash. You MUST include this exact prefix when targeting these lines to disambiguate which specific instance you want to edit. + + Do not attempt to generate, guess, or calculate these duplicate IDs yourself. Always use the exact line contents and prefixes provided from the most recent file read. **Example File** ``` - R号定::#!/usr/bin/env python3 + ~~#!/usr/bin/env python3 + + ~~def example_process_data(data): + ~~ if not data: + ~“0车加~ return None - ،即游::def example_method(): - ป止나:: return "example" + ~~ data.clean() - 么으q::def foo(): - 内능索:: return "bar" + ~~ if data.is_stale(): + ~引0车加~ return None - 增せе::def bar(): - 打Б2:: return example_method() + " " + foo() + ~~ try: + ~~ data.save() + ~~ except Exception: + ~니0车加~ return None ``` ## Core Workflow diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 133b10671c1..5e2f2c9b30b 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -1,4 +1,6 @@ from cecli.helpers.hashline import ( + HASH_DELIMITER, + UNIQUE_HASH_DELIMITER, ContentHashError, apply_hashline_operations, get_hashline_diff, @@ -44,15 +46,16 @@ class Tool(BaseTool): "function": { "name": "EditFile", "description": ( - "Edit text in one or more files using content ID markers. " + "Edit text in one or more files using virtual identifiers. " "You can perform multiple 'replace' or 'delete' operations in a single call. " "CRITICAL RULES: " - "1. Start and end content IDs are INCLUSIVE. Both will be modified or deleted. " - "2. Content IDs MUST include the `::` demarcator. " - "3. Edits within the same file MUST NOT be adjacent or overlapping. " - "4. For empty files, you MUST use '@000' as the content ID reference. " - "5. After an edit, only IDs within ~5 lines of the change are regenerated. " - "IDs farther from the edit site remain usable." + "1. Start and end markers are INCLUSIVE. Both will be modified or deleted. " + f"2. To target unique lines (prefixed with '{UNIQUE_HASH_DELIMITER}'), use their exact literal text as the marker. " # noqa + f"3. To target duplicate lines, you MUST include the exact hashed prefix (e.g., '{HASH_DELIMITER}“0车加{HASH_DELIMITER}'). " # noqa + "4. Edits within the same file MUST NOT be adjacent or overlapping. " + "5. For empty files, you MUST use '@000' as the reference. " + "6. Identifiers track global occurrences. Adding, modifying, or deleting a line can instantly " + "change the prefixes of identical lines anywhere else in the file. Re-read to get fresh IDs after editing." # noqa ), "parameters": { "type": "object", @@ -92,14 +95,14 @@ class Tool(BaseTool): "type": "string", "description": ( "The exact content ID and demarcator for the start of the edit " - "(e.g., 'abc::'). For empty files, use '@000'." + f"(e.g., '{HASH_DELIMITER}abcd{HASH_DELIMITER}'). For empty files, use '@000'." ), }, "end_line": { "type": "string", "description": ( "The exact content ID and demarcator for the end of the edit " - "(e.g., 'xyz::'). For empty files, use '@000'." + f"(e.g., '{HASH_DELIMITER}wxyz{HASH_DELIMITER}'). For empty files, use '@000'." ), }, }, @@ -226,12 +229,13 @@ def execute( edit_start_line, edit_end_line = resolve_content_to_hashline_ids( original_content, edit_start_line, edit_end_line ) - # 4. Auto-sanitize malformed boundaries (strip accidentally appended code) - if isinstance(edit_start_line, str) and "::" in edit_start_line: - edit_start_line = normalize_hashline(edit_start_line) - if isinstance(edit_end_line, str) and "::" in edit_end_line: - edit_end_line = normalize_hashline(edit_end_line) + if isinstance(edit_start_line, str): + test_line = normalize_hashline(edit_start_line, throw=False) + edit_start_line = test_line if test_line else edit_start_line + if isinstance(edit_end_line, str): + test_line = normalize_hashline(edit_end_line, throw=False) + edit_end_line = test_line if test_line else edit_end_line # --------------------------------------------------------- diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index ff59c2a3ee2..e9a12aeda40 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -29,38 +29,22 @@ class Tool(BaseTool): "function": { "name": "ReadFile", "description": ( - "Get content ID prefixed content between start and end markers in files." - " Accepts an array of `read` objects, each with file_path, range_start, range_end." - " They can contain up to 3 lines of content. Avoid using singular generic keywords and" - " symbols. Special markers @000 and 000@ represent the file boundaries and can be" - " used for range_start and range_end for the first and last lines of the file" - " respectively. Use '@L' followed by a line number (e.g., '@L10', '@L25')" - " for structured line-range lookups." + "Get prefixed content between start and end markers. Accepts an array of `read` objects " + "(file_path, range_start, range_end). Range markers can be text patterns (up to 3 lines), " + "file boundaries (@000, 000@), or exact line numbers (@L10). " + "Contextual end markers (@C{num}, @P{num}, @N{num}) expand symmetrically, previously, or next " + "around a single unique range_start match." + " Use line hints (e.g., 'my_func @L150') to disambiguate text matches. " "" - " Contextual range_end markers (@C, @P, @N) expand around a unique range_start match:" - " '@C{number}' includes number lines before AND after the match (symmetric context)." - " '@P{number}' includes number lines PREVIOUS to (before) the match — the match" - " itself becomes the end of the range. '@N{number}' includes number lines NEXT" - " (after) the match — the match itself becomes the start of the range." - " These markers require range_start to match exactly one location." + "File lines are prefixed with virtual, deterministic identifiers generated on-the-fly: " + "Because identifiers track global occurrences, " + "adding or deleting a line can instantly change the prefix " + "of identical lines anywhere else in the file as well as content after the edit." + "Always re-read the file to get fresh IDs after making edits." "" - " Line hints: append ' @L' to any text pattern (e.g., 'my_func @L1506')" - " to disambiguate which match is intended when a pattern matches in many places." - " The @L hint is a proximity guide, not a strict line number." - "" - " It is best to use function names," - " variable declarations, entire line contents and other meaningful identifiers" - " Content IDs persist across reads and only change locally (~5 lines) around an edit site." - " IDs far from any edit remain valid." - " Only use the same pattern for range_start and range_end when fetching full method definitions." - " Do not use empty strings for the range_start and range_end." - " Content IDs returned by ReadFile can be used as range markers for EditFile." - " Re-read the file if you need fresh IDs after editing near your target region." - " Always use the ReadFile tool instead of cli tools for reading file contents." - " Line number and special marker ranges greater than 200 lines will return" - " preview content for further, more scoped investigation." - " Call this tool sequentially on increasingly finer grained searches " - " to help with understanding important structural features in large files." + "Avoid generic keywords; use meaningful identifiers like function names. Do not use empty strings. " + "Always use ReadFile instead of CLI tools. Ranges >200 lines return a structural preview. " + "Call sequentially with increasingly fine-grained searches to drill down into large files." ), "parameters": { "type": "object", diff --git a/cecli/tools/utils/responses.py b/cecli/tools/utils/responses.py index 8b92252bfaa..561f78d0496 100644 --- a/cecli/tools/utils/responses.py +++ b/cecli/tools/utils/responses.py @@ -15,7 +15,7 @@ class ToolResponse: Each result item has a ``content`` key (the primary output) and an ``_`` key (metadata). Use ``append_result(content, metadata=None)`` - Usage inside a tool's ``execute`` method:: + Usage inside a tool's ``execute`` method: response = ToolResponse("my_tool", result_type="str") response.append_result("Operation completed successfully") diff --git a/cecli/tools/validations/validations.py b/cecli/tools/validations/validations.py index 15fc22ab460..e6dc8e5cd90 100644 --- a/cecli/tools/validations/validations.py +++ b/cecli/tools/validations/validations.py @@ -8,7 +8,7 @@ for list iteration) to lists of validation method names that are executed sequentially on the parameter value. -Example:: +Example: VALIDATIONS = { "delegations": ["coerce_list"], From 0b53c42210bfbe5d9fafa25a69645cdbff886f3f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 23:04:17 -0400 Subject: [PATCH 32/68] Fix edit_file guidance --- cecli/tools/edit_file.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 5e2f2c9b30b..57742294e3b 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -94,15 +94,23 @@ class Tool(BaseTool): "start_line": { "type": "string", "description": ( - "The exact content ID and demarcator for the start of the edit " - f"(e.g., '{HASH_DELIMITER}abcd{HASH_DELIMITER}'). For empty files, use '@000'." + "The exact reference for the start of the edit. " + "For duplicate lines with a specific hash, " + "use the 4-character hash wrapped in tildes (e.g., '~WecX~'). " + "For unique lines marked with the generic '~~' prefix, " + "provide the exact full text of the line. " + "For empty files, use '@000'." ), }, "end_line": { "type": "string", "description": ( - "The exact content ID and demarcator for the end of the edit " - f"(e.g., '{HASH_DELIMITER}wxyz{HASH_DELIMITER}'). For empty files, use '@000'." + "The exact reference for the end of the edit. " + "For duplicate lines with a specific hash, " + "use the 4-character hash wrapped in tildes (e.g., '~WecX~'). " + "For unique lines marked with the generic '~~' prefix, " + "provide the exact full text of the line. " + "For empty files, use '@000'." ), }, }, From 5d482726fa9e31c62b03c8654c02397a19103a5c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 08:16:08 -0400 Subject: [PATCH 33/68] Make region resolver work with tilde based hashpos --- .../helpers/orchestration/region_resolver.py | 8 +++ cecli/tools/edit_file.py | 58 +++++++++++++++++++ tests/helpers/test_orchestration.py | 20 +++---- 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index 4aa20c69d65..137bc6eaca8 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -188,6 +188,14 @@ def _resolve(self, name: str) -> tuple[str, str, int, int]: start_hint = (explicit_start - 1) if explicit_start is not None else extracted_start end_hint = (explicit_end - 1) if explicit_end is not None else extracted_end + # Strip hashline prefixes from text patterns — the LLM may have copied + # content-ID-prefixed lines from a ReadFile response (e.g. ~XYZ12::text). + # Content-ID patterns and special markers are left untouched. + if not self._looks_like_content_id(start_pattern) and start_pattern not in ("@000", "000@"): + start_pattern = HashPos.strip_prefix(start_pattern) + if not self._looks_like_content_id(end_pattern) and end_pattern not in ("@000", "000@"): + end_pattern = HashPos.strip_prefix(end_pattern) + # Validate uniqueness for text-based patterns (not content IDs or special markers). self._validate_pattern_uniqueness(start_pattern, start_hint, "start", name, lines) self._validate_pattern_uniqueness(end_pattern, end_hint, "end", name, lines) diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 57742294e3b..8ca9a2e122c 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -1,9 +1,11 @@ from cecli.helpers.hashline import ( HASH_DELIMITER, + HASHLINE_PREFIX_RE, UNIQUE_HASH_DELIMITER, ContentHashError, apply_hashline_operations, get_hashline_diff, + hashline, normalize_hashline, resolve_content_to_hashline_ids, strip_hashline, @@ -232,6 +234,62 @@ def execute( edit_start_line = "@000" edit_end_line = "@000" + # 2.5 Resolve @L{num} notation to line content or hash ID + if ( + isinstance(edit_start_line, str) + and edit_start_line.startswith("@L") + and len(edit_start_line) > 2 + and edit_start_line[2:].isdigit() + ): + if original_content and original_content.strip(): + source_lines = original_content.splitlines() + line_num = ( + int(edit_start_line[2:]) - 1 + ) # 1-indexed to 0-indexed + if 0 <= line_num < len(source_lines): + line_text = source_lines[line_num] + if source_lines.count(line_text) == 1: + # Unique line — let resolve_content_to_hashline_ids handle it + edit_start_line = line_text + else: + # Duplicate line — resolve directly to content ID + hashed_content = hashline(original_content) + hashed_lines = hashed_content.splitlines() + match = HASHLINE_PREFIX_RE.match(hashed_lines[line_num]) + if match: + edit_start_line = match.group(1) + else: + raise ToolError( + f"@L reference line {int(edit_start_line[2:])} is out of range " + f"(file has {len(source_lines)} lines)" + ) + if ( + isinstance(edit_end_line, str) + and edit_end_line.startswith("@L") + and len(edit_end_line) > 2 + and edit_end_line[2:].isdigit() + ): + if original_content and original_content.strip(): + source_lines = original_content.splitlines() + line_num = int(edit_end_line[2:]) - 1 + if 0 <= line_num < len(source_lines): + line_text = source_lines[line_num] + if source_lines.count(line_text) == 1: + # Unique line — let resolve_content_to_hashline_ids handle it + edit_end_line = line_text + else: + # Duplicate line — resolve directly to content ID + hashed_content = hashline(original_content) + hashed_lines = hashed_content.splitlines() + match = HASHLINE_PREFIX_RE.match(hashed_lines[line_num]) + if match: + edit_end_line = match.group(1) + else: + raise ToolError( + f"@L reference line {int(edit_end_line[2:])} is out of range " + f"(file has {len(source_lines)} lines)" + ) + # 3. Resolve non-hashline content values to content IDs first # (before normalize_hashline which would fail on arbitrary content) edit_start_line, edit_end_line = resolve_content_to_hashline_ids( diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 4acf3d611da..76faf5cce29 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -779,8 +779,8 @@ def test_agent_region_basic(tmp_path): start_id = regions.get_start("foo") end_id = regions.get_end("foo") - assert "::" in start_id - assert "::" in end_id + assert "~" in start_id + assert "~" in end_id start_line = regions.get_start_line("foo") end_line = regions.get_end_line("foo") @@ -878,8 +878,8 @@ def main(): start_id = regions.get_start(name) end_id = regions.get_end(name) - assert "::" in start_id, f"{name} start_id should be a content ID: {start_id}" - assert "::" in end_id, f"{name} end_id should be a content ID: {end_id}" + assert "~" in start_id, f"{name} start_id should be a content ID: {start_id}" + assert "~" in end_id, f"{name} end_id should be a content ID: {end_id}" assert regions.get_start_line(name) > 0 assert regions.get_end_line(name) > 0 @@ -974,7 +974,7 @@ def farewell(name): # First resolution should snapshot the line content result_id = regions2.get_start("greet") - assert "::" in result_id + assert "~" in result_id # Now modify the file (simulating an edit that shifts hashlines) new_source = """\ @@ -993,7 +993,7 @@ def farewell(name): # The original content ID is now stale, but the snapshot should let # us resolve via content matching result_id2 = regions2.get_start("greet") - assert "::" in result_id2, f"Should resolve via fallback content match, got: {result_id2!r}" + assert "~" in result_id2, f"Should resolve via fallback content match, got: {result_id2!r}" def test_agent_region_rejects_ambiguous_pattern(tmp_path): @@ -1072,8 +1072,8 @@ def baz(): start_id = regions.get_start("bar") end_id = regions.get_end("bar") - assert "::" in start_id - assert "::" in end_id + assert "~" in start_id + assert "~" in end_id assert regions.get_start_line("bar") == 4 assert regions.get_end_line("bar") == 5 @@ -1150,8 +1150,8 @@ def baz(): start_id = regions.get_start("bar") end_id = regions.get_end("bar") - assert "::" in start_id - assert "::" in end_id + assert "~" in start_id + assert "~" in end_id assert regions.get_start_line("bar") == 4 assert regions.get_end_line("bar") == 5 From 4aad93967351cac59476814e5720e22e11d05ff3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 08:24:43 -0400 Subject: [PATCH 34/68] Allow peak to recurse in to classes --- cecli/helpers/orchestration/agent_proxy.py | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index 36747105424..9ecfa5ce7bb 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -178,6 +178,19 @@ def _inspect_structure(obj: Any, prefix: str = "", depth: int = 0) -> str: lines.append(f"{path}: list[{len(value)}] {type(first).__name__}") else: lines.append(f"{path}: list (empty)") + elif ( + hasattr(value, "keys") + and hasattr(value, "items") + and not isinstance(value, (str, bytes)) + ): + sub_keys = list(value.keys())[:max_keys] + sub_suffix = "..." if len(value) > max_keys else "" + lines.append( + f"{path}: {type(value).__name__}(keys: [{', '.join(sub_keys)}{sub_suffix}])" + ) + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) else: preview = AgentProxy._content_preview(value) lines.append(f"{path}: {type(value).__name__} = {preview}") @@ -196,6 +209,52 @@ def _inspect_structure(obj: Any, prefix: str = "", depth: int = 0) -> str: lines.append(f"list[{len(obj)}] {type(first).__name__}") else: lines.append("list (empty)") + + elif hasattr(obj, "keys") and hasattr(obj, "items") and not isinstance(obj, (str, bytes)): + keys = list(obj.keys()) + display_keys = keys[:max_keys] + suffix = "..." if len(keys) > max_keys else "" + lines.append(f"{type(obj).__name__}(keys: [{', '.join(display_keys)}{suffix}])") + for key in display_keys: + value = obj[key] + path = f"{prefix}.{key}" if prefix else key + + if isinstance(value, dict): + sub_keys = list(value.keys())[:max_keys] + sub_suffix = "..." if len(value) > max_keys else "" + lines.append(f"{path}: dict[{' | '.join(sub_keys)}{sub_suffix}]") + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + elif isinstance(value, list): + if value: + first = value[0] + if isinstance(first, dict): + lines.append(f"{path}: list[{len(value)}] dict") + inner = AgentProxy._inspect_structure(first, f"{path}[0]", depth + 1) + if inner: + lines.append(inner) + else: + lines.append(f"{path}: list[{len(value)}] {type(first).__name__}") + else: + lines.append(f"{path}: list (empty)") + elif ( + hasattr(value, "keys") + and hasattr(value, "items") + and not isinstance(value, (str, bytes)) + ): + sub_keys = list(value.keys())[:max_keys] + sub_suffix = "..." if len(value) > max_keys else "" + lines.append( + f"{path}: {type(value).__name__}(keys: [{', '.join(sub_keys)}{sub_suffix}])" + ) + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + else: + preview = AgentProxy._content_preview(value) + lines.append(f"{path}: {type(value).__name__} = {preview}") + else: lines.append(f"{type(obj).__name__}") From afebc0d448dd488bb7d2e60205c869b3270061ee Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 09:10:56 -0400 Subject: [PATCH 35/68] Update quote detection in parser --- cecli/helpers/orchestration/safe_methods.py | 26 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index 0665a150190..f3d406efa2a 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -225,6 +225,26 @@ def _safe_dir(obj: Any) -> list: ] +def _find_closing_quote(code: str, quote: str, start: int) -> int: + """Find the position of the closing *quote* in *code*, skipping + backslash-escaped characters so that ``\"`` (escaped quote) is not + mistaken for the string terminator. + + Returns -1 if no unescaped closing quote is found. + """ + + i = start + while i < len(code): + if code[i] == chr(92): # backslash — skip the next char + i += 2 + continue + if code[i : i + len(quote)] == quote: + return i + i += 1 + + return -1 + + def _naive_escape_newlines(code: str, NL: str, BSN: str) -> str: """Replace literal newlines with ``\\n`` inside all string literals. @@ -242,7 +262,7 @@ def _naive_escape_newlines(code: str, NL: str, BSN: str) -> str: quote = quote * 3 result.append(quote) idx += len(quote) - end = code.find(quote, idx) + end = _find_closing_quote(code, quote, idx) if end == -1: result.append(code[idx:]) idx = len(code) @@ -330,7 +350,7 @@ def _process_fstring_body(code: str, idx: int, quote: str, NL: str, BSN: str, re if pos + 2 < len(code) and code[pos : pos + 3] == nq * 3: nq = nq * 3 pos += len(nq) - ne = code.find(nq, pos) + ne = _find_closing_quote(code, nq, pos) pos = (ne + len(nq)) if ne != -1 else len(code) continue @@ -376,7 +396,7 @@ def _fstring_aware_escape_newlines(code: str, NL: str, BSN: str) -> str: if is_fstring: idx = _process_fstring_body(code, idx, quote, NL, BSN, result) else: - end = code.find(quote, idx) + end = _find_closing_quote(code, quote, idx) if end == -1: result.append(code[idx:]) idx = len(code) From dce0e9ee1533d6198122b273a2192372081cfd8c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 09:24:04 -0400 Subject: [PATCH 36/68] Instead of crashing on statements, replace them with __security_raise so exceptions can be caught in orchestrate scripts --- cecli/helpers/orchestration/environment.py | 9 +- cecli/helpers/orchestration/security.py | 105 ++++++++++++++++----- 2 files changed, 85 insertions(+), 29 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 1adb7ac6f53..ea90c571b08 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -38,6 +38,7 @@ SecurityError, SecurityFilter, _cooperative_yield, + _security_raise, ) from cecli.helpers.orchestration.tool_proxy import ToolProxy @@ -272,6 +273,7 @@ def _allowed_tools(): "state": self.state, "shared_state": AgentExecutionEnv._shared_state, "__yield": _cooperative_yield, + "__security_raise": _security_raise, "NEWLINE": "\n", "allowed_methods": _allowed_methods, "allowed_tools": _allowed_tools, @@ -403,12 +405,7 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: code = f"Syntax Error in orchestration code: {e}" return {"results": code, "state_variables": self._state_snapshot()} - try: - SecurityFilter().visit(tree) - except SecurityError as e: - code = f"Security Error: {e}" - return {"results": code, "state_variables": self._state_snapshot()} - + tree = SecurityFilter().visit(tree) tree = LoopYieldInjector().visit(tree) ast.fix_missing_locations(tree) diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py index caadab2cc4c..7ecb0ec4be5 100644 --- a/cecli/helpers/orchestration/security.py +++ b/cecli/helpers/orchestration/security.py @@ -14,11 +14,30 @@ class SecurityError(Exception): """Raised when generated code violates security constraints.""" -class SecurityFilter(ast.NodeVisitor): +def _security_raise(message: str): + """Runtime raise helper injected into sandbox globals. + + Called by AST-rewritten forbidden expressions so that private-access + violations raise at *runtime* rather than rejecting the entire script + during the pre-execution AST walk. This allows try/except blocks to + gracefully handle unreachable code paths. + """ + raise SecurityError(message) + + +class SecurityFilter(ast.NodeTransformer): """ - AST node visitor that blocks dangerous constructs before they compile. + AST node transformer that rewrites dangerous constructs into runtime + ``__security_raise(...)`` calls. - Blocks: + Instead of rejecting the entire script during the pre-execution walk, + forbidden constructs are replaced with a call that raises + ``SecurityError`` at *runtime*. This means code inside ``try/except`` + blocks can gracefully handle unreachable paths while the security + boundary is preserved — any actual execution of private access still + fails. + + Rewrites: - All import statements (import X, from X import Y) - Access to private/dunder attributes (__class__, __subclasses__, etc.) - Calls to eval, exec, open, __import__, compile, breakpoint @@ -38,35 +57,75 @@ class SecurityFilter(ast.NodeVisitor): "delattr", } - def visit_Import(self, node: ast.Import) -> None: - raise SecurityError("Imports are disabled in the agent orchestration environment.") + _SAFE_DUNDER: set[str] = {"__name__", "__doc__"} - def visit_ImportFrom(self, node: ast.ImportFrom) -> None: - raise SecurityError("Imports are disabled in the agent orchestration environment.") + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + @staticmethod + def _make_raise_expr(message: str) -> ast.Call: + """Return an AST Call node that invokes ``_security_raise(message)``. + + The function ``_security_raise`` is injected into the sandbox + globals at execution time. + """ + return ast.Call( + func=ast.Name(id="__security_raise", ctx=ast.Load()), + args=[ast.Constant(value=message)], + keywords=[], + ) - _SAFE_DUNDER: set[str] = {"__name__", "__doc__"} + @staticmethod + def _make_raise_stmt(message: str) -> ast.Expr: + """Return an AST Expr statement wrapping ``_security_raise(message)``. - def visit_Attribute(self, node: ast.Attribute) -> None: - if node.attr.startswith("_"): - if node.attr in self._SAFE_DUNDER: - self.generic_visit(node) - return - raise SecurityError(f"Access to private/dunder attribute '{node.attr}' is forbidden.") - self.generic_visit(node) + Used when the forbidden construct is itself a statement + (import / global / nonlocal) rather than an expression. + """ + return ast.Expr(value=SecurityFilter._make_raise_expr(message)) - def visit_Call(self, node: ast.Call) -> None: - if isinstance(node.func, ast.Name) and node.func.id in self._DANGEROUS_BUILTINS: - raise SecurityError(f"Calling '{node.func.id}' is forbidden.") - self.generic_visit(node) + # ------------------------------------------------------------------ + # Statement visitors (import / global / nonlocal) + # ------------------------------------------------------------------ - def visit_Global(self, node: ast.Global) -> None: - raise SecurityError("The 'global' statement is disabled in the orchestration environment.") + def visit_Import(self, node: ast.Import) -> ast.Expr: + return self._make_raise_stmt("Imports are disabled in the agent orchestration environment.") - def visit_Nonlocal(self, node: ast.Nonlocal) -> None: - raise SecurityError( + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.Expr: + return self._make_raise_stmt("Imports are disabled in the agent orchestration environment.") + + def visit_Global(self, node: ast.Global) -> ast.Expr: + return self._make_raise_stmt( + "The 'global' statement is disabled in the orchestration environment." + ) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.Expr: + return self._make_raise_stmt( "The 'nonlocal' statement is disabled in the orchestration environment." ) + # ------------------------------------------------------------------ + # Expression visitors (attribute access / dangerous calls) + # ------------------------------------------------------------------ + + def visit_Attribute(self, node: ast.Attribute): + if node.attr.startswith("_"): + if node.attr in self._SAFE_DUNDER: + return self.generic_visit(node) + + return self._make_raise_expr( + f"Access to private/dunder attribute '{node.attr}' is forbidden." + ) + + return self.generic_visit(node) + + def visit_Call(self, node: ast.Call): + if isinstance(node.func, ast.Name) and node.func.id in self._DANGEROUS_BUILTINS: + return self._make_raise_expr(f"Calling '{node.func.id}' is forbidden.") + + return self.generic_visit(node) + class LoopYieldInjector(ast.NodeTransformer): """ From 8dddb6836c3e35b49fa58aeac8d151cd286b079a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 09:40:43 -0400 Subject: [PATCH 37/68] Add additional mehods to sandbox --- cecli/helpers/orchestration/agent_proxy.py | 18 ++++++++++++++++++ cecli/helpers/orchestration/environment.py | 6 ++++++ cecli/helpers/orchestration/safe_methods.py | 3 ++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index 9ecfa5ce7bb..b6763152efe 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -134,6 +134,24 @@ def peek(self, result: Any) -> str: return self._inspect_structure(result) + def get_value(self, result: Any, path: str, default: Any = None) -> Any: + """Safely access nested values in a tool result using dot-notation. + + Tool results have deeply nested dicts (e.g., ``result["result"][0]["_"]["file_path"]``). + ``get_value()`` provides a concise shorthand using ``nested.getter()``. + + Example: + + output = await grep_tool.call(pattern="TODO", file_glob="*.py") + file_path = Agent.get_value(output, "result.0._.file_path") + content = Agent.get_value(output, "result.0.content") + + Returns *default* if the path does not exist. + """ + from cecli.helpers import nested + + return nested.getter(result, path, default) + @staticmethod def _content_preview(value: Any, max_chars: int = 20) -> str: """Return a short preview of a scalar value's stringified content.""" diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index ea90c571b08..0c4264da2e1 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -224,6 +224,7 @@ def __init__(self, coder: Any) -> None: "filter": filter, "map": map, "chr": chr, + "next": next, "Exception": Exception, "ValueError": ValueError, "TypeError": TypeError, @@ -232,6 +233,10 @@ def __init__(self, coder: Any) -> None: "AttributeError": AttributeError, "RuntimeError": RuntimeError, "NameError": NameError, + "ZeroDivisionError": ZeroDivisionError, + "StopIteration": StopIteration, + "ArithmeticError": ArithmeticError, + "LookupError": LookupError, "re": _SafeModuleProxy(re), "math": _SafeModuleProxy(math), "itertools": _SafeModuleProxy(itertools), @@ -480,6 +485,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `Agent.get_tool(name)` | Get a tool proxy (case-insensitive, accepts `Local--` or `Server--` prefix) | | `await tool.call(**params)` | Execute a tool; returns `{"result": [...], "errors": [...], "details": [...]}` — each result item is `{"content": ..., "_": {...}}` | | `Agent.peek(result)` | Inspect a tool result's structure and leaf content — returns a string; use `print(Agent.peek(result))` to see it | +| `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | | `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | | `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index f3d406efa2a..c8807fffac0 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -468,6 +468,7 @@ def _escape_newlines_in_strings(code: str) -> str: "collections", "datetime", "traceback", + "json", } ) @@ -476,7 +477,7 @@ def _strip_allowed_imports(code: str) -> tuple[str, dict[str, object]]: """Strip import lines for modules already pre-imported in the sandbox. The sandbox provides ``re``, ``math``, ``itertools``, ``collections``, - ``datetime``, and ``traceback`` as read-only proxies. + ``datetime``, ``traceback``, and ``json`` as read-only proxies. Returns ``(code, extra_globals)`` where *extra_globals* maps imported names to their resolved values. The caller must inject these into the From 7bb299ac72181b440ef4bcbfc3d50cdf9431125c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 10:02:43 -0400 Subject: [PATCH 38/68] Add more specific error message including match content for region resolver --- .../helpers/orchestration/region_resolver.py | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index 137bc6eaca8..f7b56a402c8 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -318,25 +318,43 @@ def _validate_pattern_uniqueness( else: matches_for_display = matches - line_nums = [str(i + 1) for i in sorted(matches_for_display)] - display = ", ".join(line_nums[:10]) - if len(line_nums) > 10: - display += f", ... ({len(line_nums)} total)" + # Build detailed match display with line content + max_preview = 10 + sorted_matches = sorted(matches_for_display) + all_line_nums = [str(i + 1) for i in sorted_matches] + line_nums_display = ", ".join(all_line_nums) + if len(all_line_nums) > 50: + line_nums_display = ( + ", ".join(all_line_nums[:50]) + f", ... ({len(all_line_nums)} total)" + ) + + # Only show content previews for the first 10 matches + match_lines = [] + for i in sorted_matches[:max_preview]: + line_content = lines[i] + if len(line_content) > 120: + line_content = line_content[:117] + "..." + match_lines.append(f" @L{i + 1}: {line_content}") + match_details = "\n".join(match_lines) + if len(sorted_matches) > max_preview: + match_details += f"\n ... and {len(sorted_matches) - max_preview} more match(es)" if hint is not None: raise ValueError( f"{boundary.capitalize()} pattern '{pattern}' for region " - f"'{name}' has {len(matches)} matches; @L{hint + 1} hint ties " - f"between {len(matches_for_display)} equally-close locations " - f"(lines {display}). Use a more specific pattern." + f"'{name}' has {len(matches)} matches (lines {line_nums_display}); " + f"@L{hint + 1} hint ties between " + f"{len(matches_for_display)} equally-close locations:\n" + f"{match_details}\n" + f"Use a more specific pattern." ) raise ValueError( f"{boundary.capitalize()} pattern '{pattern}' for region " - f"'{name}' matches {len(matches)} locations " - f"(lines {display}). " + f"'{name}' matches {len(matches)} locations (lines {line_nums_display}):\n" + f"{match_details}\n" f"Use a more specific pattern or append ' @L' to " - f"disambiguate (e.g., '{pattern} @L{line_nums[0]}')." + f"disambiguate (e.g., '{pattern} @L{all_line_nums[0]}')." ) @staticmethod From 07c2a23a692eb7ab5cd78d95dd98ec6beebd2c70 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 11:26:44 -0400 Subject: [PATCH 39/68] Add pathlib wrapper --- cecli/helpers/orchestration/environment.py | 3 + cecli/helpers/orchestration/safe_methods.py | 113 ++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 0c4264da2e1..83ce875f77a 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -31,6 +31,7 @@ _safe_vars, _SafeJson, _SafeModuleProxy, + _SafePathlib, _strip_allowed_imports, ) from cecli.helpers.orchestration.security import ( @@ -243,6 +244,7 @@ def __init__(self, coder: Any) -> None: "collections": _SafeModuleProxy(collections), "datetime": _SafeModuleProxy(datetime), "traceback": _SafeModuleProxy(traceback), + "pathlib": _SafePathlib, } def _allowed_methods(): @@ -512,6 +514,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `itertools` | Combinatorics: `itertools.chain(a, b)`, `itertools.product(...)` | | `collections` | Container helpers: `collections.Counter(...)`, `collections.defaultdict(...)` | | `datetime` | Date/time: `datetime.datetime.now()`, `datetime.timedelta(...)` | +| `pathlib` | Safe filesystem paths: `pathlib.Path("/tmp/foo")`, `.parent`, `.name`, `/` joining. I/O methods (``read_text``, ``write_text``, etc.) blocked | | `traceback` | Traceback formatting: `traceback.format_exc()`, `traceback.format_tb(...)` | ### Usage diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index c8807fffac0..e3eefdb1ff6 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -9,9 +9,121 @@ import asyncio import json +import pathlib from typing import Any # --------------------------------------------------------------------------- +# Safe Path wrapper +# --------------------------------------------------------------------------- + + +class _SafePath: + """Wrapper around pathlib.Path that blocks dangerous I/O methods. + + The following methods are blocked and raise SecurityError: + - ``read_text()``, ``read_bytes()`` — file reads + - ``write_text()``, ``write_bytes()`` — file writes + - ``open()`` — file handle access + + All other Path properties and methods (``.parent``, ``/`` joining, + ``.exists()``, ``.is_dir()``, ``.glob()``, etc.) work normally. + Path-like return values are automatically wrapped to maintain safety. + """ + + _BLOCKED_METHODS: frozenset[str] = frozenset( + {"read_text", "read_bytes", "write_text", "write_bytes", "open"} + ) + + def __init__(self, *args, **kwargs): + import pathlib as _pathlib + + object.__setattr__(self, "_path", _pathlib.Path(*args, **kwargs)) + + def __getattr__(self, name: str): + if name.startswith("_"): + raise AttributeError(f"Cannot access private attribute {name!r} on Path wrapper") + if name in self._BLOCKED_METHODS: + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + f"Cannot call '{name}' on Path objects in the sandbox. " + f"Filesystem I/O is restricted to the Command tool." + ) + import pathlib as _pathlib + + attr = getattr(self._path, name) + if callable(attr): + + def _wrapped(*a, **kw): + result = attr(*a, **kw) + if isinstance(result, _pathlib.PurePath): + return _SafePath._from_path(result) + return result + + return _wrapped + if isinstance(attr, _pathlib.PurePath): + return _SafePath._from_path(attr) + return attr + + def __truediv__(self, other): + return _SafePath._from_path(self._path / other) + + def __rtruediv__(self, other): + import pathlib as _pathlib + + return _SafePath._from_path(_pathlib.Path(other) / self._path) + + def __repr__(self): + return repr(self._path) + + def __str__(self): + return str(self._path) + + def __fspath__(self): + import os + + return os.fspath(self._path) + + def __eq__(self, other): + if isinstance(other, _SafePath): + return self._path == other._path + return self._path == other + + def __hash__(self): + return hash(self._path) + + @staticmethod + def _from_path(p): + sp = object.__new__(_SafePath) + object.__setattr__(sp, "_path", p) + return sp + + +class _SafePathlib: + """Drop-in ``pathlib`` module proxy with Path I/O methods blocked. + + Usage in sandbox code:: + + p = pathlib.Path("/tmp/foo") + parent = p.parent # works, returns wrapped Path + name = p.name # works + content = p.read_text() # raises SecurityError + + ``pathlib.Path()`` returns ``_SafePath`` wrappers. Pure path classes + (``PurePath``, ``PurePosixPath``, ``PureWindowsPath``) are safe to pass + through directly — they have no I/O methods. Concrete ``PosixPath`` and + ``WindowsPath`` are intentionally excluded; use ``Path()`` instead. + """ + + PurePath = pathlib.PurePath + PurePosixPath = pathlib.PurePosixPath + PureWindowsPath = pathlib.PureWindowsPath + + @staticmethod + def Path(*args, **kwargs): + return _SafePath(*args, **kwargs) + + # Safe primitives exposed to the sandbox # --------------------------------------------------------------------------- @@ -469,6 +581,7 @@ def _escape_newlines_in_strings(code: str) -> str: "datetime", "traceback", "json", + "pathlib", } ) From 9bef7252cdae4be924c866dca446e21cd502a4ed Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 11:38:57 -0400 Subject: [PATCH 40/68] state.get() can check both local and global stored items --- cecli/helpers/orchestration/environment.py | 23 +++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 83ce875f77a..a3e9614a7e1 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -132,6 +132,27 @@ def popitem(self): self._owner._record_mutation(key, is_shared=self._is_shared) return super().popitem() + def get(self, key, default=None): + """Return value for *key*, falling through to ``shared_state`` if not found locally. + + When called on a local state dict (i.e., ``state``, not ``shared_state`` + itself), keys not present locally are looked up in the shared state + before returning *default*. This makes it easy to read globally-set + values without an explicit fallback:: + + val = state.get("my_global") # checks local, then shared_state + """ + if key in self: + return self[key] + # Don't fall through when this IS the shared_state (avoid infinite loop) + if self._is_shared: + return default + if self._owner is not None: + shared = type(self._owner)._shared_state + if key in shared: + return shared[key] + return default + # --------------------------------------------------------------------------- # Safe primitives exposed to the sandbox @@ -495,7 +516,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `gather(**named_tasks)` | Run tasks concurrently; returns an iterable with `.key` / `["key"]` access | -| `state` / `shared_state` | `state` persists across *all* `Orchestrate` calls within the same agent session (not just one call). `shared_state` persists across *all* agent sessions globally | +| `state` / `shared_state` | `state` persists across *all* `Orchestrate` calls within the same agent session (not just one call). `state.get(key)` falls through to ``shared_state`` when the key is not local. `shared_state` persists across *all* agent sessions globally | | `json.loads(s)` / `json.dumps(obj, indent=..., sort_keys=...)` | Parse / serialize JSON with optional formatting | | `sleep(seconds)` | Pause execution (0-120s max) | | `print(...)` / `reset(local_vars=True, state=False)` | Output messages; clear local namespace (and optionally state) | From 06e703df68750b5d314105df3d2b123821368683 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 11:59:48 -0400 Subject: [PATCH 41/68] Centralize Agent utilities in orchestrate, allow brackets in nested getter strings --- cecli/helpers/nested.py | 15 +++++- cecli/helpers/orchestration/agent_proxy.py | 46 +++++++++++++++++++ cecli/helpers/orchestration/environment.py | 53 +++++++--------------- tests/helpers/test_orchestration.py | 4 +- 4 files changed, 78 insertions(+), 40 deletions(-) diff --git a/cecli/helpers/nested.py b/cecli/helpers/nested.py index c77be72b717..3f70302f0e6 100644 --- a/cecli/helpers/nested.py +++ b/cecli/helpers/nested.py @@ -1,5 +1,10 @@ +import re from typing import Any, Dict, List, Union +# Precompiled regex for normalizing bracket notation to dot notation in paths +# e.g., "result[0]._.file_path" -> "result.0._.file_path" +_BRACKET_RE = re.compile(r"\[(\d+)\]") + def arg_resolver(obj: Union[List[Any], Dict[str, Any], Any], key: str, default: Any = None) -> Any: """ @@ -54,7 +59,12 @@ def arg_resolver(obj: Union[List[Any], Dict[str, Any], Any], key: str, default: def getter( data: Union[List[Any], Dict[str, Any], Any], path: Union[str, List[str]], default: Any = None ) -> Any: - """Safely access nested dicts, lists, and objects using normalized dot-notation.""" + """Safely access nested dicts, lists, and objects using normalized dot-notation. + + Supports both: + getter(data, "result.0._.file_path") # dot notation + getter(data, "result[0]._.file_path") # bracket notation + """ if data is None: return default @@ -65,6 +75,9 @@ def getter( else: paths = path + # Normalize bracket notation to dot notation + paths = [_BRACKET_RE.sub(r".\1", p) for p in paths] + # Try each path, return first valid result for path_str in paths: current = data diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index b6763152efe..74d63cde862 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -33,6 +33,7 @@ class AgentProxy: def __init__(self, coder: Any) -> None: self._coder = coder + self._env = None # Set by AgentExecutionEnv after creation def get_tool(self, tool_name: str) -> ToolProxy: from cecli.tools.utils.registry import ToolRegistry @@ -425,6 +426,51 @@ async def edit_region( change_id=change_id, ) + async def sleep(self, seconds: float) -> None: + """Safe sleep - pauses execution (0-120 seconds max). + + Usage:: + + await Agent.sleep(1) # pause for 1 second + """ + from cecli.helpers.orchestration.safe_methods import _safe_sleep + + await _safe_sleep(seconds) + + def allowed_tools(self) -> list[str]: + """Return a sorted list of available tool names. + + Usage:: + + tools = Agent.allowed_tools() + """ + from cecli.helpers import nested + + tool_names = [] + tool_list = self._coder.get_tool_list() + for tool in tool_list: + name = nested.getter(tool, "function.name", "") + if name: + tool_names.append(name) + return sorted(tool_names) + + def allowed_methods(self) -> list[str]: + """Return a sorted list of all available functions and objects in the sandbox. + + Usage:: + + methods = Agent.allowed_methods() + """ + if self._env is None: + return [] + builtins = sorted(k for k in self._env._safe_builtins.keys() if not k.startswith("_")) + globals_list = sorted( + k + for k in self._env.globals.keys() + if not k.startswith("__") and k not in ("__builtins__", "NEWLINE") + ) + return builtins + globals_list + @staticmethod def _validate_edit_regions( edits: list[dict[str, object]], diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index a3e9614a7e1..6f01b1160ae 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -186,10 +186,9 @@ class AgentExecutionEnv: Sandboxed REPL environment for executing LLM-generated orchestration code. Provides: - - `Agent` : proxy to look up and call tools + - `Agent` : proxy to look up and call tools and other core agentic utility methods - `gather` : safe parallel execution helper - `state` : persistent dict (survives across Orchestrate calls) - - `sleep` : safe sleep (0-120 seconds) - `print` : captured output - `range`, `len`, `int`, `str`, `list`, `dict`, `bool`, `Exception` @@ -212,7 +211,7 @@ def __init__(self, coder: Any) -> None: self.globals: dict[str, Any] = {} self.locals: dict[str, Any] = {} - _safe_builtins: dict[str, Any] = { + self._safe_builtins: dict[str, Any] = { "print": print, "range": range, "len": len, @@ -268,43 +267,21 @@ def __init__(self, coder: Any) -> None: "pathlib": _SafePathlib, } - def _allowed_methods(): - """Return a sorted list of all available functions and objects in the sandbox.""" - builtins = sorted(k for k in _safe_builtins.keys() if not k.startswith("__")) - globals_list = sorted( - k - for k in self.globals.keys() - if not k.startswith("__") and k not in ("__builtins__", "NEWLINE") - ) - return builtins + globals_list - - def _allowed_tools(): - """Return a sorted list of available tool names for use with Agent.get_tool().""" - from cecli.helpers import nested - - tool_names = [] - tool_list = coder.get_tool_list() - for tool in tool_list: - name = nested.getter(tool, "function.name", "") - if name: - tool_names.append(name) - return sorted(tool_names) - self.globals.clear() + _agent = AgentProxy(coder) + _agent._env = self + self.globals.update( { - "__builtins__": _HelpfulBuiltins(_safe_builtins), - "Agent": AgentProxy(coder), + "__builtins__": _HelpfulBuiltins(self._safe_builtins), + "Agent": _agent, "gather": _safe_gather, - "sleep": _safe_sleep, "json": _SafeJson, "state": self.state, "shared_state": AgentExecutionEnv._shared_state, "__yield": _cooperative_yield, "__security_raise": _security_raise, "NEWLINE": "\n", - "allowed_methods": _allowed_methods, - "allowed_tools": _allowed_tools, } ) self.locals.clear() @@ -499,14 +476,18 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. This is much more efficient than making individual tool calls for loop-heavy workflows. Variables and methods defined in a script are persisted in subsequent turns. -As such, results from previous calls can be reused and helper methods can be defined to enhance usage of the environment. +As such, results from previous calls can be reused and helper methods can be defined to enhance ease of use within the environment. ### Primitives | Primitive | Description | |-----------|-------------| -| `Agent.get_tool(name)` | Get a tool proxy (case-insensitive, accepts `Local--` or `Server--` prefix) | +| `Agent.allowed_methods()` | List all available builtin function names | +| `Agent.allowed_tools()` | List all available tool names | + +| `Agent.get_tool(name)` | Get a tool proxy (case-insensitive, accepts `Local--` or `{{Server}}--` prefix) | | `await tool.call(**params)` | Execute a tool; returns `{"result": [...], "errors": [...], "details": [...]}` — each result item is `{"content": ..., "_": {...}}` | + | `Agent.peek(result)` | Inspect a tool result's structure and leaf content — returns a string; use `print(Agent.peek(result))` to see it | | `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | @@ -514,28 +495,26 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | | `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | +| `await Agent.sleep(seconds)` | Pause execution (0-120s max) | | `gather(**named_tasks)` | Run tasks concurrently; returns an iterable with `.key` / `["key"]` access | | `state` / `shared_state` | `state` persists across *all* `Orchestrate` calls within the same agent session (not just one call). `state.get(key)` falls through to ``shared_state`` when the key is not local. `shared_state` persists across *all* agent sessions globally | -| `json.loads(s)` / `json.dumps(obj, indent=..., sort_keys=...)` | Parse / serialize JSON with optional formatting | -| `sleep(seconds)` | Pause execution (0-120s max) | | `print(...)` / `reset(local_vars=True, state=False)` | Output messages; clear local namespace (and optionally state) | | `typeof(x)` / `isinstance(x, t)` / `hasattr(x, n)` / `repr(x)` / `vars(obj)` | Type inspection and debugging | -| `allowed_methods()` | List all available builtin function names | -| `allowed_tools()` | List all available tool names for use with ``Agent.get_tool()`` | ### Available Modules Pre-imported, read-only standard library modules: | Module | Common uses | -|--------|------------| +|--------|-------------| | `re` | Regular expressions: `re.search(r"pat", s)`, `re.findall(...)` | | `math` | Math functions: `math.ceil(n)`, `math.sqrt(n)` | | `itertools` | Combinatorics: `itertools.chain(a, b)`, `itertools.product(...)` | | `collections` | Container helpers: `collections.Counter(...)`, `collections.defaultdict(...)` | | `datetime` | Date/time: `datetime.datetime.now()`, `datetime.timedelta(...)` | | `pathlib` | Safe filesystem paths: `pathlib.Path("/tmp/foo")`, `.parent`, `.name`, `/` joining. I/O methods (``read_text``, ``write_text``, etc.) blocked | +| `json` | Parse / serialize: `json.loads(s)`, `json.dumps(obj, indent=2)` | | `traceback` | Traceback formatting: `traceback.format_exc()`, `traceback.format_tb(...)` | ### Usage diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 76faf5cce29..7366ab1d5bb 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -498,10 +498,10 @@ async def test_env_whitespace_code(): @pytest.mark.asyncio async def test_env_sleep_works(): - """AgentExecutionEnv's sleep primitive works.""" + """AgentExecutionEnv's Agent.sleep primitive works.""" env = _make_env() result = await env.execute( - "sleep(0.01)\nprint('done')", + "await Agent.sleep(0.01)\nprint('done')", ) assert result["results"] == "done", f"Expected 'done' after sleep, got: {result!r}" From 37cd1f2d1dbee1f759771da115124f61d5504539 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 12:07:48 -0400 Subject: [PATCH 42/68] Use ToolResult classes for joing property/attribute access for results --- cecli/helpers/orchestration/tool_proxy.py | 137 +++++++++++++++------- 1 file changed, 94 insertions(+), 43 deletions(-) diff --git a/cecli/helpers/orchestration/tool_proxy.py b/cecli/helpers/orchestration/tool_proxy.py index 10743f0ebcb..4eb9b098316 100644 --- a/cecli/helpers/orchestration/tool_proxy.py +++ b/cecli/helpers/orchestration/tool_proxy.py @@ -12,6 +12,41 @@ from typing import Any +class ToolResult(dict): + """A result dict that supports both key and attribute access. + + Wraps tool call results so LLM code can use either style: + + result = await tool.call(...) + print(result["result"]) # key access (dict-compatible) + print(result.result) # attribute access (convenient) + print(result.errors) # attribute access + + Since ToolResult is a dict subclass, all dict methods + (``.get()``, ``.keys()``, ``in``, etc.) work as expected. + """ + + def __getattr__(self, key: str): + if key.startswith("_"): + raise AttributeError(f"Cannot access private attribute {key!r} on ToolResult") + try: + return self[key] + except KeyError: + raise AttributeError( + f"ToolResult has no key {key!r}. " + f"Available keys: {', '.join(sorted(self.keys()))}" + ) from None + + def __setattr__(self, key: str, value) -> None: + self[key] = value + + def __delattr__(self, key: str) -> None: + try: + del self[key] + except KeyError: + raise AttributeError(f"ToolResult has no key {key!r}") from None + + class ToolProxy: """ Proxy for a single tool — local or MCP. @@ -130,21 +165,25 @@ async def call(self, **kwargs: Any): raise ValueError(f"No executor for tool '{self._tool_name}'") @staticmethod - def _normalize_result(result: Any) -> dict: - """Normalize a tool result into a unified dict with ``result``, ``errors``, ``details`` keys. + def _normalize_result(result: Any) -> "ToolResult": + """Normalize a tool result into a ToolResult with ``result``, ``errors``, ``details`` keys. Each item in the ``result`` list has the shape ``{"content": ..., "_": {...}}``. + Supports both key access (``result["result"]``) and + attribute access (``result.result``). """ from cecli.tools.utils.responses import ToolResponse if isinstance(result, ToolResponse): data = result.to_dict() - return { - "result": data.get("result", []), - "errors": data.get("errors", []), - "details": data.get("details", []), - } + return ToolResult( + { + "result": data.get("result", []), + "errors": data.get("errors", []), + "details": data.get("details", []), + } + ) if isinstance(result, str): # Attempt to auto-parse a JSON-stringified ToolResponse @@ -155,40 +194,48 @@ def _normalize_result(result: Any) -> dict: result_list = parsed["result"] if not isinstance(result_list, list): result_list = [result_list] if result_list else [] - return { - "result": [ - ( - item - if isinstance(item, dict) and "content" in item - else {"content": item, "_": {}} - ) - for item in result_list - ], - "errors": parsed.get("errors", []), - "details": parsed.get("details", []), + return ToolResult( + { + "result": [ + ( + item + if isinstance(item, dict) and "content" in item + else {"content": item, "_": {}} + ) + for item in result_list + ], + "errors": parsed.get("errors", []), + "details": parsed.get("details", []), + } + ) + return ToolResult( + { + "result": [{"content": parsed, "_": {}}], + "errors": [], + "details": [], } - return { - "result": [{"content": parsed, "_": {}}], - "errors": [], - "details": [], - } + ) except (json.JSONDecodeError, TypeError): pass # Error strings from handle_tool_error if result.startswith("Error in "): - return { - "result": [], - "errors": [result], - "details": [], - } + return ToolResult( + { + "result": [], + "errors": [result], + "details": [], + } + ) # Plain string result - return { - "result": [{"content": result, "_": {}}], - "errors": [], - "details": [], - } + return ToolResult( + { + "result": [{"content": result, "_": {}}], + "errors": [], + "details": [], + } + ) if isinstance(result, dict): if "result" in result: @@ -207,17 +254,21 @@ def _normalize_result(result: Any) -> dict: ] out.setdefault("errors", []) out.setdefault("details", []) - return out + return ToolResult(out) - return { - "result": [{"content": result, "_": {}}], + return ToolResult( + { + "result": [{"content": result, "_": {}}], + "errors": [], + "details": [], + } + ) + + # Fallback: wrap anything else + return ToolResult( + { + "result": [{"content": str(result), "_": {}}] if result is not None else [], "errors": [], "details": [], } - - # Fallback: wrap anything else - return { - "result": [{"content": str(result), "_": {}}] if result is not None else [], - "errors": [], - "details": [], - } + ) From 8ee4955328481cb2a8c71b9b997a9d7d73794cb0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 12:09:42 -0400 Subject: [PATCH 43/68] Allow line numbers to also be used to resolve locations in file --- cecli/helpers/orchestration/environment.py | 2 +- .../helpers/orchestration/region_resolver.py | 19 +++++++++++++++++++ cecli/tools/edit_file.py | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 6f01b1160ae..b45bdf19d69 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -492,7 +492,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | | `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | -| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | +| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; also supports `@L{num}` notation to reference specific lines by number. Ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | | `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | | `await Agent.sleep(seconds)` | Pause execution (0-120s max) | diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index f7b56a402c8..15191657079 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -395,6 +395,25 @@ def _resolve_pattern( if pattern in ("@000", "000@"): return pattern + # @L{num} notation — resolve to line text or content ID + import re as _re + + _m = _re.match(r"^@L(\d+)$", pattern) + if _m: + _line_num = int(_m.group(1)) - 1 + if _line_num < 0 or _line_num >= len(lines): + raise ValueError( + f"@L reference line {_m.group(1)} is out of range " + f"(file has {len(lines)} lines)" + ) + _line_text = lines[_line_num] + if lines.count(_line_text) == 1: + return _line_text # Unique — let resolve_content_to_hashline_ids handle it + # Duplicate — get content ID directly via HashPos + _occurrence = 1 + sum(1 for i in range(_line_num) if lines[i] == _line_text) + _public_id = hp.generate_public_id(_line_text, _line_num, _occurrence) + return hp.get_wrapped_id(_public_id) + if not self._looks_like_content_id(pattern): return pattern diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 8ca9a2e122c..23d4d481be0 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -290,6 +290,15 @@ def execute( f"(file has {len(source_lines)} lines)" ) + # 2.6 Strip ~~ virtual prefixes from ReadFile output lines + # These are display-only markers that confuse the resolution logic. + if isinstance(edit_start_line, str) and edit_start_line.startswith( + "~~" + ): + edit_start_line = edit_start_line.replace("~~", "").lstrip() + if isinstance(edit_end_line, str) and edit_end_line.startswith("~~"): + edit_end_line = edit_end_line.replace("~~", "").lstrip() + # 3. Resolve non-hashline content values to content IDs first # (before normalize_hashline which would fail on arbitrary content) edit_start_line, edit_end_line = resolve_content_to_hashline_ids( From abb7c7476e1e6bb9c404fce82512c2fea26a38b7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 12:25:36 -0400 Subject: [PATCH 44/68] Give the LLM nicer error messages --- cecli/helpers/orchestration/environment.py | 11 +++++++--- cecli/helpers/orchestration/security.py | 25 +++++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index b45bdf19d69..ac7bdc5adbb 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -410,9 +410,14 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: code = f"Syntax Error in orchestration code: {e}" return {"results": code, "state_variables": self._state_snapshot()} - tree = SecurityFilter().visit(tree) - tree = LoopYieldInjector().visit(tree) - ast.fix_missing_locations(tree) + try: + tree = SecurityFilter().visit(tree) + tree = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(tree) + except SecurityError as e: + return _build_result(f"Security Error: {e}") + except Exception as e: + return _build_result(f"AST Transform Error: {e}") wrapper_func = ast.AsyncFunctionDef( name="__agent_async_runner", diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py index 7ecb0ec4be5..58c5775f18b 100644 --- a/cecli/helpers/orchestration/security.py +++ b/cecli/helpers/orchestration/security.py @@ -90,18 +90,26 @@ def _make_raise_stmt(message: str) -> ast.Expr: # ------------------------------------------------------------------ def visit_Import(self, node: ast.Import) -> ast.Expr: - return self._make_raise_stmt("Imports are disabled in the agent orchestration environment.") + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "Imports are disabled in the agent orchestration environment." + ) def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.Expr: - return self._make_raise_stmt("Imports are disabled in the agent orchestration environment.") + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "Imports are disabled in the agent orchestration environment." + ) def visit_Global(self, node: ast.Global) -> ast.Expr: return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " "The 'global' statement is disabled in the orchestration environment." ) def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.Expr: return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " "The 'nonlocal' statement is disabled in the orchestration environment." ) @@ -114,7 +122,15 @@ def visit_Attribute(self, node: ast.Attribute): if node.attr in self._SAFE_DUNDER: return self.generic_visit(node) + if isinstance(node.ctx, (ast.Store, ast.Del)): + verb = "assign to" if isinstance(node.ctx, ast.Store) else "delete" + raise SecurityError( + f"Security filter error at line {node.lineno}: " + f"cannot {verb} private attribute '{node.attr}'" + ) + return self._make_raise_expr( + f"Security filter error at line {node.lineno}: " f"Access to private/dunder attribute '{node.attr}' is forbidden." ) @@ -122,7 +138,10 @@ def visit_Attribute(self, node: ast.Attribute): def visit_Call(self, node: ast.Call): if isinstance(node.func, ast.Name) and node.func.id in self._DANGEROUS_BUILTINS: - return self._make_raise_expr(f"Calling '{node.func.id}' is forbidden.") + return self._make_raise_expr( + f"Security filter error at line {node.lineno}: " + f"Calling '{node.func.id}' is forbidden." + ) return self.generic_visit(node) From a7e91b69a65e7189b813f0e86f19aaba0d944f7e Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 12:35:11 -0400 Subject: [PATCH 45/68] Use json escaping for safe string construction --- cecli/helpers/orchestration/safe_methods.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index e3eefdb1ff6..17bc30cbe0b 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -380,7 +380,7 @@ def _naive_escape_newlines(code: str, NL: str, BSN: str) -> str: idx = len(code) break inner = code[idx:end] - inner = inner.replace(NL, BSN) + inner = json.dumps(inner, ensure_ascii=False)[1:-1] result.append(inner) result.append(quote) idx = end + len(quote) @@ -433,7 +433,7 @@ def _process_fstring_body(code: str, idx: int, quote: str, NL: str, BSN: str, re if brace_depth == 0: # Flush the string-literal portion up to { inner = code[literal_start:pos] - inner = inner.replace(NL, BSN) + inner = json.dumps(inner, ensure_ascii=False)[1:-1] result.append(inner) result.append(c) literal_start = pos + 1 @@ -471,7 +471,7 @@ def _process_fstring_body(code: str, idx: int, quote: str, NL: str, BSN: str, re # Flush the final string-literal portion if literal_start < pos: inner = code[literal_start:pos] - inner = inner.replace(NL, BSN) + inner = json.dumps(inner, ensure_ascii=False)[1:-1] result.append(inner) if pos < len(code): @@ -514,7 +514,7 @@ def _fstring_aware_escape_newlines(code: str, NL: str, BSN: str) -> str: idx = len(code) break inner = code[idx:end] - inner = inner.replace(NL, BSN) + inner = json.dumps(inner, ensure_ascii=False)[1:-1] result.append(inner) result.append(quote) idx = end + len(quote) From cc34dfc997ee52ffd4fe0f1d792d8ab6e16997e6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 13:26:26 -0400 Subject: [PATCH 46/68] Simplify edit_text range parsing --- cecli/tools/edit_file.py | 138 ++++++++++++---------------- tests/helpers/test_orchestration.py | 40 +++++++- tests/tools/test_insert_block.py | 4 +- 3 files changed, 100 insertions(+), 82 deletions(-) diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 23d4d481be0..424b64cdb6f 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -1,15 +1,13 @@ from cecli.helpers.hashline import ( HASH_DELIMITER, - HASHLINE_PREFIX_RE, UNIQUE_HASH_DELIMITER, ContentHashError, apply_hashline_operations, get_hashline_diff, - hashline, - normalize_hashline, resolve_content_to_hashline_ids, strip_hashline, ) +from cecli.helpers.hashpos.hashpos import HashPos from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, @@ -202,6 +200,18 @@ def execute( coder, file_path_key ) + # Build HashPos index once per file for @L{num} resolution + hp = ( + HashPos(original_content) + if original_content and original_content.strip() + else None + ) + source_lines = ( + original_content.splitlines() + if original_content and original_content.strip() + else [] + ) + # Process all edits for this file using batch operations operations = [] file_metadata = [] @@ -234,83 +244,22 @@ def execute( edit_start_line = "@000" edit_end_line = "@000" - # 2.5 Resolve @L{num} notation to line content or hash ID - if ( - isinstance(edit_start_line, str) - and edit_start_line.startswith("@L") - and len(edit_start_line) > 2 - and edit_start_line[2:].isdigit() - ): - if original_content and original_content.strip(): - source_lines = original_content.splitlines() - line_num = ( - int(edit_start_line[2:]) - 1 - ) # 1-indexed to 0-indexed - if 0 <= line_num < len(source_lines): - line_text = source_lines[line_num] - if source_lines.count(line_text) == 1: - # Unique line — let resolve_content_to_hashline_ids handle it - edit_start_line = line_text - else: - # Duplicate line — resolve directly to content ID - hashed_content = hashline(original_content) - hashed_lines = hashed_content.splitlines() - match = HASHLINE_PREFIX_RE.match(hashed_lines[line_num]) - if match: - edit_start_line = match.group(1) - else: - raise ToolError( - f"@L reference line {int(edit_start_line[2:])} is out of range " - f"(file has {len(source_lines)} lines)" - ) - if ( - isinstance(edit_end_line, str) - and edit_end_line.startswith("@L") - and len(edit_end_line) > 2 - and edit_end_line[2:].isdigit() - ): - if original_content and original_content.strip(): - source_lines = original_content.splitlines() - line_num = int(edit_end_line[2:]) - 1 - if 0 <= line_num < len(source_lines): - line_text = source_lines[line_num] - if source_lines.count(line_text) == 1: - # Unique line — let resolve_content_to_hashline_ids handle it - edit_end_line = line_text - else: - # Duplicate line — resolve directly to content ID - hashed_content = hashline(original_content) - hashed_lines = hashed_content.splitlines() - match = HASHLINE_PREFIX_RE.match(hashed_lines[line_num]) - if match: - edit_end_line = match.group(1) - else: - raise ToolError( - f"@L reference line {int(edit_end_line[2:])} is out of range " - f"(file has {len(source_lines)} lines)" - ) - - # 2.6 Strip ~~ virtual prefixes from ReadFile output lines - # These are display-only markers that confuse the resolution logic. - if isinstance(edit_start_line, str) and edit_start_line.startswith( - "~~" - ): - edit_start_line = edit_start_line.replace("~~", "").lstrip() - if isinstance(edit_end_line, str) and edit_end_line.startswith("~~"): - edit_end_line = edit_end_line.replace("~~", "").lstrip() - - # 3. Resolve non-hashline content values to content IDs first - # (before normalize_hashline which would fail on arbitrary content) + # Resolve @L{num} notation directly to content ID + edit_start_line = cls._resolve_at_l_num( + edit_start_line, hp, source_lines, file_path_key + ) + edit_end_line = cls._resolve_at_l_num( + edit_end_line, hp, source_lines, file_path_key + ) + + # Strip ~~ virtual prefixes from ReadFile output lines + edit_start_line = cls._strip_readfile_prefix(edit_start_line) + edit_end_line = cls._strip_readfile_prefix(edit_end_line) + + # Resolve remaining non-hashline content values to content IDs edit_start_line, edit_end_line = resolve_content_to_hashline_ids( original_content, edit_start_line, edit_end_line ) - # 4. Auto-sanitize malformed boundaries (strip accidentally appended code) - if isinstance(edit_start_line, str): - test_line = normalize_hashline(edit_start_line, throw=False) - edit_start_line = test_line if test_line else edit_start_line - if isinstance(edit_end_line, str): - test_line = normalize_hashline(edit_end_line, throw=False) - edit_end_line = test_line if test_line else edit_end_line # --------------------------------------------------------- @@ -628,6 +577,41 @@ def format_output(cls, coder, mcp_server, tool_response): tool_footer(coder=coder, tool_response=tool_response, params=params) + @classmethod + def _resolve_at_l_num(cls, line_spec, hp, source_lines, file_path): + """Resolve @L{num} notation to a content ID using a pre-built HashPos index. + + Returns the input unchanged if it's not an @L{num} spec. + Raises ToolError if the line number is out of range. + """ + if not ( + isinstance(line_spec, str) + and line_spec.startswith("@L") + and len(line_spec) > 2 + and line_spec[2:].isdigit() + ): + return line_spec + + line_num = int(line_spec[2:]) - 1 + if line_num < 0 or line_num >= len(source_lines): + from cecli.tools.utils.helpers import ToolError + + raise ToolError( + f"@L reference line {int(line_spec[2:])} is out of range " + f"(file has {len(source_lines)} lines)" + ) + + line_text = source_lines[line_num] + occurrence = 1 + sum(1 for i in range(line_num) if source_lines[i] == line_text) + return hp.get_wrapped_id(hp.generate_public_id(line_text, line_num, occurrence)) + + @staticmethod + def _strip_readfile_prefix(value): + """Strip the ``~~`` virtual prefix from a ReadFile output line reference.""" + if isinstance(value, str) and value.startswith("~~"): + return value[2:].lstrip() + return value + @classmethod def _categorize_edit_error(cls, error_msg: str) -> str: """Categorize an edit error message into a user-friendly display category.""" diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index 7366ab1d5bb..d1f8353f9a1 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -20,6 +20,7 @@ LoopYieldInjector, SecurityError, SecurityFilter, + _security_raise, ) # --------------------------------------------------------------------------- @@ -38,9 +39,42 @@ class _MockCoder: def _run_security_filter_on(code: str) -> None: - """Run the SecurityFilter on a code snippet. Raises SecurityError if blocked.""" - tree = ast.parse(code, mode="exec") - SecurityFilter().visit(tree) + """Run the SecurityFilter and execute the rewritten AST. Raises SecurityError if blocked. + + Only executes code that was actually rewritten by the SecurityFilter + (i.e., contains ``__security_raise`` calls). Safe code that passes + through the filter unmodified is not executed, avoiding NameError/ + TypeError issues from test-scope variable names in the namespace. + """ + import copy + + original_tree = ast.parse(code, mode="exec") + original_tree = copy.deepcopy(original_tree) + rewritten_tree = SecurityFilter().visit(copy.deepcopy(original_tree)) + ast.fix_missing_locations(rewritten_tree) + + # Compare AST dumps to detect whether the filter rewrote anything. + # We deep-copy because ast.NodeTransformer.visit() mutates in place. + original_dump = ast.dump(original_tree, indent=0) + rewritten_dump = ast.dump(rewritten_tree, indent=0) + if original_dump == rewritten_dump: + return + + # Compile with top-level await support (needed for snippets like + # ``await gather()`` which may appear in user code). + compiled_code = compile( + rewritten_tree, + "", + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + + # Provide a minimal namespace with ``__security_raise`` so that + # rewritten constructs actually raise SecurityError at runtime. + ns: dict = { + "__security_raise": _security_raise, + } + exec(compiled_code, ns) def _run_security_filter_safe(code: str) -> bool: diff --git a/tests/tools/test_insert_block.py b/tests/tools/test_insert_block.py index 9478e5ac7f9..340f34f914e 100644 --- a/tests/tools/test_insert_block.py +++ b/tests/tools/test_insert_block.py @@ -97,7 +97,7 @@ def test_position_top_succeeds_with_no_patterns(coder_with_file): ], ) - assert "Applied" in result.to_dict()["result"][0] + assert "Applied" in result.to_dict()["result"][0]["content"] lines = file_path.read_text().splitlines() # Inserted line replaces first line (inclusive bounds) assert lines[1] == "second line" # Original second line shifts up @@ -221,7 +221,7 @@ def test_line_number_beyond_file_length_appends(coder_with_file): ], ) - assert "Applied" in result.to_dict()["result"][0] + assert "Applied" in result.to_dict()["result"][0]["content"] content = file_path.read_text() assert content == "first line\nappended line\n" coder.io.tool_error.assert_not_called() From ec1eac244eb9dd48ae61e79d950b9435f51ffb9a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 15:49:52 -0400 Subject: [PATCH 47/68] Allow before/after {second pattern} specification --- cecli/helpers/orchestration/environment.py | 2 +- .../helpers/orchestration/region_resolver.py | 113 +++++++++++++++--- cecli/tools/read_file.py | 82 +++++++++++-- 3 files changed, 170 insertions(+), 27 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index ac7bdc5adbb..1b133932b16 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -497,7 +497,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | | `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | -| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs; also supports `@L{num}` notation to reference specific lines by number. Ambiguous patterns raise immediately with clear error messages. Use `start_line_hint` / `end_line_hint` to disambiguate. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | +| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs. Use `start_line_hint` / `end_line_hint` in region spec entries for disambiguation (supports `@L`, `@A{{regex}}`, `@B{{regex}}` — same hint syntax as ReadFile). Ambiguous patterns raise immediately. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | | `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | | `await Agent.sleep(seconds)` | Pause execution (0-120s max) | diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index 15191657079..02edbac3a3f 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -183,10 +183,41 @@ def _resolve(self, name: str) -> tuple[str, str, int, int]: # Always strip @L hints from patterns — they are metadata, not literal text. # Explicit hints (start_line_hint / end_line_hint) override any @L in patterns. - start_pattern, extracted_start = self._extract_l_hint(start_pattern) - end_pattern, extracted_end = self._extract_l_hint(end_pattern) - start_hint = (explicit_start - 1) if explicit_start is not None else extracted_start - end_hint = (explicit_end - 1) if explicit_end is not None else extracted_end + start_pattern, extracted_start, start_hint_type = self._extract_l_hint(start_pattern, lines) + end_pattern, extracted_end, end_hint_type = self._extract_l_hint(end_pattern, lines) + + # Handle explicit hints (start_line_hint / end_line_hint) + # Integers are treated as @L (1-based line numbers). + # Strings support the full @L, @A, @B syntax (same as ReadFile). + if explicit_start is not None: + if isinstance(explicit_start, str): + # String hint — parse through _extract_l_hint + _, start_hint, start_hint_type = self._extract_l_hint(explicit_start, lines) + if start_hint is None: + raise ValueError( + f"start_line_hint '{explicit_start}' for region " + f"'{name}' could not be resolved" + ) + else: + # Integer hint — treat as @L (1-based, converted to 0-based) + start_hint = explicit_start - 1 + start_hint_type = "L" + else: + start_hint = extracted_start + + if explicit_end is not None: + if isinstance(explicit_end, str): + _, end_hint, end_hint_type = self._extract_l_hint(explicit_end, lines) + if end_hint is None: + raise ValueError( + f"end_line_hint '{explicit_end}' for region " + f"'{name}' could not be resolved" + ) + else: + end_hint = explicit_end - 1 + end_hint_type = "L" + else: + end_hint = extracted_end # Strip hashline prefixes from text patterns — the LLM may have copied # content-ID-prefixed lines from a ReadFile response (e.g. ~XYZ12::text). @@ -197,8 +228,10 @@ def _resolve(self, name: str) -> tuple[str, str, int, int]: end_pattern = HashPos.strip_prefix(end_pattern) # Validate uniqueness for text-based patterns (not content IDs or special markers). - self._validate_pattern_uniqueness(start_pattern, start_hint, "start", name, lines) - self._validate_pattern_uniqueness(end_pattern, end_hint, "end", name, lines) + self._validate_pattern_uniqueness( + start_pattern, start_hint, "start", name, lines, start_hint_type + ) + self._validate_pattern_uniqueness(end_pattern, end_hint, "end", name, lines, end_hint_type) start_id, end_id = resolve_content_to_hashline_ids(content, start_pattern, end_pattern) @@ -261,18 +294,58 @@ def _search_in_lines(lines: list[str], pattern: str) -> list[int]: return indices @staticmethod - def _extract_l_hint(pattern: str) -> tuple[str, int | None]: - """Extract an @L line hint suffix from a pattern. - - Returns (stripped_pattern, 0_based_line_number_or_None). + def _extract_l_hint( + pattern: str, lines: list[str] | None = None + ) -> tuple[str, int | None, str | None]: + """Extract a hint suffix from a pattern string. + + Supports @L (direct line number), @A{{regex}} (filter to matches AFTER + the first regex match), and @B{{regex}} (filter to matches BEFORE the last + regex match) hints. + + Returns (stripped_pattern, hint_value, hint_type) where: + - hint_value is a 0-based line number or None + - hint_type is 'L', 'A', 'B', or None """ - import re + # Try @L hint (direct line number - always resolvable) m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) if m: - return pattern[: m.start()], int(m.group(1)) - 1 - return pattern, None + return pattern[: m.start()], int(m.group(1)) - 1, "L" + + # Try @A{{regex}} hint (first regex match — filter to lines AFTER) + m = re.search(r"[ \t]+@A\{\{(.+?)\}\}[ \t]*$", pattern) + if m: + stripped = pattern[: m.start()] + if lines is not None: + regex_str = m.group(1) + try: + for i, line in enumerate(lines): + if re.search(regex_str, line): + return stripped, i, "A" + except re.error: + pass + return stripped, None, None + + # Try @B{{regex}} hint (last regex match — filter to lines BEFORE) + m = re.search(r"[ \t]+@B\{\{(.+?)\}\}[ \t]*$", pattern) + if m: + stripped = pattern[: m.start()] + if lines is not None: + regex_str = m.group(1) + try: + last_match = None + for i, line in enumerate(lines): + if re.search(regex_str, line): + last_match = i + if last_match is not None: + return stripped, last_match, "B" + except re.error: + pass + return stripped, None, None + + return pattern, None, None @staticmethod def _narrow_by_proximity(indices: list[int], target: int, max_results: int = 5) -> list[int]: @@ -291,6 +364,7 @@ def _validate_pattern_uniqueness( boundary: str, name: str, lines: list[str], + hint_type: str | None = None, ) -> None: """Raise ValueError if *pattern* matches multiple locations. @@ -304,11 +378,20 @@ def _validate_pattern_uniqueness( return matches = self._search_in_lines(lines, pattern) + + # Apply @A/@B directional filtering — keep only closest match in direction + if hint_type == "A" and hint is not None: + after = [m for m in matches if m > hint] + matches = [min(after)] if after else [] + elif hint_type == "B" and hint is not None: + before = [m for m in matches if m < hint] + matches = [max(before)] if before else [] + if len(matches) <= 1: return - # Try @L hint narrowing — find the unique closest match - if hint is not None: + # Try proximity narrowing — find the unique closest match (only for @L hints) + if hint is not None and hint_type == "L": best = min(matches, key=lambda i: abs(i - hint)) best_dist = abs(best - hint) conflicts = [i for i in matches if i != best and abs(i - hint) == best_dist] diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index e9a12aeda40..1bcb00c781a 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -34,7 +34,9 @@ class Tool(BaseTool): "file boundaries (@000, 000@), or exact line numbers (@L10). " "Contextual end markers (@C{num}, @P{num}, @N{num}) expand symmetrically, previously, or next " "around a single unique range_start match." - " Use line hints (e.g., 'my_func @L150') to disambiguate text matches. " + " Use line hints (e.g., 'my_func @L150', '@A{{def foo}}', '@B{{return x}}') to disambiguate. " + "@A{{regex}} keeps only the closest match **after** the regex hit; " + "@B{{regex}} keeps only the closest match **before** the regex hit. " "" "File lines are prefixed with virtual, deterministic identifiers generated on-the-fly: " "Because identifiers track global occurrences, " @@ -182,8 +184,6 @@ def execute(cls, coder, read, **kwargs): start_hint = None end_hint = None - range_start, start_hint = cls._extract_l_hint(range_start) - range_end, end_hint = cls._extract_l_hint(range_end) # 2. Resolve path abs_path, rel_path = resolve_paths(coder, file_path) @@ -220,6 +220,10 @@ def execute(cls, coder, read, **kwargs): lines = content.splitlines() num_lines = len(lines) + # Resolve hints after file content available (@A/@B regex hints need lines) + range_start, start_hint, start_hint_type = cls._extract_l_hint(range_start, lines) + range_end, end_hint, end_hint_type = cls._extract_l_hint(range_end, lines) + if num_lines == 0: new_context_details.append( { @@ -251,6 +255,27 @@ def execute(cls, coder, read, **kwargs): start_indices = cls._find_start_indices(lines, range_start, rt, num_lines) end_indices = cls._find_end_indices(lines, range_end, rt, num_lines) + # Step 2a: Apply @A/@B directional filtering — keep only closest match in direction + if start_hint_type == "A" and start_hint is not None: + after = [i for i in start_indices if i > start_hint] + start_indices = [min(after)] if after else [] + start_hint = None + elif start_hint_type == "B" and start_hint is not None: + before = [i for i in start_indices if i < start_hint] + start_indices = [max(before)] if before else [] + start_hint = None + + if end_hint_type == "A" and end_hint is not None: + after = [i for i in end_indices if i > end_hint] + end_indices = [min(after)] if after else [] + end_hint = None + elif end_hint_type == "B" and end_hint is not None: + before = [i for i in end_indices if i < end_hint] + end_indices = [max(before)] if before else [] + end_hint = None + end_indices = [i for i in end_indices if i < end_hint] + end_hint = None + # Step 3: Apply contextual marker (@C/@P/@N) if rt["end_is_contextual"]: result, ctx_error = cls._apply_contextual_marker( @@ -1085,21 +1110,56 @@ def _is_line_ref(s): } @classmethod - def _extract_l_hint(cls, pattern): - """Extract @L line hint suffix from a pattern string. + def _extract_l_hint(cls, pattern, lines=None): + """Extract hint suffix from a pattern string. - Supports patterns like 'my_function @L1506' where the @L suffix - provides a proximity hint for disambiguation. + Supports @L (direct line number), @A{{regex}} (filter to matches AFTER + the first regex match), and @B{{regex}} (filter to matches BEFORE the last + regex match) hints. - Returns (stripped_pattern, line_hint) where line_hint is a 0-based - line number or None if no @L suffix is found. + Returns (stripped_pattern, hint_value, hint_type) where: + - hint_value is a 0-based line number or None + - hint_type is 'L', 'A', 'B', or None """ import re + # Try @L hint (direct line number - always resolvable) m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) if m: - return pattern[: m.start()], int(m.group(1)) - 1 - return pattern, None + return pattern[: m.start()], int(m.group(1)) - 1, "L" + + # Try @A{{regex}} hint (first regex match — filter to lines AFTER) + m = re.search(r"[ \t]+@A\{\{(.+?)\}\}[ \t]*$", pattern) + if m: + stripped = pattern[: m.start()] + if lines is not None: + regex_str = m.group(1) + try: + for i, line in enumerate(lines): + if re.search(regex_str, line): + return stripped, i, "A" + except re.error: + pass + return stripped, None, None + + # Try @B{{regex}} hint (last regex match — filter to lines BEFORE) + m = re.search(r"[ \t]+@B\{\{(.+?)\}\}[ \t]*$", pattern) + if m: + stripped = pattern[: m.start()] + if lines is not None: + regex_str = m.group(1) + try: + last_match = None + for i, line in enumerate(lines): + if re.search(regex_str, line): + last_match = i + if last_match is not None: + return stripped, last_match, "B" + except re.error: + pass + return stripped, None, None + + return pattern, None, None @classmethod def _search_in_lines(cls, lines, pattern, return_last_line=False): From c4dab5dadfb47841c80e7b1eb622eb8a9cb7bbf2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 16:13:47 -0400 Subject: [PATCH 48/68] Allow patterns that start with @ to also recieve line hints --- cecli/helpers/hashline.py | 17 +++++++++++++++++ cecli/helpers/orchestration/region_resolver.py | 7 ++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/hashline.py b/cecli/helpers/hashline.py index 55d0815b98b..388abb0562e 100644 --- a/cecli/helpers/hashline.py +++ b/cecli/helpers/hashline.py @@ -279,6 +279,7 @@ def resolve_content_to_hashline_ids( original_content: str, start_value: str, end_value: str = None, + start_hint_line: int | None = None, ) -> tuple: """ Resolve potential line content values to proper hashline content IDs. @@ -299,6 +300,8 @@ def resolve_content_to_hashline_ids( original_content: Original file content (without hash prefixes) start_value: The start_line value from the edit end_value: The end_line value from the edit (optional) + start_hint_line: 0-based line number hint used to disambiguate + non-unique start_value matches (optional) Returns: tuple: (resolved_start, resolved_end) with hash IDs or original values @@ -361,6 +364,13 @@ def _resolve_to_hash_id(lines, idx, hp): if len(containing_indices) == 1: resolved_start_idx = containing_indices[0] resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) + elif len(containing_indices) > 1 and start_hint_line is not None: + # Multiple matches - pick closest to hint line + resolved_start_idx = min( + containing_indices, + key=lambda idx: abs(idx - start_hint_line), + ) + resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) elif start_value is not None and _looks_like_content_id(start_value): # Already a content ID - try to resolve it to find the line position # for proximity matching with end_value @@ -379,6 +389,13 @@ def _resolve_to_hash_id(lines, idx, hp): if len(containing_indices) == 1: resolved_start_idx = containing_indices[0] resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) + elif len(containing_indices) > 1 and start_hint_line is not None: + # Multiple matches - pick closest to hint line + resolved_start_idx = min( + containing_indices, + key=lambda idx: abs(idx - start_hint_line), + ) + resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) # Resolve end_value based on proximity to start position resolved_end = end_value diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index 02edbac3a3f..e2e7d4e361e 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -233,7 +233,12 @@ def _resolve(self, name: str) -> tuple[str, str, int, int]: ) self._validate_pattern_uniqueness(end_pattern, end_hint, "end", name, lines, end_hint_type) - start_id, end_id = resolve_content_to_hashline_ids(content, start_pattern, end_pattern) + start_id, end_id = resolve_content_to_hashline_ids( + content, + start_pattern, + end_pattern, + start_hint_line=start_hint if start_hint_type == "L" else None, + ) # Resolve line numbers from content IDs def _line_from_id(content_id: str, default_if_not_found: int) -> int: From 9cefd5d849253804fafaae3f6f6622fa31728b30 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 18:51:22 -0400 Subject: [PATCH 49/68] Unify methods for parsing file reads --- cecli/helpers/hashpos/transformations.py | 476 ++++++++++++++++++ .../helpers/orchestration/region_resolver.py | 97 +--- cecli/tools/edit_file.py | 32 +- cecli/tools/read_file.py | 164 ++---- 4 files changed, 536 insertions(+), 233 deletions(-) create mode 100644 cecli/helpers/hashpos/transformations.py diff --git a/cecli/helpers/hashpos/transformations.py b/cecli/helpers/hashpos/transformations.py new file mode 100644 index 00000000000..162a1aca50b --- /dev/null +++ b/cecli/helpers/hashpos/transformations.py @@ -0,0 +1,476 @@ +""" +HashPos transformation utilities. + +Centralizes all operations related to searching, processing line hints, +contextual markers, and extracting content from HashPos-encoded files. +Previously scattered across read_file.py, edit_file.py, hashline.py, +and region_resolver.py. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from cecli.helpers.hashpos.hashpos import UNIQUE_HASH_DELIMITER + +if TYPE_CHECKING: + from cecli.helpers.hashpos.hashpos import HashPos + + +# ──────────────────────────────────────────────────────────── +# Pattern Matching +# ──────────────────────────────────────────────────────────── + + +def search_in_lines( + lines: list[str], + pattern: str, + *, + return_last_line: bool = False, +) -> list[int]: + """Search for a multiline pattern in lines. + + Each line of *pattern* must appear as a whole word in the corresponding + line in *lines* at the match position. A ``whole word`` means the + pattern is surrounded by non-word characters (or line boundaries) — + ``"def"`` matches ``"def foo()"`` but not ``"define"``. + + Args: + lines: Source lines to search. + pattern: Pattern string (may span multiple lines via ``\n``). + return_last_line: When True, returns the index of the **last** + line of each match instead of the first. + + Returns: + 0-based indices of each match. + """ + + pattern_lines = pattern.split("\n") + indices: list[int] = [] + offset = len(pattern_lines) - 1 if return_last_line else 0 + + for i in range(len(lines) - len(pattern_lines) + 1): + if all( + _is_whole_word_match(p_line, lines[i + j]) for j, p_line in enumerate(pattern_lines) + ): + indices.append(i + offset) + + return indices + + +def _is_whole_word_match(pattern: str, line: str) -> bool: + """Return True if *pattern* appears as a whole word in *line*. + + A match requires that the characters immediately before and after the + match position are **not** word characters (``[a-zA-Z0-9_]``) — or + that the match is at the start/end of the line respectively. + """ + + idx = line.find(pattern) + + while idx != -1: + before_ok = idx == 0 or not (line[idx - 1].isalnum() or line[idx - 1] == "_") + after_pos = idx + len(pattern) + after_ok = after_pos >= len(line) or not ( + line[after_pos].isalnum() or line[after_pos] == "_" + ) + + if before_ok and after_ok: + return True + + idx = line.find(pattern, idx + 1) + + return False + + +def find_substring_matches( + lines: list[str], + value: str, +) -> list[int]: + """Return 0-based indices of all lines containing *value* as a substring.""" + + value_stripped = value.strip() + return [i for i, line in enumerate(lines) if value_stripped in line] + + +def find_multiline_match( + lines: list[str], + value: str, +) -> int | None: + """Find the start index where the full multiline *value* matches consecutive lines. + + Returns None if *value* is a single line or no match is found. + """ + + value_lines = value.strip().splitlines() + + if len(value_lines) <= 1: + return None + + for i in range(len(lines) - len(value_lines) + 1): + if all(value_lines[j].strip() in lines[i + j] for j in range(len(value_lines))): + return i + + return None + + +# ──────────────────────────────────────────────────────────── +# Content ID Detection +# ──────────────────────────────────────────────────────────── + + +def is_content_id(value: str) -> bool: + """Return True if *value* appears to be a content ID rather than text.""" + + from cecli.helpers.hashline import ContentHashError, normalize_hashline + + if value in ("@000", "000@"): + return True + + try: + normalize_hashline(value) + + return True + except (ContentHashError, ValueError): + return False + + +# ──────────────────────────────────────────────────────────── +# @L Line Number Resolution +# ──────────────────────────────────────────────────────────── + + +def resolve_at_l( + line_spec: str, + hp: HashPos, + lines: list[str], +) -> str: + """Resolve ``@L{{num}}`` notation to a content ID using a HashPos index. + + Returns the input unchanged if it is not an @L{{num}} spec. + Raises ValueError if the line number is out of range. + """ + + if not ( + isinstance(line_spec, str) + and line_spec.startswith("@L") + and len(line_spec) > 2 + and line_spec[2:].isdigit() + ): + return line_spec + + line_num = int(line_spec[2:]) - 1 + + if line_num < 0 or line_num >= len(lines): + raise ValueError( + f"@L reference line {int(line_spec[2:])} is out of range " + f"(file has {len(lines)} lines)" + ) + + line_text = lines[line_num] + occurrence = 1 + sum(1 for i in range(line_num) if lines[i] == line_text) + + return hp.get_wrapped_id(hp.generate_public_id(line_text, line_num, occurrence)) + + +# ──────────────────────────────────────────────────────────── +# Hint Extraction (@L / @A / @B) +# ──────────────────────────────────────────────────────────── + + +def extract_hint( + pattern: str, + lines: list[str] | None = None, +) -> tuple[str, int | None, str | None]: + """Extract a hint suffix from a pattern string. + + Supports three hint types: + + * ``@L`` — direct line number (always resolvable). + * ``@A{{{{regex}}}}`` — keep only matches **after** the first regex hit. + * ``@B{{{{regex}}}}`` — keep only matches **before** the last regex hit. + + Returns a ``(stripped_pattern, hint_value, hint_type)`` tuple: + + * ``hint_value`` — 0-based line number, or None. + * ``hint_type`` — ``'L'``, ``'A'``, ``'B'``, or None. + """ + + # @L hint — direct line number, always resolvable + m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) + + if m: + return pattern[: m.start()], int(m.group(1)) - 1, "L" + + # @A{{regex}} hint — first regex match, filter to lines AFTER + m = re.search(r"[ \t]+@A\{\{(.+?)\}\}[ \t]*$", pattern) + + if m: + stripped = pattern[: m.start()] + + if lines is not None: + regex_str = m.group(1) + + try: + for i, line in enumerate(lines): + if re.search(regex_str, line): + return stripped, i, "A" + except re.error: + pass + + return stripped, None, None + + # @B{{regex}} hint — last regex match, filter to lines BEFORE + m = re.search(r"[ \t]+@B\{\{(.+?)\}\}[ \t]*$", pattern) + + if m: + stripped = pattern[: m.start()] + + if lines is not None: + regex_str = m.group(1) + + try: + last_match = None + + for i, line in enumerate(lines): + if re.search(regex_str, line): + last_match = i + + if last_match is not None: + return stripped, last_match, "B" + except re.error: + pass + + return stripped, None, None + + return pattern, None, None + + +# ──────────────────────────────────────────────────────────── +# Search Type Classification +# ──────────────────────────────────────────────────────────── + + +def classify_search_type( + range_start: str, + range_end: str, +) -> dict[str, bool]: + """Classify range markers into structured, text, mixed, or contextual types. + + The returned dict contains boolean flags usable by downstream resolvers + to decide how to find start/end line indices. + """ + + def _is_line_ref(s: str) -> bool: + return s.startswith("@L") and s[2:].isdigit() and len(s) > 2 + + start_is_lr = _is_line_ref(range_start) + end_is_lr = _is_line_ref(range_end) + start_is_sp = range_start in ("@000", "000@") + end_is_sp = range_end in ("@000", "000@") + + return { + "start_is_line_ref": start_is_lr, + "end_is_line_ref": end_is_lr, + "start_is_special": start_is_sp, + "end_is_special": end_is_sp, + "start_is_text": not start_is_lr and not start_is_sp, + "end_is_text": not end_is_lr and not end_is_sp, + "both_structured": (start_is_lr or start_is_sp) and (end_is_lr or end_is_sp), + "mixed_special": ( + (start_is_sp and not end_is_lr and not end_is_sp) + or (end_is_sp and not start_is_lr and not start_is_sp) + ), + "end_is_contextual": ( + range_end.startswith(("@C", "@P", "@N")) + and len(range_end) > 2 + and range_end[2:].isdigit() + ), + } + + +# ──────────────────────────────────────────────────────────── +# Contextual Markers (@C / @P / @N) +# ──────────────────────────────────────────────────────────── + + +def apply_contextual_marker( + start_indices: list[int], + range_start: str, + range_end: str, + num_lines: int, +) -> tuple[list[int], list[int]]: + """Expand range using @C{{num}}, @P{{num}}, or @N{{num}} contextual markers. + + Requires exactly one start match. Raises ValueError when the number + of start matches is not 1 (the caller is expected to format the error + message with its own context like file path / operation index). + + Returns ``(new_start_indices, new_end_indices)``. + """ + + if len(start_indices) != 1: + raise ValueError( + f"Start pattern '{range_start}' must match exactly one" + f" location when using @C/@P/@N end markers." + f" Found {len(start_indices)} matches." + ) + + ctx_s_idx = start_indices[0] + ctx_num = int(range_end[2:]) + marker_type = range_end[1] + + if marker_type == "C": + s_idx = max(0, ctx_s_idx - ctx_num) + e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) + elif marker_type == "P": + s_idx = max(0, ctx_s_idx - ctx_num) + e_idx = ctx_s_idx + else: # 'N' + s_idx = ctx_s_idx + e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) + + return [s_idx], [e_idx] + + +# ──────────────────────────────────────────────────────────── +# Proximity / Narrowing +# ──────────────────────────────────────────────────────────── + + +def narrow_by_proximity( + indices: list[int], + target: int, + max_results: int = 5, +) -> list[int]: + """Return up to *max_results* indices closest to *target* (0-based). + + Results are sorted by ascending distance from *target*. + """ + + if not indices: + return [] + + scored = [(abs(i - target), i) for i in indices] + scored.sort(key=lambda x: x[0]) + + return [idx for _, idx in scored[:max_results]] + + +# ──────────────────────────────────────────────────────────── +# Pair / Range Computation +# ──────────────────────────────────────────────────────────── + + +def compute_best_pair( + start_indices: list[int], + end_indices: list[int], + *, + allow_inverted: bool = False, +) -> tuple[int, int] | None: + """Find the smallest-range valid (start, end) pair. + + When *allow_inverted* is True, also considers pairs where end < start + (the LLM may have swapped the order) and returns them in correct order. + """ + + min_dist = float("inf") + best_pair: tuple[int, int] | None = None + + for s in start_indices: + valid_ends = end_indices if allow_inverted else [e for e in end_indices if e >= s] + + for e in valid_ends: + dist = abs(e - s) + + if dist < min_dist: + min_dist = dist + best_pair = (s, e) + + if allow_inverted and best_pair and best_pair[1] < best_pair[0]: + best_pair = (best_pair[1], best_pair[0]) + + return best_pair + + +def reposition_indices( + target_idx: int, + start_idx: int, + end_idx: int, + total_lines: int = 20, +) -> tuple[int, int]: + """Calculate clamped start/end indices for a centered window. + + Returns ``(slice_start, slice_end)`` compatible with Python slicing + (i.e. *slice_end* is exclusive). + """ + + half_window = total_lines // 2 + + left = target_idx - half_window + right = target_idx + half_window + + if left < start_idx: + right += start_idx - left + left = start_idx + + if right > end_idx: + left -= right - end_idx + right = end_idx + + left = max(start_idx, left) + + return left, right + 1 + + +# ──────────────────────────────────────────────────────────── +# Prefix / Content ID Utilities +# ──────────────────────────────────────────────────────────── + + +def strip_hashline_prefix(value: str) -> str: + """Strip the ``~~`` virtual prefix from a ReadFile output line reference.""" + + if isinstance(value, str) and value.startswith(UNIQUE_HASH_DELIMITER): + return value[len(UNIQUE_HASH_DELIMITER) :].lstrip() + + return value + + +def try_resolve_as_unique_line( + hp: HashPos, + value: str, +) -> str | None: + """Try to resolve a value by matching it against unique lines in the source. + + If *value* (after stripping) matches exactly one line in the source + **and** that line appears only once (unique), returns a tilde-wrapped + hash ID that can be resolved to line indices by the HashPos engine. + + Returns None if resolution fails. + """ + + if not hp or not value: + return None + + value_stripped = value.strip() + + if not value_stripped: + return None + + matching_lines: list[tuple[int, str]] = [] + + for i, line in enumerate(hp.lines): + if line.strip() == value_stripped: + matching_lines.append((i, line)) + + if len(matching_lines) == 1: + idx, matched_line = matching_lines[0] + + if hp.line_counts.get(matched_line, 0) == 1: + hash_id = hp.generate_public_id(matched_line, idx, 1) + + return hp.get_wrapped_id(hash_id) + + return None diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py index e2e7d4e361e..f19fcbe543b 100644 --- a/cecli/helpers/orchestration/region_resolver.py +++ b/cecli/helpers/orchestration/region_resolver.py @@ -10,6 +10,14 @@ from typing import Any +from cecli.helpers.hashpos.transformations import ( + extract_hint, + is_content_id, + narrow_by_proximity, + resolve_at_l, + search_in_lines, +) + class AgentRegion: """ @@ -287,16 +295,14 @@ def _line_from_id(content_id: str, default_if_not_found: int) -> int: def _search_in_lines(lines: list[str], pattern: str) -> list[int]: """Return 0-based indices of all lines where *pattern* matches. - Supports multiline patterns (each line of *pattern* must be - a substring of the corresponding line in *lines*). + Supports multiline patterns (each line of *pattern* must appear + as a whole word in the corresponding line of *lines*). + + Delegates to the core ``search_in_lines`` in + ``cecli.helpers.hashpos.transformations`` for unified matching. """ - pattern_lines = pattern.split("\n") - indices = [] - for i in range(len(lines) - len(pattern_lines) + 1): - if all(p_line in lines[i + j] for j, p_line in enumerate(pattern_lines)): - indices.append(i) - return indices + return search_in_lines(lines, pattern) @staticmethod def _extract_l_hint( @@ -312,55 +318,14 @@ def _extract_l_hint( - hint_value is a 0-based line number or None - hint_type is 'L', 'A', 'B', or None """ - import re - - # Try @L hint (direct line number - always resolvable) - m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) - if m: - return pattern[: m.start()], int(m.group(1)) - 1, "L" - - # Try @A{{regex}} hint (first regex match — filter to lines AFTER) - m = re.search(r"[ \t]+@A\{\{(.+?)\}\}[ \t]*$", pattern) - if m: - stripped = pattern[: m.start()] - if lines is not None: - regex_str = m.group(1) - try: - for i, line in enumerate(lines): - if re.search(regex_str, line): - return stripped, i, "A" - except re.error: - pass - return stripped, None, None - - # Try @B{{regex}} hint (last regex match — filter to lines BEFORE) - m = re.search(r"[ \t]+@B\{\{(.+?)\}\}[ \t]*$", pattern) - if m: - stripped = pattern[: m.start()] - if lines is not None: - regex_str = m.group(1) - try: - last_match = None - for i, line in enumerate(lines): - if re.search(regex_str, line): - last_match = i - if last_match is not None: - return stripped, last_match, "B" - except re.error: - pass - return stripped, None, None - - return pattern, None, None + + return extract_hint(pattern, lines) @staticmethod def _narrow_by_proximity(indices: list[int], target: int, max_results: int = 5) -> list[int]: """Return up to *max_results* indices closest to *target* (0-based).""" - if not indices: - return [] - scored = [(abs(i - target), i) for i in indices] - scored.sort(key=lambda x: x[0]) - return [idx for _, idx in scored[:max_results]] + return narrow_by_proximity(indices, target, max_results=max_results) def _validate_pattern_uniqueness( self, @@ -449,17 +414,7 @@ def _validate_pattern_uniqueness( def _looks_like_content_id(value: str) -> bool: """Return True if *value* appears to be a content ID rather than text.""" - from cecli.helpers.hashline import ContentHashError, normalize_hashline - - if value in ("@000", "000@"): - return True - - try: - normalize_hashline(value) - - return True - except (ContentHashError, ValueError): - return False + return is_content_id(value) def _resolve_pattern( self, @@ -483,24 +438,12 @@ def _resolve_pattern( if pattern in ("@000", "000@"): return pattern - # @L{num} notation — resolve to line text or content ID + # @L{num} notation — resolve via shared utility import re as _re _m = _re.match(r"^@L(\d+)$", pattern) if _m: - _line_num = int(_m.group(1)) - 1 - if _line_num < 0 or _line_num >= len(lines): - raise ValueError( - f"@L reference line {_m.group(1)} is out of range " - f"(file has {len(lines)} lines)" - ) - _line_text = lines[_line_num] - if lines.count(_line_text) == 1: - return _line_text # Unique — let resolve_content_to_hashline_ids handle it - # Duplicate — get content ID directly via HashPos - _occurrence = 1 + sum(1 for i in range(_line_num) if lines[i] == _line_text) - _public_id = hp.generate_public_id(_line_text, _line_num, _occurrence) - return hp.get_wrapped_id(_public_id) + return resolve_at_l(pattern, hp, lines) if not self._looks_like_content_id(pattern): return pattern diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 424b64cdb6f..c71e20eeecb 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -8,6 +8,7 @@ strip_hashline, ) from cecli.helpers.hashpos.hashpos import HashPos +from cecli.helpers.hashpos.transformations import resolve_at_l, strip_hashline_prefix from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, @@ -584,33 +585,20 @@ def _resolve_at_l_num(cls, line_spec, hp, source_lines, file_path): Returns the input unchanged if it's not an @L{num} spec. Raises ToolError if the line number is out of range. """ - if not ( - isinstance(line_spec, str) - and line_spec.startswith("@L") - and len(line_spec) > 2 - and line_spec[2:].isdigit() - ): - return line_spec - - line_num = int(line_spec[2:]) - 1 - if line_num < 0 or line_num >= len(source_lines): - from cecli.tools.utils.helpers import ToolError - - raise ToolError( - f"@L reference line {int(line_spec[2:])} is out of range " - f"(file has {len(source_lines)} lines)" - ) - line_text = source_lines[line_num] - occurrence = 1 + sum(1 for i in range(line_num) if source_lines[i] == line_text) - return hp.get_wrapped_id(hp.generate_public_id(line_text, line_num, occurrence)) + from cecli.tools.utils.helpers import ToolError + + try: + return resolve_at_l(line_spec, hp, source_lines) + + except ValueError as e: + raise ToolError(str(e)) from e @staticmethod def _strip_readfile_prefix(value): """Strip the ``~~`` virtual prefix from a ReadFile output line reference.""" - if isinstance(value, str) and value.startswith("~~"): - return value[2:].lstrip() - return value + + return strip_hashline_prefix(value) @classmethod def _categorize_edit_error(cls, error_msg: str) -> str: diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index 1bcb00c781a..462ddc3408c 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -3,6 +3,15 @@ from typing import Dict, List from cecli.helpers.hashline import hashline_formatted, strip_hashline +from cecli.helpers.hashpos.transformations import ( + apply_contextual_marker, + classify_search_type, + compute_best_pair, + extract_hint, + narrow_by_proximity, + reposition_indices, + search_in_lines, +) from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, @@ -134,7 +143,7 @@ def execute(cls, coder, read, **kwargs): file_path = read_op.get("file_path") range_start = read_op.get("range_start") range_end = read_op.get("range_end") - padding = 5 + padding = 0 if file_path is None: error_outputs.append( @@ -801,27 +810,8 @@ def _reposition_indices( Calculates the clamped start and end indices for a centered window. Returns a tuple of (slice_start, slice_end) compatible with python slicing. """ - # 1. Calculate ideal half-window size - half_window = total_lines // 2 - - # 2. Calculate initial left/right bounds - left = target_idx - half_window - right = target_idx + half_window - - # 3. Slide the window if it overflows boundaries - if left < start_idx: - right += start_idx - left - left = start_idx - - if right > end_idx: - left -= right - end_idx - right = end_idx - # 4. Final safety clamp in case the range itself is smaller than total_lines - left = max(start_idx, left) - - # Return right + 1 so it's ready-to-use for standard Python slicing [start:end] - return left, right + 1 + return reposition_indices(target_idx, start_idx, end_idx, total_lines) @classmethod def clear_old_messages(cls, coder): @@ -1082,32 +1072,7 @@ def _try_fuzzy_narrow_indices( def _classify_search_type(cls, range_start, range_end): """Classify range markers into structured, text, mixed, or contextual search types.""" - def _is_line_ref(s): - return s.startswith("@L") and s[2:].isdigit() and len(s) > 2 - - start_is_lr = _is_line_ref(range_start) - end_is_lr = _is_line_ref(range_end) - start_is_sp = range_start in ("@000", "000@") - end_is_sp = range_end in ("@000", "000@") - - return { - "start_is_line_ref": start_is_lr, - "end_is_line_ref": end_is_lr, - "start_is_special": start_is_sp, - "end_is_special": end_is_sp, - "start_is_text": not start_is_lr and not start_is_sp, - "end_is_text": not end_is_lr and not end_is_sp, - "both_structured": (start_is_lr or start_is_sp) and (end_is_lr or end_is_sp), - "mixed_special": ( - (start_is_sp and not end_is_lr and not end_is_sp) - or (end_is_sp and not start_is_lr and not start_is_sp) - ), - "end_is_contextual": ( - range_end.startswith(("@C", "@P", "@N")) - and len(range_end) > 2 - and range_end[2:].isdigit() - ), - } + return classify_search_type(range_start, range_end) @classmethod def _extract_l_hint(cls, pattern, lines=None): @@ -1121,45 +1086,8 @@ def _extract_l_hint(cls, pattern, lines=None): - hint_value is a 0-based line number or None - hint_type is 'L', 'A', 'B', or None """ - import re - - # Try @L hint (direct line number - always resolvable) - m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) - if m: - return pattern[: m.start()], int(m.group(1)) - 1, "L" - - # Try @A{{regex}} hint (first regex match — filter to lines AFTER) - m = re.search(r"[ \t]+@A\{\{(.+?)\}\}[ \t]*$", pattern) - if m: - stripped = pattern[: m.start()] - if lines is not None: - regex_str = m.group(1) - try: - for i, line in enumerate(lines): - if re.search(regex_str, line): - return stripped, i, "A" - except re.error: - pass - return stripped, None, None - - # Try @B{{regex}} hint (last regex match — filter to lines BEFORE) - m = re.search(r"[ \t]+@B\{\{(.+?)\}\}[ \t]*$", pattern) - if m: - stripped = pattern[: m.start()] - if lines is not None: - regex_str = m.group(1) - try: - last_match = None - for i, line in enumerate(lines): - if re.search(regex_str, line): - last_match = i - if last_match is not None: - return stripped, last_match, "B" - except re.error: - pass - return stripped, None, None - - return pattern, None, None + + return extract_hint(pattern, lines) @classmethod def _search_in_lines(cls, lines, pattern, return_last_line=False): @@ -1167,14 +1095,12 @@ def _search_in_lines(cls, lines, pattern, return_last_line=False): Returns list of matching indices. When return_last_line is True, returns the index of the last line of each match instead of the first. + + Delegates to the core ``search_in_lines`` in + ``cecli.helpers.hashpos.transformations`` for unified matching. """ - pattern_lines = pattern.split("\n") - indices = [] - offset = len(pattern_lines) - 1 if return_last_line else 0 - for i in range(len(lines) - len(pattern_lines) + 1): - if all(p_line in lines[i + j] for j, p_line in enumerate(pattern_lines)): - indices.append(i + offset) - return indices + + return search_in_lines(lines, pattern, return_last_line=return_last_line) @classmethod def _find_start_indices(cls, lines, range_start, classification, num_lines): @@ -1216,36 +1142,23 @@ def _apply_contextual_marker( Returns ((new_start_indices, new_end_indices), error). Error is None on success, or a formatted error string on failure. """ - if len(start_indices) != 1: + + try: + result = apply_contextual_marker(start_indices, range_start, range_end, num_lines) + + return result, None + + except ValueError as e: error = cls.format_error( coder, - ( - f"Start pattern '{range_start}' must match exactly one" - f" location when using @C/@P/@N end markers." - f" Found {len(start_indices)} matches." - ), + str(e), file_path, range_start, range_end, read_index, ) - return None, error - - ctx_s_idx = start_indices[0] - ctx_num = int(range_end[2:]) - marker_type = range_end[1] - - if marker_type == "C": - s_idx = max(0, ctx_s_idx - ctx_num) - e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) - elif marker_type == "P": - s_idx = max(0, ctx_s_idx - ctx_num) - e_idx = ctx_s_idx - else: # 'N' - s_idx = ctx_s_idx - e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) - return ([s_idx], [e_idx]), None + return None, error @classmethod def _disambiguate_start_indices( @@ -1355,12 +1268,8 @@ def _narrow_by_proximity(cls, indices, target, max_results=5): Returns up to max_results indices sorted by ascending distance from the target (0-based line number). """ - if not indices: - return [] - scored = [(abs(i - target), i) for i in indices] - scored.sort(key=lambda x: x[0]) - return [idx for _, idx in scored[:max_results]] + return narrow_by_proximity(indices, target, max_results=max_results) @classmethod def _resolve_to_final_indices( @@ -1482,21 +1391,8 @@ def _compute_best_pair(cls, start_indices, end_indices, allow_inverted=False): When allow_inverted is True, also considers pairs where end < start (LLM swapped the order) and returns them in correct order. """ - min_dist = float("inf") - best_pair = None - - for s in start_indices: - valid_ends = end_indices if allow_inverted else [e for e in end_indices if e >= s] - for e in valid_ends: - dist = abs(e - s) - if dist < min_dist: - min_dist = dist - best_pair = (s, e) - - if allow_inverted and best_pair and best_pair[1] < best_pair[0]: - best_pair = (best_pair[1], best_pair[0]) - return best_pair + return compute_best_pair(start_indices, end_indices, allow_inverted=allow_inverted) @classmethod def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=True): From 15d5d95763a805b862141740eb64e93b14ecc2d1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 19:08:51 -0400 Subject: [PATCH 50/68] Add clearer error messages to the hashpos system --- cecli/helpers/hashline.py | 112 +++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/cecli/helpers/hashline.py b/cecli/helpers/hashline.py index 388abb0562e..6a0222d4343 100644 --- a/cecli/helpers/hashline.py +++ b/cecli/helpers/hashline.py @@ -1807,6 +1807,106 @@ def _try_resolve_as_unique_line( return None +def _raise_clear_hash_error(hp, op, key): + """Raise a ContentHashError with a clear message explaining why a value couldn't be resolved. + + Checks if the value matches line content in the file and provides specific guidance + about how to fix the reference (use hashed ID prefix, check for duplicates, etc.). + """ + value = op.get(key, "") + value_stripped = value.strip() if value else "" + + if not hp or not value_stripped: + raise ContentHashError( + f"Could not resolve '{value}' as a content ID. " + "Use a hashed ID prefix (e.g., `~XXXX~`) to target the line." + ) + + # Count how many lines contain this value as a substring + matching_count = 0 + for line in hp.lines: + if value_stripped in line: + matching_count += 1 + + if matching_count > 1: + raise ContentHashError( + f"Line '{value_stripped}' appears {matching_count} times in the file. " + "Use the hashed ID prefix (e.g., `~XXXX~`) to disambiguate " + "which occurrence to target." + ) + elif matching_count == 1: + raise ContentHashError( + f"Line '{value_stripped}' was found once in the file but could not be " + "resolved as a unique content ID. Use the hashed ID prefix (e.g., `~XXXX~`) " + "to target it precisely." + ) + else: + raise ContentHashError( + f"Line '{value_stripped}' was not found in the file. " + "Use the exact line content or a hashed ID prefix (e.g., `~XXXX~`) " + "to target the desired line." + ) + + +def _detect_overlapping_ranges(resolved_ops): + """Check resolved operations for genuinely overlapping ranges. + + Detects operations where ranges overlap in a way that can't be resolved + by simple deduplication (same-start) or containment. These non-trivial + overlaps (e.g., a delete range partially overlapping a replace range) + are flagged and returned as failed operations with a clear message. + The overlapping operations' indices are also returned so the caller + can exclude them from `resolved_ops`. + + Returns a tuple of (failed_ops, indices_to_remove). + - failed_ops: list of failed operation dicts with overlap errors + - indices_to_remove: set of operation indices to remove from resolved_ops + """ + if len(resolved_ops) < 2: + return [], set() + + failed_ops = [] + indices_to_remove = set() + for i in range(len(resolved_ops)): + a = resolved_ops[i] + a_start = a["start_idx"] + a_end = a["end_idx"] + + for j in range(i + 1, len(resolved_ops)): + b = resolved_ops[j] + b_start = b["start_idx"] + b_end = b["end_idx"] + + # Skip trivial cases handled elsewhere: + # 1. Same start line - handled by _deduplicate_ranges + if a_start == b_start: + continue + + # 2. One range fully contained within the other - handled by _merged_contained_ranges + if (a_start <= b_start and b_end <= a_end) or (b_start <= a_start and a_end <= b_end): + continue + + # Check if ranges genuinely overlap (not just adjacent/contiguous) + # Two ranges [a_start, a_end] and [b_start, b_end] overlap if + # a_start <= b_end and b_start <= a_end + if a_start <= b_end and b_start <= a_end: + op_a = a["op"] + op_b = b["op"] + error_msg = ( + f"Operation {a['index'] + 1} ({op_a['operation']}, lines " + f"{a_start}-{a_end}) overlaps with operation {b['index'] + 1} " + f"({op_b['operation']}, lines {b_start}-{b_end}). " + "Edits with overlapping ranges are not supported. " + "Combine them into a single edit or use non-overlapping ranges." + ) + failed_ops.append({"index": a["index"], "error": error_msg, "operation": op_a}) + failed_ops.append({"index": b["index"], "error": error_msg, "operation": op_b}) + indices_to_remove.add(i) + indices_to_remove.add(j) + + return failed_ops, indices_to_remove + + def apply_hashline_operations( original_content: str, operations: list, @@ -1847,7 +1947,7 @@ def apply_hashline_operations( if resolved is not None: normalized_op["start_line_hash"] = resolved else: - raise + _raise_clear_hash_error(hp, op, "start_line_hash") if "end_line_hash" in op: # Normalize end line hash if present @@ -1859,7 +1959,7 @@ def apply_hashline_operations( if resolved is not None: normalized_op["end_line_hash"] = resolved else: - raise + _raise_clear_hash_error(hp, op, "end_line_hash") normalized_operations.append(normalized_op) except Exception as e: @@ -1955,6 +2055,14 @@ def apply_hashline_operations( except Exception as e: failed_ops.append({"index": i, "error": str(e), "operation": op}) + # Check for overlapping ranges among resolved operations + overlap_failures, overlap_indices = _detect_overlapping_ranges(resolved_ops) + for overlap_failure in overlap_failures: + failed_ops.append(overlap_failure) + # Remove overlapping operations from resolved_ops so they don't get applied + if overlap_indices: + resolved_ops = [op for i, op in enumerate(resolved_ops) if i not in overlap_indices] + # Honor cancellations: remove operations that are cancelled by later cancel operations resolved_ops = _honor_cancellations(resolved_ops) # Deduplicate: if multiple operations start on the same line, keep only the latest one From 38e0b5d1ae185dd89ce5da6418b5f43185759207 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 19:14:29 -0400 Subject: [PATCH 51/68] Fix small grep errors --- cecli/tools/grep.py | 67 +++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index f650ebe5680..d2da25d09e2 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -90,20 +90,48 @@ def _parse_content_into_files(output): Returns list of dicts with keys: path, match_count, content_lines Content lines preserve the original grep format (with : for matches, - for context). + Merges entries for the same file that appear in non-contiguous match groups. """ if not output: return [] - files = [] lines = output.splitlines() if not lines: return [] + # Use a dict to accumulate file groups, keyed by filepath + # This handles interleaved results (file1 -> file2 -> file1) by merging + file_groups = {} # filepath -> {path, match_count, content_parts} + file_order = [] # ordered list of filepaths as they first appear + current_file = None current_lines = [] match_count = 0 - for i, line in enumerate(lines): + def _flush_current(): + nonlocal current_file, current_lines, match_count + if current_file is None or not current_lines: + return + + if current_file in file_groups: + # Merge into existing entry + existing = file_groups[current_file] + existing["match_count"] += match_count + existing["content_parts"].append("\n".join(current_lines)) + else: + # New file entry + file_groups[current_file] = { + "path": current_file, + "match_count": match_count, + "content_parts": ["\n".join(current_lines)], + } + file_order.append(current_file) + + current_file = None + current_lines = [] + match_count = 0 + + for line in lines: # Skip separator lines ("--" between non-contiguous match groups) if line == "--": continue @@ -129,14 +157,8 @@ def _parse_content_into_files(output): if is_match_line: match_count += 1 else: - # New file - save previous, start new - files.append( - { - "path": current_file, - "match_count": match_count, - "content": "\n".join(current_lines), - } - ) + # Different file - flush current block before switching + _flush_current() current_file = filepath match_count = 1 if is_match_line else 0 current_lines = [line] @@ -145,13 +167,18 @@ def _parse_content_into_files(output): if current_file is not None: current_lines.append(line) - # Don't forget the last file - if current_file is not None and current_lines: + # Flush the last file's accumulated block + _flush_current() + + # Build final list preserving first-seen order + files = [] + for fpath in file_order: + group = file_groups[fpath] files.append( { - "path": current_file, - "match_count": match_count, - "content": "\n".join(current_lines), + "path": group["path"], + "match_count": group["match_count"], + "content": "\n".join(group["content_parts"]), } ) @@ -432,8 +459,8 @@ def execute( # --- PASS 2: Get content with context --- content_cmd = ( base_cmd - + (["-B", str(context_before)] if context_before > 0 else []) - + (["-A", str(context_after)] if context_after > 0 else []) + + (["-B", str(context_before)] if context_before >= 0 else []) + + (["-A", str(context_after)] if context_after >= 0 else []) + case_flag + pattern_flag + exclude_args @@ -486,13 +513,13 @@ def execute( match_line_re = re.compile(r"^" + filepath_escaped + r":\d+:") match_lines_found = [ln for ln in file_lines if match_line_re.match(ln)] + # Normalize path to be relative to repo root + pf["path"] = os.path.relpath(pf["path"], repo.root) + if len(match_lines_found) > MAX_MATCHES_PER_FILE: truncated_files.append(pf["path"]) trimmed = "\n".join(match_lines_found[:MAX_MATCHES_PER_FILE]) pf["content"] = trimmed - - # Normalize path to be relative to repo root - pf["path"] = os.path.relpath(pf["path"], repo.root) total_matches += pf.get("count_from_pass", 0) total_files += 1 From d42884dc8af5ab7de762aaee734a473b7e886d70 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 19:21:51 -0400 Subject: [PATCH 52/68] Specify that no other modules can be imported --- cecli/helpers/orchestration/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 1b133932b16..cccd6c62522 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -522,6 +522,8 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `json` | Parse / serialize: `json.loads(s)`, `json.dumps(obj, indent=2)` | | `traceback` | Traceback formatting: `traceback.format_exc()`, `traceback.format_tb(...)` | +All other module imports will fail. + ### Usage ```python From 4c0ead286a63dec28d0e465f953459ece1953dc3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 19:42:02 -0400 Subject: [PATCH 53/68] Update orcehstration system prompt --- cecli/helpers/orchestration/environment.py | 47 +++++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index cccd6c62522..907dcba32f9 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -536,6 +536,41 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non print(tool_outputs["b"]) # key access ``` +### Working with Results + +Use `Agent.peek()` to discover a result's structure, then `Agent.get_value()` +to extract specific fields by dot-path:: + + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + print(Agent.peek(result)) + # Shows: result[0].content: str = '...' + # result[0]._.file_path: str = 'foo.py' + + content = Agent.get_value(result, "result.0.content") + file_path = Agent.get_value(result, "result.0._.file_path", "unknown") + +Result list items are plain dicts — use item['content'] / item.get('content') +and item['_'] / item.get('_') to access data and metadata respectively. + +### Creating New Files + +Use `ResourceManager` to create an empty file, then `EditFile` to write +initial content (use `@000` for start/end on empty files):: + + rm = Agent.get_tool("ResourceManager") + edit = Agent.get_tool("EditFile") + + await rm.call(create=["path/to/new_file.py"]) + await edit.call(edits=[{ + "file_path": "path/to/new_file.py", + "operation": "replace", + "start_line": "@000", + "end_line": "@000", + "text": "def greet(name):\n return f\"Hello, {name}!\"\n", + }]) + +After creation, use `resolve_regions` and `edit_region` for targeted edits. + ### Editing with Regions Use `Agent.resolve_regions()` to convert text patterns into content IDs, then `Agent.edit_region()` to apply edits using the resolved IDs. @@ -562,18 +597,8 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non #### Alternative: Call `EditFile` directly with `regions.get_start()` / `regions.get_end()` -```python -edit_tool = Agent.get_tool("EditFile") -await edit_tool.call(edits=[{ - "file_path": "foo.py", - "operation": "replace", - "start_line": regions.get_start("my_func"), - "end_line": regions.get_end("my_func"), - "text": "def my_func():\n return 42", -}]) -``` - ### Gotchas + - **Types**: compare with `typeof(x) == dict` or `isinstance(x, dict)` — NOT `typeof(x) == "dict"` - **Args**: use keyword args only — `tool.call(file_path="f", ...)` - **gather**: always use named `gather(x=a, y=b)` — positional args are not supported From e810465edc6039c00578d00d4b946c8218c82850 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 19:48:08 -0400 Subject: [PATCH 54/68] Unify command_interactive into a parameter inside the command tool --- cecli/coders/agent_coder.py | 2 - cecli/tools/__init__.py | 2 - cecli/tools/command.py | 95 ++++++++++++++- cecli/tools/command_interactive.py | 181 ----------------------------- 4 files changed, 93 insertions(+), 187 deletions(-) delete mode 100644 cecli/tools/command_interactive.py diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 4f3b0f78589..ee22f4be61a 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -62,7 +62,6 @@ def __init__(self, *args, **kwargs): self.max_tool_vector_history = 20 self.read_tools = { "command", - "commandinteractive", "explorecode", "ls", "readfile", @@ -72,7 +71,6 @@ def __init__(self, *args, **kwargs): } self.write_tools = { "command", - "commandinteractive", "editfile", "undochange", } diff --git a/cecli/tools/__init__.py b/cecli/tools/__init__.py index 3898d4cb733..62ecc0cf1c9 100644 --- a/cecli/tools/__init__.py +++ b/cecli/tools/__init__.py @@ -4,7 +4,6 @@ from . import ( _yield, command, - command_interactive, delegate, edit_file, explore_code, @@ -27,7 +26,6 @@ # List of all available tool modules for dynamic discovery TOOL_MODULES = [ command, - command_interactive, delegate, edit_file, explore_code, diff --git a/cecli/tools/command.py b/cecli/tools/command.py index 8055d12c8e5..06d694c97ed 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -15,7 +15,7 @@ import xxhash from cecli.helpers.background_commands import BackgroundCommandManager -from cecli.run_cmd import run_cmd_subprocess +from cecli.run_cmd import run_cmd, run_cmd_subprocess from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header @@ -78,6 +78,16 @@ class Tool(BaseTool): ), "default": False, }, + "user_input_required": { + "type": "boolean", + "description": ( + "When True, runs the command interactively using a " + "pseudo-terminal (PTY), allowing the user to provide " + "inputs like passwords or navigate terminal interfaces. " + "Handles TUI suspension automatically." + ), + "default": False, + }, }, "required": [], }, @@ -103,12 +113,16 @@ async def execute( action=None, stdin=None, pty=False, + user_input_required=False, **kwargs, ): """ Execute a shell command or interact with background processes. For new commands: provide 'command' (and optionally 'background', 'stdin', 'pty'). + When 'user_input_required' is True, runs the command interactively using a + pseudo-terminal (PTY), allowing the user to provide inputs like passwords + or navigate terminal interfaces. For background interactions: provide 'background_key' + 'action' (stdin/stop). Commands run with timeout based on agent_config['command_timeout'] (default: 30 seconds). @@ -172,7 +186,9 @@ async def execute( if hasattr(coder, "agent_config"): timeout = coder.agent_config.get("command_timeout", 30) - if background: + if user_input_required: + return await cls._execute_interactive(coder, command) + elif background: return await cls._execute_background(coder, command, use_pty=pty, stdin=stdin) elif timeout > 0: return await cls._execute_with_timeout(coder, command, timeout, use_pty=pty) @@ -505,6 +521,76 @@ async def _execute_foreground(cls, coder, command_string): ) return response + @classmethod + async def _execute_interactive(cls, coder, command_string): + """ + Execute command interactively, allowing the user to provide inputs + like passwords or navigate terminal interfaces. + Handles TUI suspension automatically. + """ + import asyncio + + response = ToolResponse(cls.NORM_NAME) + + coder.io.tool_output( + f"\u26ed Starting interactive shell command: {command_string}", type="tool-result" + ) + + tui = coder.tui() if coder.tui else None + + def _run_interactive(): + return run_cmd( + command_string, + verbose=coder.verbose, + error_print=coder.io.tool_error, + cwd=coder.root, + should_print=True, + ) + + if tui: + coder.io.tool_output( + ">>> Suspending TUI for interactive command <<<", type="tool-result" + ) + exit_status, combined_output = tui.run_obstructive(_run_interactive) + else: + coder.io.tool_output( + ">>> You may need to interact with the command below <<<", type="tool-result" + ) + coder.io.tool_output(" \n") + await coder.io.stop_input_task() + await asyncio.sleep(1) + exit_status, combined_output = _run_interactive() + await asyncio.sleep(1) + coder.io.tool_output(" \n", type="tool-result") + coder.io.tool_output(" \n", type="tool-result") + + coder.io.tool_output(">>> Interactive command finished <<<", type="tool-result") + + # Format the output for the result message, include more content + output_content = combined_output or "" + output_limit = coder.large_file_token_threshold + if coder.context_management_enabled and len(output_content) > output_limit: + output_content = ( + output_content[:output_limit] + + f"\n... (output truncated at {output_limit} characters, based on" + " large_file_token_threshold)" + ) + + cls.clear_invocation_cache() + + if exit_status == 0: + response.append_result( + "Interactive command finished successfully (exit code 0)." + f" Output:\n{output_content}" + ) + return response + else: + response.append_result( + f"Interactive command finished with exit code {exit_status}." + f" Output:\n{output_content}" + ) + return response + @classmethod async def _stop_background_command(cls, coder, command_key): """ @@ -555,6 +641,9 @@ def format_output(cls, coder, mcp_server, tool_response): action = params.get("action") stdin = params.get("stdin") pty = params.get("pty", False) + user_input_required = params.get("user_input_required", False) + + coder.io.tool_output("") coder.io.tool_output("") @@ -566,6 +655,8 @@ def format_output(cls, coder, mcp_server, tool_response): extras.append(f"action={action}") if pty: extras.append("pty=True") + if user_input_required: + extras.append("user_input_required=True") if extras: coder.io.tool_output(f"{color_start}Options:{color_end} {', '.join(extras)}") diff --git a/cecli/tools/command_interactive.py b/cecli/tools/command_interactive.py deleted file mode 100644 index 3b2579bcb29..00000000000 --- a/cecli/tools/command_interactive.py +++ /dev/null @@ -1,181 +0,0 @@ -# Import necessary functions -import asyncio -import fnmatch - -import xxhash - -from cecli.run_cmd import run_cmd -from cecli.tools.utils.base_tool import BaseTool -from cecli.tools.utils.responses import ToolResponse - - -class Tool(BaseTool): - NORM_NAME = "commandinteractive" - TRACK_INVOCATIONS = False - ALLOWED_SESSION_COMMANDS = {} - SCHEMA = { - "type": "function", - "function": { - "name": "CommandInteractive", - "description": ( - "Execute a shell command interactively." - " Useful when you need the user to provide inputs like passwords" - " or navigating terminal interfaces." - ), - "parameters": { - "type": "object", - "properties": { - "command_string": { - "type": "string", - "description": "The interactive shell command to execute.", - }, - }, - "required": ["command_string"], - }, - }, - } - - @staticmethod - def _is_command_allowed(coder, command_string): - """Check if command matches any allowed_commands patterns.""" - if hasattr(coder, "agent_config"): - allowed_commands = coder.agent_config.get("allowed_commands", []) - if allowed_commands: - for pattern in allowed_commands: - if fnmatch.fnmatch(command_string, pattern): - return True - return False - - @staticmethod - def _hash_command(command): - """Compute an xxhash of the full command text for session tracking.""" - if not command: - return command - - return xxhash.xxh64(command).hexdigest() - - @classmethod - async def _get_confirmation(cls, coder, command_string): - """Get user confirmation for command execution.""" - # Hash command for dict key lookup - command_hash = cls._hash_command(command_string) - - # Check if command is already handled for this session - if command_hash in cls.ALLOWED_SESSION_COMMANDS: - if cls.ALLOWED_SESSION_COMMANDS[command_hash]: - return True # Previously approved for session - # Previously declined - skip session question, continue to normal confirmation - - if coder.skip_cli_confirmations: - return True - - # Check if command matches any allowed_commands patterns - if cls._is_command_allowed(coder, command_string): - return True - - formatted_command = coder.format_command_with_prefix(command_string) - - confirmed = await coder.io.confirm_ask( - "Allow execution of this command?", - subject=formatted_command, - explicit_yes_required=True, - allow_never=True, - group_response="Command Interactive Tool", - ) - - if not confirmed: - return False - - # Ask if user wants to allow for the entire session (only once per command) - if command_hash not in cls.ALLOWED_SESSION_COMMANDS: - session_allowed = await coder.io.confirm_ask( - "Allow this command for the rest of the session?", - subject=formatted_command, - ) - cls.ALLOWED_SESSION_COMMANDS[command_hash] = session_allowed - - return True - - @classmethod - async def execute(cls, coder, command_string, **kwargs): - """ - Execute an interactive shell command using run_cmd (which uses pexpect/PTY). - """ - try: - response = ToolResponse(cls.NORM_NAME) - confirmed = await cls._get_confirmation(coder, command_string) - if not confirmed: - response.append_result("Shell command execution skipped by user.") - return response - - command_string = coder.format_command_with_prefix(command_string) - - coder.io.tool_output( - f"⛭ Starting interactive shell command: {command_string}", type="tool-result" - ) - - tui = coder.tui() if coder.tui else None - - def _run_interactive(): - return run_cmd( - command_string, - verbose=coder.verbose, - error_print=coder.io.tool_error, - cwd=coder.root, - should_print=True, - ) - - if tui: - coder.io.tool_output( - ">>> Suspending TUI for interactive command <<<", type="tool-result" - ) - exit_status, combined_output = tui.run_obstructive(_run_interactive) - else: - coder.io.tool_output( - ">>> You may need to interact with the command below <<<", type="tool-result" - ) - coder.io.tool_output(" \n") - await coder.io.stop_input_task() - await asyncio.sleep(1) - exit_status, combined_output = _run_interactive() - await asyncio.sleep(1) - coder.io.tool_output(" \n", type="tool-result") - coder.io.tool_output(" \n", type="tool-result") - - coder.io.tool_output(">>> Interactive command finished <<<", type="tool-result") - - # Format the output for the result message, include more content - output_content = combined_output or "" - output_limit = coder.large_file_token_threshold - if coder.context_management_enabled and len(output_content) > output_limit: - output_content = ( - output_content[:output_limit] - + f"\n... (output truncated at {output_limit} characters, based on" - " large_file_token_threshold)" - ) - - cls.clear_invocation_cache() - - if exit_status == 0: - response.append_result( - "Interactive command finished successfully (exit code 0)." - f" Output:\n{output_content}" - ) - return response - else: - response.append_result( - f"Interactive command finished with exit code {exit_status}." - f" Output:\n{output_content}" - ) - return response - - except Exception as e: - coder.io.tool_error( - f"Error executing interactive shell command '{command_string}': {str(e)}" - ) - # Optionally include traceback for debugging if verbose - # if coder.verbose: - # coder.io.tool_error(traceback.format_exc()) - response = ToolResponse(cls.NORM_NAME) - response.append_result(f"Error executing interactive command: {str(e)}") - return response From 5575d3301026886433978c5438db76e897987533 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 20 Jul 2026 02:38:01 -0400 Subject: [PATCH 55/68] Cache bust file reads on edit --- cecli/coders/agent_coder.py | 1 + cecli/helpers/orchestration/environment.py | 56 ++-------------------- cecli/tools/edit_file.py | 10 ++-- cecli/tools/read_file.py | 11 ++++- 4 files changed, 19 insertions(+), 59 deletions(-) diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index ee22f4be61a..5e65e444c48 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -81,6 +81,7 @@ def __init__(self, *args, **kwargs): self.change_tracker = ChangeTracker() self.args = kwargs.get("args") self.files_added_in_exploration = set() + self.file_read_cache = set() self.tool_call_count = 0 self.turn_count = 0 self.max_reflections = 15 diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 907dcba32f9..a025f0c138a 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -478,8 +478,7 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non return None return """ -The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. -This is much more efficient than making individual tool calls for loop-heavy workflows. +The `Orchestrate` tool lets you coordinate chains of sub agents and tools with Python code in a limited, secure sandbox across multiple turns. Variables and methods defined in a script are persisted in subsequent turns. As such, results from previous calls can be reused and helper methods can be defined to enhance ease of use within the environment. @@ -496,10 +495,6 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `Agent.peek(result)` | Inspect a tool result's structure and leaf content — returns a string; use `print(Agent.peek(result))` to see it | | `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | -| `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | -| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs. Use `start_line_hint` / `end_line_hint` in region spec entries for disambiguation (supports `@L`, `@A{{regex}}`, `@B{{regex}}` — same hint syntax as ReadFile). Ambiguous patterns raise immediately. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | -| `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | - | `await Agent.sleep(seconds)` | Pause execution (0-120s max) | | `gather(**named_tasks)` | Run tasks concurrently; returns an iterable with `.key` / `["key"]` access | @@ -529,8 +524,8 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non ```python tool = Agent.get_tool("delegate") tool_outputs = await gather( - a=tool.call(prompt="A"), - b=tool.call(prompt="B"), + a=tool.call(prompt="A", async=False), + b=tool.call(prompt="B", async=False), ) print(tool_outputs.a) # attribute access print(tool_outputs["b"]) # key access @@ -552,51 +547,6 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non Result list items are plain dicts — use item['content'] / item.get('content') and item['_'] / item.get('_') to access data and metadata respectively. -### Creating New Files - -Use `ResourceManager` to create an empty file, then `EditFile` to write -initial content (use `@000` for start/end on empty files):: - - rm = Agent.get_tool("ResourceManager") - edit = Agent.get_tool("EditFile") - - await rm.call(create=["path/to/new_file.py"]) - await edit.call(edits=[{ - "file_path": "path/to/new_file.py", - "operation": "replace", - "start_line": "@000", - "end_line": "@000", - "text": "def greet(name):\n return f\"Hello, {name}!\"\n", - }]) - -After creation, use `resolve_regions` and `edit_region` for targeted edits. - -### Editing with Regions - -Use `Agent.resolve_regions()` to convert text patterns into content IDs, then `Agent.edit_region()` to apply edits using the resolved IDs. - -#### Step 1 — resolve region boundaries once - -```python -regions = Agent.resolve_regions("foo.py", [ - {"name": "my_func", "start": "def my_func", "end": "return result"}, - {"name": "init", "start": "def __init__", "end": "self.x = x"}, -]) -``` - -#### Step 2 — Use `regions.get(name)` with `Agent.edit_region()` (recommended shorthand) - -```python -await Agent.edit_region( - file_path="foo.py", - edits=[ - {"region": regions.get("my_func"), "text": "def my_func():\n return 42"}, - ], -) -``` - -#### Alternative: Call `EditFile` directly with `regions.get_start()` / `regions.get_end()` - ### Gotchas - **Types**: compare with `typeof(x) == dict` or `isinstance(x, dict)` — NOT `typeof(x) == "dict"` diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index c71e20eeecb..98e336ba564 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -201,12 +201,11 @@ def execute( coder, file_path_key ) + if abs_path: + coder.file_read_cache.discard(abs_path) + # Build HashPos index once per file for @L{num} resolution - hp = ( - HashPos(original_content) - if original_content and original_content.strip() - else None - ) + hp = HashPos(original_content or "") source_lines = ( original_content.splitlines() if original_content and original_content.strip() @@ -437,6 +436,7 @@ def execute( if total_successful_edits == 0: coder.edit_allowed = True error_msg = "No edits were successfully applied:\n" + "\n".join(all_failed_edits) + response.append_error(all_failed_edits) raise ToolError(error_msg) # 5. Format and return result diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index 462ddc3408c..4c3a91371cb 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -133,6 +133,7 @@ def execute(cls, coder, read, **kwargs): all_outputs = [] already_up_to_details = [] new_context_details = [] + seen_files = set() all_outputs_set = set() new_context_set = set() already_up_to_set = set() @@ -210,6 +211,13 @@ def execute(cls, coder, read, **kwargs): ) continue + if abs_path not in seen_files: + seen_files.add(abs_path) + + if abs_path not in coder.file_read_cache: + coder.file_read_cache.add(abs_path) + ConversationService.get_files(coder).clear_ranges(abs_path) + # 3. Read file content content: str = coder.io.read_text(abs_path) @@ -545,7 +553,8 @@ def execute(cls, coder, read, **kwargs): ConversationService.get_files(coder).clear_ranges(abs_path) ConversationService.get_files(coder).push_range(abs_path, tuples) - ConversationService.get_chunks(coder).add_file_context_messages() + if new_context_details: + ConversationService.get_chunks(coder).add_file_context_messages() # if ( # ConversationService.get_chunks(coder).last_clear_count > 20 From fcbbcc38c82f147498442089f685386e8005a195 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 20 Jul 2026 08:40:26 -0400 Subject: [PATCH 56/68] Revert orchestrate context block to more verbose one, update grep tool --- cecli/helpers/orchestration/environment.py | 56 ++++++++++++++++++++-- cecli/tools/grep.py | 36 +++++++------- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index a025f0c138a..907dcba32f9 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -478,7 +478,8 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non return None return """ -The `Orchestrate` tool lets you coordinate chains of sub agents and tools with Python code in a limited, secure sandbox across multiple turns. +The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. +This is much more efficient than making individual tool calls for loop-heavy workflows. Variables and methods defined in a script are persisted in subsequent turns. As such, results from previous calls can be reused and helper methods can be defined to enhance ease of use within the environment. @@ -495,6 +496,10 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non | `Agent.peek(result)` | Inspect a tool result's structure and leaf content — returns a string; use `print(Agent.peek(result))` to see it | | `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | +| `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | +| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs. Use `start_line_hint` / `end_line_hint` in region spec entries for disambiguation (supports `@L`, `@A{{regex}}`, `@B{{regex}}` — same hint syntax as ReadFile). Ambiguous patterns raise immediately. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | +| `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | + | `await Agent.sleep(seconds)` | Pause execution (0-120s max) | | `gather(**named_tasks)` | Run tasks concurrently; returns an iterable with `.key` / `["key"]` access | @@ -524,8 +529,8 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non ```python tool = Agent.get_tool("delegate") tool_outputs = await gather( - a=tool.call(prompt="A", async=False), - b=tool.call(prompt="B", async=False), + a=tool.call(prompt="A"), + b=tool.call(prompt="B"), ) print(tool_outputs.a) # attribute access print(tool_outputs["b"]) # key access @@ -547,6 +552,51 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non Result list items are plain dicts — use item['content'] / item.get('content') and item['_'] / item.get('_') to access data and metadata respectively. +### Creating New Files + +Use `ResourceManager` to create an empty file, then `EditFile` to write +initial content (use `@000` for start/end on empty files):: + + rm = Agent.get_tool("ResourceManager") + edit = Agent.get_tool("EditFile") + + await rm.call(create=["path/to/new_file.py"]) + await edit.call(edits=[{ + "file_path": "path/to/new_file.py", + "operation": "replace", + "start_line": "@000", + "end_line": "@000", + "text": "def greet(name):\n return f\"Hello, {name}!\"\n", + }]) + +After creation, use `resolve_regions` and `edit_region` for targeted edits. + +### Editing with Regions + +Use `Agent.resolve_regions()` to convert text patterns into content IDs, then `Agent.edit_region()` to apply edits using the resolved IDs. + +#### Step 1 — resolve region boundaries once + +```python +regions = Agent.resolve_regions("foo.py", [ + {"name": "my_func", "start": "def my_func", "end": "return result"}, + {"name": "init", "start": "def __init__", "end": "self.x = x"}, +]) +``` + +#### Step 2 — Use `regions.get(name)` with `Agent.edit_region()` (recommended shorthand) + +```python +await Agent.edit_region( + file_path="foo.py", + edits=[ + {"region": regions.get("my_func"), "text": "def my_func():\n return 42"}, + ], +) +``` + +#### Alternative: Call `EditFile` directly with `regions.get_start()` / `regions.get_end()` + ### Gotchas - **Types**: compare with `typeof(x) == dict` or `isinstance(x, dict)` — NOT `typeof(x) == "dict"` diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index d2da25d09e2..2067ddcf134 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -228,25 +228,18 @@ class Tool(BaseTool): "default": True, "description": "Whether to perform a case-insensitive search.", }, - "count": { - "type": "boolean", - "default": True, - "description": ( - "Include match counts per file in the output summary." - ), - }, "context_before": { "type": "integer", "default": 2, "description": ( - "Number of context lines to show before each match." + "Number of context lines to show before each match. Max 5" ), }, "context_after": { "type": "integer", "default": 2, "description": ( - "Number of context lines to show after each match." + "Number of context lines to show after each match. Max 5" ), }, }, @@ -362,8 +355,8 @@ def execute( directory = search_op.get("directory", search_op.get("path", ".")) use_regex = search_op.get("use_regex", False) case_insensitive = search_op.get("case_insensitive", True) - context_before = search_op.get("context_before", 2) - context_after = search_op.get("context_after", 2) + context_before = max(min(int(search_op.get("context_before", 2)), 5), 0) + context_after = max(min(int(search_op.get("context_after", 2)), 5), 0) count_enabled = search_op.get("count", True) op_result = { @@ -502,7 +495,7 @@ def execute( MAX_MATCHES_PER_FILE = 10 MAX_FILES = 20 - truncated_files = [] + truncated_files = {} total_matches = 0 total_files = 0 @@ -510,14 +503,21 @@ def execute( file_lines = pf["content"].splitlines() # Find actual match lines (lines with `:LINE:` pattern) filepath_escaped = re.escape(pf["path"]) - match_line_re = re.compile(r"^" + filepath_escaped + r":\d+:") + match_line_re = re.compile(r"^" + filepath_escaped + r":(\d+):") match_lines_found = [ln for ln in file_lines if match_line_re.match(ln)] # Normalize path to be relative to repo root pf["path"] = os.path.relpath(pf["path"], repo.root) if len(match_lines_found) > MAX_MATCHES_PER_FILE: - truncated_files.append(pf["path"]) + # Extract line numbers from excess matches (beyond first 10) + extra_lines = match_lines_found[MAX_MATCHES_PER_FILE:] + extra_line_nums = [] + for xline in extra_lines: + xm = match_line_re.match(xline) + if xm: + extra_line_nums.append(int(xm.group(1))) + truncated_files[pf["path"]] = extra_line_nums trimmed = "\n".join(match_lines_found[:MAX_MATCHES_PER_FILE]) pf["content"] = trimmed total_matches += pf.get("count_from_pass", 0) @@ -536,7 +536,7 @@ def execute( { "file": pf["path"], "match_count": pf.get("count_from_pass", 0), - "truncated": pf["path"] in truncated_files, + "additional_matched_lines": truncated_files.get(pf["path"], []), "content": pf["content"], } for pf in parsed_files[:MAX_FILES] @@ -573,7 +573,7 @@ def execute( response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) for op_result in all_operation_results: files = op_result.get("files", []) - metadata = {k: v for k, v in op_result.items() if k != "files"} + metadata = op_result.copy() # Build a human-readable string summary as content pattern = op_result.get("pattern", "") @@ -584,7 +584,7 @@ def execute( else: lines = [f"[{pattern}]"] for f in files: - truncated_mark = " (truncated)" if f.get("truncated") else "" + truncated_mark = " (truncated)" if f.get("additional_matched_lines") else "" lines.append(f"{f['file']}: {f['match_count']} match(es){truncated_mark}") if op_result.get("has_more_files"): lines.append(f"... ({op_result['total_files']} files total)") @@ -616,7 +616,7 @@ def format_output(cls, coder, mcp_server, tool_response): for i, search_op in enumerate(searches): pattern = search_op.get("pattern", "") file_pattern = search_op.get("file_glob", "*") - directory = search_op.get("directory", ".") + directory = search_op.get("directory", search_op.get("path", ".")) use_regex = search_op.get("use_regex", False) case_insensitive = search_op.get("case_insensitive", True) context_before = search_op.get("context_before", 2) From ddead555a59f1c7b5a8d8dae32010e26bbb899e6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 20 Jul 2026 09:02:06 -0400 Subject: [PATCH 57/68] Update hashpos to lead with stable identifiers --- cecli/helpers/hashpos/hashpos.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index 81622256d66..3a5698f98c8 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -101,15 +101,14 @@ def generate_public_id(self, text: str, line_idx: int, occurrence: int) -> str: packed = (line_hash << 20) | (idx_bits << 6) | occ_bits res = "" - for _ in range(4): - res += self.B1024[packed % 1024] - packed //= 1024 + for i in range(3, -1, -1): + res += self.B1024[(packed >> (10 * i)) & 0x3FF] return res def unpack_public_id(self, public_id: str) -> tuple[int, int, int]: packed = 0 for i, char in enumerate(public_id): - packed |= self.B1024.index(char) << (10 * i) + packed |= self.B1024.index(char) << (10 * (3 - i)) occ_bits = packed & 0x3F idx_bits = (packed >> 6) & 0x3FFF From 98591f5a3ad677245a548454e1fcd8440c8c6117 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 11:53:59 -0400 Subject: [PATCH 58/68] Allow configuration of the orchestration sandbox to open it up for more powerfull operations --- cecli/helpers/orchestration/environment.py | 86 ++++- cecli/helpers/orchestration/safe_methods.py | 27 +- cecli/helpers/orchestration/security.py | 96 +++++- cecli/helpers/orchestration/service.py | 6 +- cecli/website/docs/config/agent-mode.md | 60 +++- tests/helpers/test_orchestration.py | 334 ++++++++++++++++++++ 6 files changed, 584 insertions(+), 25 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 907dcba32f9..67db6ff8f5e 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -202,16 +202,30 @@ class AgentExecutionEnv: # Shared across all AgentExecutionEnv instances — any agent can read/write _shared_state: TrackedDict = TrackedDict() - def __init__(self, coder: Any) -> None: + def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = None) -> None: + from typing import Any as _Any + + self._orchestration_config: dict[str, _Any] = orchestration_config or {} + self._DANGEROUS_BUILTINS: set[str] = { + "eval", + "exec", + "open", + "__import__", + "compile", + "breakpoint", + "globals", + "locals", + } + self.state = TrackedDict() self.state._set_owner(self) self._shared_state._set_owner(self, is_shared=True) # Initialize early so _safe_builtins can reference them - self.globals: dict[str, Any] = {} - self.locals: dict[str, Any] = {} + self.globals: dict[str, _Any] = {} + self.locals: dict[str, _Any] = {} - self._safe_builtins: dict[str, Any] = { + self._safe_builtins: dict[str, _Any] = { "print": print, "range": range, "len": len, @@ -267,6 +281,32 @@ def __init__(self, coder: Any) -> None: "pathlib": _SafePathlib, } + # Task 3: allowed_builtins — extend _safe_builtins with user-requested builtins + allowed_builtins: list[str] = self._orchestration_config.get("allowed_builtins", []) + for name in allowed_builtins: + if name.startswith("_"): + raise ValueError(f"Cannot allow private builtin '{name}'") + if name in self._DANGEROUS_BUILTINS and not self._orchestration_config.get( + "disable_security" + ): + raise ValueError(f"Cannot allow dangerous builtin '{name}'") + self._safe_builtins[name] = __builtins__[name] + + # When security is fully disabled, expose dangerous builtins + if self._orchestration_config.get("disable_security", False): + for name in self._DANGEROUS_BUILTINS: + self._safe_builtins[name] = __builtins__[name] + + # When allow_classes is enabled, expose __build_class__ needed by Python + if self._orchestration_config.get("allow_classes", False): + self._safe_builtins["__build_class__"] = __builtins__["__build_class__"] + + # When allowed_imports is configured, expose __import__ so imports work, + # and add __name__ since imported modules reference it. + if self._orchestration_config.get("allowed_imports"): + self._safe_builtins["__import__"] = __builtins__["__import__"] + self._safe_builtins["__name__"] = "__sandbox__" + self.globals.clear() _agent = AgentProxy(coder) _agent._env = self @@ -382,7 +422,8 @@ async def execute(self, code_str: str) -> dict: code_str = code_str.strip() code_str = _escape_newlines_in_strings(code_str) - code_str, extra_globals = _strip_allowed_imports(code_str) + extra_allowed = frozenset(self._orchestration_config.get("allowed_imports", [])) + code_str, extra_globals = _strip_allowed_imports(code_str, extra_allowed=extra_allowed) self.globals.update(extra_globals) if not code_str: return {"results": "", "state_variables": self._state_snapshot()} @@ -411,8 +452,13 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: return {"results": code, "state_variables": self._state_snapshot()} try: - tree = SecurityFilter().visit(tree) - tree = LoopYieldInjector().visit(tree) + if not self._orchestration_config.get("disable_security", False): + tree = SecurityFilter( + allowed_imports=extra_allowed, + allow_classes=self._orchestration_config.get("allow_classes", False), + ).visit(tree) + if not self._orchestration_config.get("disable_loop_protection", False): + tree = LoopYieldInjector().visit(tree) ast.fix_missing_locations(tree) except SecurityError as e: return _build_result(f"Security Error: {e}") @@ -477,7 +523,9 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non if not agent_config.get("allow_orchestration", True): return None - return """ + orchestration_config = agent_config.get("orchestration", {}) + + context = """ The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. This is much more efficient than making individual tool calls for loop-heavy workflows. Variables and methods defined in a script are persisted in subsequent turns. @@ -611,6 +659,28 @@ def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | Non 4. Use `print(...)` to output results — only printed output is returned """ + # Task 6: Append sandbox configuration overrides when non-empty + if orchestration_config: + overrides: list[str] = [] + if orchestration_config.get("allowed_imports"): + overrides.append( + f"- Allowed extra imports: {orchestration_config['allowed_imports']}" + ) + if orchestration_config.get("allowed_builtins"): + overrides.append( + f"- Allowed extra builtins: {orchestration_config['allowed_builtins']}" + ) + if orchestration_config.get("allow_classes"): + overrides.append("- Class definitions are allowed (__init__, __str__, etc.)") + if orchestration_config.get("disable_security"): + overrides.append("- ⚠ Security filtering is DISABLED") + if orchestration_config.get("disable_loop_protection"): + overrides.append("- ⚠ Loop protection is DISABLED") + if overrides: + context += "\n### Sandbox Configuration Overrides\n" + "\n".join(overrides) + "\n" + + return context + # flake8: noqa # fmt: on diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index 17bc30cbe0b..231c0029f18 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -586,12 +586,21 @@ def _escape_newlines_in_strings(code: str) -> str: ) -def _strip_allowed_imports(code: str) -> tuple[str, dict[str, object]]: +def _strip_allowed_imports( + code: str, + extra_allowed: frozenset[str] | None = None, +) -> tuple[str, dict[str, object]]: """Strip import lines for modules already pre-imported in the sandbox. The sandbox provides ``re``, ``math``, ``itertools``, ``collections``, ``datetime``, ``traceback``, and ``json`` as read-only proxies. + When *extra_allowed* is provided, import statements for those modules + are also allowed through (neither stripped nor commented out). The caller + is responsible for ensuring those names are resolvable at execution time + (either by pre-importing them into the sandbox globals or letting the + ``SecurityFilter`` allow the ``import`` statement through). + Returns ``(code, extra_globals)`` where *extra_globals* maps imported names to their resolved values. The caller must inject these into the execution namespace before running the code. @@ -606,7 +615,11 @@ def _strip_allowed_imports(code: str) -> tuple[str, dict[str, object]]: import re as _re + preimported = _PREIMPORTED_MODULES + skip_allowed = extra_allowed or frozenset() + lines = code.splitlines() + result = [] extra_globals: dict[str, object] = {} @@ -619,7 +632,11 @@ def _strip_allowed_imports(code: str) -> tuple[str, dict[str, object]]: r"import\s+(\w+)(?:\s+as\s+(\w+))?\s*$", stripped, ) - if m and m.group(1) in _PREIMPORTED_MODULES: + if m and m.group(1) in skip_allowed: + result.append(line) + continue + + if m and m.group(1) in preimported: mod_name = m.group(1) alias = m.group(2) or mod_name @@ -637,7 +654,11 @@ def _strip_allowed_imports(code: str) -> tuple[str, dict[str, object]]: r"from\s+(\w+)\s+import\s+(.*)$", stripped, ) - if m and m.group(1) in _PREIMPORTED_MODULES: + if m and m.group(1) in skip_allowed: + result.append(line) + continue + + if m and m.group(1) in preimported: mod_name = m.group(1) names_clause = m.group(2) diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py index 58c5775f18b..97fd2d8e430 100644 --- a/cecli/helpers/orchestration/security.py +++ b/cecli/helpers/orchestration/security.py @@ -59,6 +59,41 @@ class SecurityFilter(ast.NodeTransformer): _SAFE_DUNDER: set[str] = {"__name__", "__doc__"} + _SAFE_CLASS_DUNDERS: frozenset[str] = frozenset( + { + "__init__", + "__str__", + "__repr__", + "__iter__", + "__next__", + "__len__", + "__getitem__", + "__setitem__", + "__contains__", + "__enter__", + "__exit__", + "__aenter__", + "__aexit__", + "__await__", + "__anext__", + "__aiter__", + "__del__", + "__init_subclass__", + "__set_name__", + "__class_getitem__", + } + ) + + def __init__( + self, + allowed_imports: frozenset[str] | None = None, + allow_classes: bool = False, + ): + super().__init__() + self._allowed_imports = allowed_imports or frozenset() + self._allow_classes = allow_classes + self._class_depth = 0 + # ------------------------------------------------------------------ # Helpers # ------------------------------------------------------------------ @@ -89,17 +124,24 @@ def _make_raise_stmt(message: str) -> ast.Expr: # Statement visitors (import / global / nonlocal) # ------------------------------------------------------------------ - def visit_Import(self, node: ast.Import) -> ast.Expr: - return self._make_raise_stmt( - f"Security filter error at line {node.lineno}: " - "Imports are disabled in the agent orchestration environment." - ) + def visit_Import(self, node: ast.Import) -> ast.Expr | ast.Import: + for alias in node.names: + if alias.name not in self._allowed_imports: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + f"Import '{alias.name}' is not in allowed_imports." + ) - def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.Expr: - return self._make_raise_stmt( - f"Security filter error at line {node.lineno}: " - "Imports are disabled in the agent orchestration environment." - ) + return node + + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.Expr | ast.ImportFrom: + if node.module is None or node.module not in self._allowed_imports: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + f"Import from '{node.module}' is not in allowed_imports." + ) + + return node def visit_Global(self, node: ast.Global) -> ast.Expr: return self._make_raise_stmt( @@ -114,17 +156,47 @@ def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.Expr: ) # ------------------------------------------------------------------ - # Expression visitors (attribute access / dangerous calls) + def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef | ast.Expr: + if self._allow_classes: + self._class_depth += 1 + self.generic_visit(node) + self._class_depth -= 1 + return node + + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "Class definitions are disabled in the orchestration environment." + ) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef | ast.Expr: + if self._allow_classes and self._class_depth > 0 and node.name.startswith("__"): + if node.name not in self._SAFE_CLASS_DUNDERS: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + f"Method name '{node.name}' is not allowed. " + f"Only safe dunder methods are permitted inside class bodies." + ) + + return self.generic_visit(node) + # ------------------------------------------------------------------ + # Expression visitors (attribute access / dangerous calls) def visit_Attribute(self, node: ast.Attribute): if node.attr.startswith("_"): if node.attr in self._SAFE_DUNDER: return self.generic_visit(node) + if ( + self._allow_classes + and self._class_depth > 0 + and node.attr in self._SAFE_CLASS_DUNDERS + ): + return self.generic_visit(node) + if isinstance(node.ctx, (ast.Store, ast.Del)): verb = "assign to" if isinstance(node.ctx, ast.Store) else "delete" - raise SecurityError( + return self._make_raise_expr( f"Security filter error at line {node.lineno}: " f"cannot {verb} private attribute '{node.attr}'" ) diff --git a/cecli/helpers/orchestration/service.py b/cecli/helpers/orchestration/service.py index 2a907996f85..9856da3091e 100644 --- a/cecli/helpers/orchestration/service.py +++ b/cecli/helpers/orchestration/service.py @@ -24,7 +24,11 @@ def get_instance(cls, coder) -> AgentExecutionEnv: cls._instances[coder] = instance return instance - instance = AgentExecutionEnv(coder) + orchestration_config = {} + if hasattr(coder, "agent_config"): + orchestration_config = coder.agent_config.get("orchestration", {}) + + instance = AgentExecutionEnv(coder, orchestration_config=orchestration_config) cls._instances[coder] = instance cls._uuid_index[coder.uuid] = instance return instance diff --git a/cecli/website/docs/config/agent-mode.md b/cecli/website/docs/config/agent-mode.md index 797e7d61150..92982d52cc2 100644 --- a/cecli/website/docs/config/agent-mode.md +++ b/cecli/website/docs/config/agent-mode.md @@ -166,6 +166,55 @@ Agent Mode can also be configured directly in your configuration file. See the [ - **`command_timeout`**: Time in seconds to wait for shell commands to finish before automatic backgrounding occurs (default: None) - **`diff_colors`**: When enabled, diff output in edit tool responses uses color-coded lines - removed lines in magenta, added lines in light green, and context lines in plain text (default: true) +- **`orchestration`**: A nested configuration object for the Orchestrate tool's Python sandbox. + When absent or empty, the sandbox runs with default restrictions. See [Orchestration + Configuration](#orchestration-configuration) below for details. + +#### Orchestration Configuration + +The `Orchestrate` tool runs user code in a secure Python sandbox with the following +restrictions: +- **No imports** — only pre-imported modules (`re`, `math`, `itertools`, `collections`, + `datetime`, `traceback`, `json`, `pathlib`) are available +- **No private/dunder access** — attributes starting with `_` are blocked +- **No dangerous builtins** — `eval`, `exec`, `open`, `compile`, `breakpoint`, + `__import__`, `globals`, `locals`, `setattr`, `delattr` are blocked +- **No global/nonlocal statements** +- **Loop protection** — `while`/`for` loops yield cooperatively to prevent hangs + +These restrictions can be selectively relaxed via the `orchestration` config block: + +- **`allowed_imports`**: Array of module names to allow importing. Example: + `["os", "typing"]`. Only standard library modules should be allowed; third-party + modules may execute arbitrary code at import time. + +- **`allowed_builtins`**: Array of builtin function names to add to the sandbox. + Example: `["setattr", "property"]`. Dangerous builtins (`eval`, `exec`, `open`, etc.) + cannot be added. + +- **`allow_classes`**: Boolean (default: `false`). When `true`, class definitions are + permitted and dunder methods (`__init__`, `__str__`, `__repr__`, `__iter__`, etc.) + are allowed inside class bodies. + +- **`disable_security`**: Boolean (default: `false`). When `true`, the AST-level security + filter is skipped entirely. ⚠ Use with extreme caution — this disables all import + blocking, dunder blocking, and dangerous builtin blocking. + +- **`disable_loop_protection`**: Boolean (default: `false`). When `true`, the cooperative + yield injection is skipped. Use only if you are certain the orchestration code has no + unbounded loops. + +Example: + +```yaml +agent-config: + orchestration: + allowed_imports: + - os + - typing + allow_classes: true +``` + #### Essential Tools Certain tools are always available regardless of includelist/excludelist settings: @@ -288,7 +337,16 @@ agent-config: allowed_commands: ["wc -l*"] # Commands matching these glob patterns will not prompt for confirmation show_lint_errors: false # When enabled, linting errors are shown in tool output (default: false) hot_reload: false # When enabled, skills configuration is hot-reloaded automatically (default: false) -фвδ:: diff_colors: true # When enabled, diff output uses color-coded lines (default: true) + diff_colors: true # When enabled, diff output uses color-coded lines (default: true) + + # Orchestration sandbox configuration (optional) + orchestration: + allowed_imports: [] # Optional: Module names to allow importing + allowed_builtins: [] # Optional: Builtin names to add + allow_classes: false # Optional: Allow class definitions + disable_security: false # Optional: Disable security filter (⚠ dangerous) + disable_loop_protection: false # Optional: Disable loop yield injection + # Skills configuration (see Skills documentation for details) skills_paths: ["~/my-skills", "./project-skills"] # Directories to search for skills skills_includelist: ["python-refactoring", "react-components"] # Optional: Whitelist of skills to include diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py index d1f8353f9a1..3f7a4585053 100644 --- a/tests/helpers/test_orchestration.py +++ b/tests/helpers/test_orchestration.py @@ -15,7 +15,9 @@ from cecli.helpers.orchestration.environment import ( AgentExecutionEnv, AgentProxy, + build_orchestration_context_block, ) +from cecli.helpers.orchestration.safe_methods import _strip_allowed_imports from cecli.helpers.orchestration.security import ( LoopYieldInjector, SecurityError, @@ -1379,3 +1381,335 @@ async def test_env_named_gather_exception_handling(): ) # Just verify it doesn't crash assert result["results"] is not None + + +# --------------------------------------------------------------------------- +# Test Group 1: allowed_imports +# --------------------------------------------------------------------------- + + +class TestAllowedImports: + """Tests for the `allowed_imports` orchestration config option.""" + + def test_import_with_allowed_imports(self): + """1.1: `import os` with allowed_imports=["os"] passes SecurityFilter.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("import os", mode="exec") + rewritten = sf.visit(tree) + # Should not contain __security_raise — imports pass through + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_import_without_allowed_imports(self): + """1.2: `import os` with empty allowed_imports is blocked.""" + assert not _run_security_filter_safe("import os") + + def test_from_import_with_allowed(self): + """1.3: `from os import path` with allowed_imports=["os"] succeeds.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("from os import path", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_from_import_as_with_allowed(self): + """1.4: `from os import path as p` with allowed_imports=["os"] succeeds.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("from os import path as p", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_mixed_import_one_not_allowed(self): + """1.5: `import os, sys` with allowed_imports=["os"] is blocked on sys.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("import os, sys", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_preimported_module_still_works(self): + """1.6: Pre-imported module (`import re`) with no extra config still works.""" + # _strip_allowed_imports strips pre-imported modules, so nothing + # reaches SecurityFilter; but if it does, default filter blocks it. + # This test verifies the stripping works. + code, extras = _strip_allowed_imports("import re\nx = 1", extra_allowed=None) + assert "auto-removed" in code # import line is commented + assert "x = 1" in code + + def test_extra_allowed_preserves_import_line(self): + """1.7: extra_allowed modules are NOT stripped by _strip_allowed_imports.""" + code, extras = _strip_allowed_imports("import os\nx = 1", extra_allowed=frozenset({"os"})) + assert "auto-removed" not in code + assert "import os" in code + assert "x = 1" in code + + +# --------------------------------------------------------------------------- +# Test Group 2: allowed_builtins +# --------------------------------------------------------------------------- + + +class TestAllowedBuiltins: + """Tests for the `allowed_builtins` orchestration config option.""" + + def test_setattr_with_allowed_builtins(self): + """2.1: `setattr` is available when allowed_builtins includes it.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["setattr"]}, + ) + assert "setattr" in env._safe_builtins + assert env._safe_builtins["setattr"] is setattr + + def test_setattr_without_allowed(self): + """2.2: `setattr` raises NameError when not allowed.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={}, + ) + assert "setattr" not in env._safe_builtins + + def test_property_with_allowed_builtins(self): + """2.3: `property` is available when allowed.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["property"]}, + ) + assert "property" in env._safe_builtins + assert env._safe_builtins["property"] is property + + def test_eval_in_allowed_builtins_raises(self): + """2.4: `eval` in allowed_builtins raises ValueError.""" + with pytest.raises(ValueError, match="dangerous builtin"): + AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["eval"]}, + ) + + def test_open_in_allowed_builtins_raises(self): + """2.5: `open` in allowed_builtins raises ValueError.""" + with pytest.raises(ValueError, match="dangerous builtin"): + AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["open"]}, + ) + + def test_dunder_in_allowed_builtins_raises(self): + """2.6: `__import__` in allowed_builtins raises ValueError.""" + with pytest.raises(ValueError, match="private builtin"): + AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["__import__"]}, + ) + + +# --------------------------------------------------------------------------- +# Test Group 3: allow_classes +# --------------------------------------------------------------------------- + + +class TestAllowClassesSecurityFilter: + """Tests for the `allow_classes` option in SecurityFilter.""" + + def test_class_def_allowed(self): + """3.1: `class A: pass` with allow_classes=True succeeds.""" + sf = SecurityFilter(allow_classes=True) + tree = ast.parse("class A:\n pass", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_class_def_blocked(self): + """3.2: `class A: pass` with allow_classes=False is blocked.""" + assert not _run_security_filter_safe("class A:\n pass") + + def test_class_with_init_allowed(self): + """3.3: __init__ inside class is allowed with allow_classes=True.""" + sf = SecurityFilter(allow_classes=True) + code = "class A:\n def __init__(self):\n pass" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_private_attr_store_inside_class_blocked(self): + """3.4: `self.__x = 1` inside class is still blocked.""" + sf = SecurityFilter(allow_classes=True) + code = "class A:\n def __init__(self):\n self.__x = 1" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_dunder_outside_class_blocked(self): + """3.5: `obj.__class__` at module level blocked even with allow_classes.""" + sf = SecurityFilter(allow_classes=True) + tree = ast.parse("x = obj.__class__", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_evil_dunder_inside_class_blocked(self): + """3.6: __evil__ inside class blocked (not in SAFE_CLASS_DUNDERS).""" + sf = SecurityFilter(allow_classes=True) + code = "class A:\n def __evil__(self):\n pass" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_other_safe_dunders_allowed(self): + """Additional: __str__, __repr__, __iter__ etc. are allowed in class body.""" + for dunder in ["__str__", "__repr__", "__iter__", "__len__", "__enter__", "__exit__"]: + sf = SecurityFilter(allow_classes=True) + code = f"class A:\n def {dunder}(self):\n pass" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump, f"{dunder} should be allowed" + + +# --------------------------------------------------------------------------- +# Test Group 4: disable_security +# --------------------------------------------------------------------------- + + +class TestDisableSecurity: + """Tests for the `disable_security` orchestration config option.""" + + @pytest.mark.asyncio + async def test_eval_with_disable_security(self): + """4.1: `eval("1+1")` with disable_security=True succeeds.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_security": True}, + ) + result = await env.execute("print(eval('1+1'))") + assert "2" in result["results"] + + @pytest.mark.asyncio + async def test_eval_without_disable_security(self): + """4.2: `eval("1+1")` with disable_security=False raises SecurityError.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={}, + ) + result = await env.execute("print(eval('1+1'))") + assert "Security Error" in result["results"] or "forbidden" in result["results"].lower() + + def test_import_os_disable_security(self): + """4.3: `import os` with disable_security=True, no allowed_imports, succeeds.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_security": True}, + ) + # The SecurityFilter is skipped entirely, so import passes AST + # (Runtime exec hit depends on the module being available) + assert env._orchestration_config["disable_security"] is True + + @pytest.mark.asyncio + async def test_dunder_access_disable_security(self): + """4.4: `obj.__class__` with disable_security=True succeeds.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_security": True}, + ) + result = await env.execute("obj = 1\nprint(obj.__class__.__name__)") + assert "int" in result["results"] + + +# --------------------------------------------------------------------------- +# Test Group 5: disable_loop_protection +# --------------------------------------------------------------------------- + + +class TestDisableLoopProtection: + """Tests for the `disable_loop_protection` orchestration config option.""" + + @pytest.mark.asyncio + async def test_while_loop_with_disable_loop_protection(self): + """5.1: while loop with disable_loop_protection=True has no yield injection.""" + code = "while True:\n break" + # Without loop protection, the tree should NOT have __yield + # (LoopYieldInjector is skipped in execute, but here we test directly) + # Default: yield IS injected + lyi = LoopYieldInjector() + with_yield = lyi.visit(ast.parse(code, mode="exec")) + assert _get_loop_yield_count(with_yield) == 1 + + @pytest.mark.asyncio + async def test_env_with_disable_loop_protection(self): + """5.3: for loop with disable_loop_protection=True runs fine.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_loop_protection": True}, + ) + result = await env.execute("for i in range(10):\n print(i)") + assert result["results"] is not None + + +# --------------------------------------------------------------------------- +# Test Group 6: Combination / Integration +# --------------------------------------------------------------------------- + + +class TestOrchestrationIntegration: + """Integration tests combining multiple orchestration config options.""" + + @pytest.mark.asyncio + async def test_combined_imports_and_classes(self): + """6.1: allowed_imports + allow_classes works end-to-end.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={ + "allowed_imports": ["os", "typing"], + "allow_classes": True, + }, + ) + result = await env.execute( + "import os\n" + "import typing\n" + "class A:\n" + " def __init__(self):\n" + " pass\n" + "print('ok')\n" + ) + assert "ok" in result["results"] + + def test_empty_orchestration_config(self): + """6.2: Empty orchestration: {} behaves same as no config.""" + env1 = AgentExecutionEnv(_make_mock_coder(), orchestration_config={}) + env2 = AgentExecutionEnv(_make_mock_coder()) + assert env1._safe_builtins.keys() == env2._safe_builtins.keys() + + def test_context_block_with_config(self): + """6.3: build_orchestration_context_block includes overrides section.""" + config = { + "allow_orchestration": True, + "orchestration": { + "allowed_imports": ["os"], + "allow_classes": True, + }, + } + block = build_orchestration_context_block(config) + assert block is not None + assert "Sandbox Configuration Overrides" in block + assert "os" in block + assert "Class definitions" in block + + def test_context_block_with_empty_config_no_overrides(self): + """6.4: build_orchestration_context_block with empty config unchanged.""" + config = {"allow_orchestration": True} + block = build_orchestration_context_block(config) + assert block is not None + assert "Sandbox Configuration Overrides" not in block + + def test_context_block_with_disable_security_warning(self): + """Context block shows warning for disable_security.""" + config = { + "allow_orchestration": True, + "orchestration": {"disable_security": True}, + } + block = build_orchestration_context_block(config) + assert "Security filtering is DISABLED" in block From 95cfc8b8861c9479164439ea18e2bd37064ff809 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 13:53:23 -0400 Subject: [PATCH 59/68] Fix background command pagination --- cecli/helpers/background_commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/background_commands.py b/cecli/helpers/background_commands.py index 825dd09ad50..83fe53968cd 100644 --- a/cecli/helpers/background_commands.py +++ b/cecli/helpers/background_commands.py @@ -767,6 +767,8 @@ def save_paginated_output( Returns: Tuple of (folder path, rel file paths list, alias paths list), or None if output fits in one page. """ + import math + if not output or len(output) <= page_size: return None @@ -774,7 +776,7 @@ def save_paginated_output( abs_folder = abs_root_path_func(folder_path) os.makedirs(abs_folder, exist_ok=True) - num_pages = (len(output) + page_size - 1) // page_size + num_pages = math.ceil((len(output) + page_size - 1) // page_size) for i in range(num_pages): page_num = i + 1 From 00a1a43de3b48d23e88a13fa31937d01fbb93d93 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 14:29:20 -0400 Subject: [PATCH 60/68] Fix tests --- tests/basic/test_hashline.py | 69 ++++++++++++------------- tests/basic/test_hashline_closure.py | 11 ++-- tests/subagents/test_finished.py | 4 +- tests/tools/test_get_lines.py | 1 + tests/tools/test_git_branch.py | 4 +- tests/tools/test_insert_block.py | 1 + tests/tools/test_read_range_execute.py | 14 ++--- tests/tools/test_remove_mcp_tool.py | 2 +- tests/tools/test_tools_load_mcp_tool.py | 2 +- tests/unit/test_load_mcp.py | 6 +-- tests/unit/test_remove_mcp.py | 2 +- tests/unit/test_unit_remove_mcp_tool.py | 2 +- 12 files changed, 59 insertions(+), 59 deletions(-) diff --git a/tests/basic/test_hashline.py b/tests/basic/test_hashline.py index 527762172e9..f058e8ad4d0 100644 --- a/tests/basic/test_hashline.py +++ b/tests/basic/test_hashline.py @@ -15,14 +15,12 @@ def test_hashline_basic(): lines = result.splitlines() assert len(lines) == 3 - # Check each line has the format "{variable-length-hash}::content" (HashPos format) + # Check each line has HashPos format: unique lines use ~~ prefix, duplicates use ~4char~ for i, line in enumerate(lines, start=1): - # Format should be "{hash}::content" - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - # Check hash fragment is between 3 and 6 chars (3 encoded bytes) - assert 3 <= len(hash_fragment) <= 6 + # Unique lines should use ~~ prefix + assert line.startswith("~~"), f"Line {i} should have ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" def test_hashline_with_start_line(): @@ -32,15 +30,12 @@ def test_hashline_with_start_line(): lines = result.splitlines() assert len(lines) == 2 - # Check format is {variable-length-hash}::content (HashPos format) - # Note: start_line parameter is ignored by HashPos but kept for compatibility + # Check format is HashPos format (start_line is ignored by HashPos) for line in lines: - # Format should be "{hash}::content" - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - # Check hash fragment is between 3 and 6 chars (3 encoded bytes) - assert 3 <= len(hash_fragment) <= 6 + # Unique lines should use ~~ prefix + assert line.startswith("~~"), f"Expected ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" """Test hashline with empty string.""" result = hashline("") assert result == "" @@ -52,13 +47,12 @@ def test_hashline_single_line(): result = hashline(text) lines = result.splitlines() assert len(lines) == 1 - # Check format is {variable-length-hash}::content (HashPos format) + # Check format is HashPos format: ~~content (unique) or ~4char~content (duplicate) line = lines[0] - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - # Check hash fragment is between 3 and 6 chars (3 encoded bytes) - assert 3 <= len(hash_fragment) <= 6 + # Unique line should use ~~ prefix + assert line.startswith("~~"), f"Expected ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" def test_hashline_preserves_newlines(): @@ -69,20 +63,20 @@ def test_hashline_preserves_newlines(): # The result should have hashes on each line but no trailing newline lines = result.splitlines() assert len(lines) == 2 - # Check each line has the correct format + # Check each line has HashPos format for line in lines: - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - assert 3 <= len(hash_fragment) <= 6 + # Unique lines should use ~~ prefix + assert line.startswith("~~"), f"Expected ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" # HashPos doesn't preserve trailing newlines in the formatted output # The splitlines() above verifies we have the right number of lines def test_strip_hashline_basic(): """Test basic strip_hashline functionality.""" - # Create a hashline-formatted text with correct HashPos format: {variable-length-hash}::content - text = "abc::Hello\nbad::World\n1a2::Test" + # Create a hashline-formatted text with HashPos format + text = "~~Hello\n~~World\n~~Test" stripped = strip_hashline(text) assert stripped == "Hello\nWorld\nTest" @@ -91,24 +85,25 @@ def test_strip_hashline_with_negative_line_numbers(): """Test strip_hashline with negative line numbers.""" # HashPos format doesn't support negative line numbers in the prefix # Test with standard HashPos format - text = "abc::Hello\nbad::World\n1a2::Test" + # HashPos format: ~~content (unique) or ~4char~content (duplicate) + text = "~~Hello\n~~World\n~~Test" stripped = strip_hashline(text) assert stripped == "Hello\nWorld\nTest" def test_strip_hashline_mixed_lines(): """Test strip_hashline with mixed hashline and non-hashline lines.""" - # HashPos format: {variable-length-hash}::content + # HashPos format: ~~content (unique) or ~4char~content (duplicate) # Plain lines without hashes should be left unchanged - text = "abc::Hello\nPlain line\nbad::World" + text = "~~Hello\nPlain line\n~~World" stripped = strip_hashline(text) assert stripped == "Hello\nPlain line\nWorld" def test_strip_hashline_preserves_newlines(): """Test that strip_hashline preserves newline characters.""" - # HashPos format: {variable-length-hash}::content - text = "abc::Line 1\nbad::Line 2\n" + # HashPos format: ~~content (unique) or ~4char~content (duplicate) + text = "~~Line 1\n~~Line 2\n" stripped = strip_hashline(text) # strip_hashline should preserve newlines assert stripped == "Line 1\nLine 2\n" @@ -158,14 +153,14 @@ def test_hashline_different_inputs(): def test_parse_hashline(): """Test parse_hashline function.""" # Test basic parsing (HashPos format) - hash_fragment, line_num_str, line_num = parse_hashline("abc") - assert hash_fragment == "abc" + hash_fragment, line_num_str, line_num = parse_hashline("~0abc~") + assert hash_fragment == "~0abc~" assert line_num_str is None # HashPos doesn't include line numbers assert line_num is None # Test with content after hash - hash_fragment, line_num_str, line_num = parse_hashline("bad::Hello World") - assert hash_fragment == "bad" + hash_fragment, line_num_str, line_num = parse_hashline("~bad1~Hello World") + assert hash_fragment == "~bad1~" assert line_num_str is None assert line_num is None diff --git a/tests/basic/test_hashline_closure.py b/tests/basic/test_hashline_closure.py index cd762903909..4399a3470db 100644 --- a/tests/basic/test_hashline_closure.py +++ b/tests/basic/test_hashline_closure.py @@ -483,15 +483,18 @@ def test_apply_closure_safeguard_passes_rejected_ops_to_failed(): Integration test: apply_hashline_operations should propagate rejected operations (from _apply_closure_safeguard) into the failed_ops list. """ - from cecli.helpers.hashline import apply_hashline_operations, hashline + from cecli.helpers.hashline import apply_hashline_operations source = """x = 1 y = 2 """ # Try to replace a line with broken syntax — this should fail via eviction - # Get the correct hashline ID for the first line - hp_content = hashline(source) - first_line_hash = hp_content.splitlines()[0].split("::")[0] + "::" + # Get the correct hashline ID for the first line using HashPos API + # New format uses ~4chars~ (tilde-wrapped 4-char hash), no :: separator + from cecli.helpers.hashpos.hashpos import HashPos + + hp = HashPos(source) + first_line_hash = hp.get_wrapped_id(hp.generate_public_id("x = 1", 0, 1)) ops = [ { diff --git a/tests/subagents/test_finished.py b/tests/subagents/test_finished.py index 0aad7e00527..2555225a91e 100644 --- a/tests/subagents/test_finished.py +++ b/tests/subagents/test_finished.py @@ -60,7 +60,7 @@ async def test_sub_agent_without_summary(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder) - assert result.to_dict()["result"] == "Yielded." + assert result.to_dict()["result"][0]["content"] == "Yielded." @pytest.mark.asyncio async def test_non_sub_agent_skips_lookup(self): @@ -73,7 +73,7 @@ async def test_non_sub_agent_skips_lookup(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder) - assert result.to_dict()["result"] == "Yielded." + assert result.to_dict()["result"][0]["content"] == "Yielded." @pytest.mark.asyncio async def test_unknown_parent_uuid_caught_gracefully(self): diff --git a/tests/tools/test_get_lines.py b/tests/tools/test_get_lines.py index c853d737c00..286e04e2556 100644 --- a/tests/tools/test_get_lines.py +++ b/tests/tools/test_get_lines.py @@ -34,6 +34,7 @@ def __init__(self, root): self.abs_fnames = set() self.abs_read_only_fnames = set() self.large_file_token_threshold = 25000 + self.file_read_cache = set() from unittest.mock import MagicMock diff --git a/tests/tools/test_git_branch.py b/tests/tools/test_git_branch.py index 4d91a2829eb..0d4ddc75ab2 100644 --- a/tests/tools/test_git_branch.py +++ b/tests/tools/test_git_branch.py @@ -29,7 +29,7 @@ def test_gitbranch_show_current_returns_branch_name(): result = git_branch.Tool.execute(coder, show_current=True) - assert result.to_dict()["result"].strip() == "feature" + assert result.to_dict()["result"][0]["content"].strip() == "feature" def test_gitbranch_show_current_handles_detached_head(): @@ -48,4 +48,4 @@ def test_gitbranch_show_current_handles_detached_head(): result = git_branch.Tool.execute(coder, show_current=True) - assert result.to_dict()["result"].strip() == "HEAD (detached)" + assert result.to_dict()["result"][0]["content"].strip() == "HEAD (detached)" diff --git a/tests/tools/test_insert_block.py b/tests/tools/test_insert_block.py index 340f34f914e..856fbaed6c0 100644 --- a/tests/tools/test_insert_block.py +++ b/tests/tools/test_insert_block.py @@ -52,6 +52,7 @@ def __init__(self, root): self.abs_read_only_fnames = set() self.abs_fnames = set() self.edit_allowed = True + self.file_read_cache = set() def abs_root_path(self, file_path): path = Path(file_path) diff --git a/tests/tools/test_read_range_execute.py b/tests/tools/test_read_range_execute.py index 461206c0063..7da45f775d7 100644 --- a/tests/tools/test_read_range_execute.py +++ b/tests/tools/test_read_range_execute.py @@ -169,7 +169,7 @@ def test_both_digits_valid_range( content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "5", "range_end": "10"}] + show = [{"file_path": self.test_file, "range_start": "@L5", "range_end": "@L10"}] result = self.Tool.execute(self.coder, show) assert "line5" in str(result) assert "line10" in str(result) @@ -181,7 +181,7 @@ def test_both_digits_same_line(self, mock_coder, mock_file_context, mock_chunks, content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "1", "range_end": "1"}] + show = [{"file_path": self.test_file, "range_start": "@L1", "range_end": "@L1"}] result = self.Tool.execute(self.coder, show) assert "line1" in str(result) finally: @@ -194,7 +194,7 @@ def test_both_digits_out_of_bounds( content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "1", "range_end": "100"}] + show = [{"file_path": self.test_file, "range_start": "@L1", "range_end": "@L100"}] result = self.Tool.execute(self.coder, show) assert "line1" in str(result) assert "line10" in str(result) @@ -208,7 +208,7 @@ def test_both_digits_inverted_order( content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "10", "range_end": "5"}] + show = [{"file_path": self.test_file, "range_start": "@L10", "range_end": "@L5"}] result = self.Tool.execute(self.coder, show) # Inverted: start=[9], end=[4], only one each -> swap to (4, 9) assert result is not None @@ -264,7 +264,7 @@ def test_special_start_digit_end( content = "line1\nline2\nline3\nline4\nline5" self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "3"}] + show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "@L3"}] result = self.Tool.execute(self.coder, show) assert "line1" in str(result) assert "line3" in str(result) @@ -278,7 +278,7 @@ def test_digit_start_special_end( content = "line1\nline2\nline3\nline4\nline5" self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "2", "range_end": "000@"}] + show = [{"file_path": self.test_file, "range_start": "@L2", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) assert "line2" in str(result) assert "line5" in str(result) @@ -415,7 +415,7 @@ def test_file_not_found(self, mock_coder, mock_file_context, mock_chunks, mock_m from cecli.tools.read_file import Tool - show = [{"file_path": "nonexistent/path.py", "range_start": "1", "range_end": "10"}] + show = [{"file_path": "nonexistent/path.py", "range_start": "@L1", "range_end": "@L10"}] result = Tool.execute(mock_coder, show) assert "not found" in str(result) or "Errors" in str(result) diff --git a/tests/tools/test_remove_mcp_tool.py b/tests/tools/test_remove_mcp_tool.py index 90d84d4e5f4..2da2e85e37c 100644 --- a/tests/tools/test_remove_mcp_tool.py +++ b/tests/tools/test_remove_mcp_tool.py @@ -64,7 +64,7 @@ async def test_no_configured_servers(self, coder): """Test when no MCP servers are configured at all.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result.to_dict()["result"] == "No MCP servers are configured." + assert result.to_dict()["result"][0]["content"] == "No MCP servers are configured." @pytest.mark.asyncio async def test_server_not_found(self, coder, mock_server): diff --git a/tests/tools/test_tools_load_mcp_tool.py b/tests/tools/test_tools_load_mcp_tool.py index 48e559a4d38..6e9071cb711 100644 --- a/tests/tools/test_tools_load_mcp_tool.py +++ b/tests/tools/test_tools_load_mcp_tool.py @@ -55,7 +55,7 @@ async def test_no_mcp_servers_found(self, coder): """Test when no MCP servers are configured.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, load_mcp=["test"]) - assert result.to_dict()["result"] == "No MCP servers found, nothing to load." + assert result.to_dict()["result"][0]["content"] == "No MCP servers found, nothing to load." @pytest.mark.asyncio async def test_server_not_found(self, coder, mock_server): diff --git a/tests/unit/test_load_mcp.py b/tests/unit/test_load_mcp.py index 441207eaf78..17380a33252 100644 --- a/tests/unit/test_load_mcp.py +++ b/tests/unit/test_load_mcp.py @@ -51,7 +51,7 @@ async def test_no_mcp_servers_found(coder): """Test when no MCP servers are configured.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, load_mcp=["test"]) - assert result.to_dict()["result"] == "No MCP servers found, nothing to load." + assert result.to_dict()["result"][0]["content"] == "No MCP servers found, nothing to load." @pytest.mark.asyncio @@ -93,8 +93,8 @@ async def test_server_not_enabled_by_default(coder, mock_server): coder.mcp_manager.connect_server = AsyncMock() result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) # Non-enabled servers are silently filtered by wildcard expansion - assert result.to_dict()["result"] == "" - coder.mcp_manager.connect_server.assert_not_called() + # Result is an empty list since nothing was loaded + assert result.to_dict()["result"] == [] @pytest.mark.asyncio diff --git a/tests/unit/test_remove_mcp.py b/tests/unit/test_remove_mcp.py index d74096926bc..2ac873697de 100644 --- a/tests/unit/test_remove_mcp.py +++ b/tests/unit/test_remove_mcp.py @@ -203,7 +203,7 @@ async def test_remove_mcp_tool_no_servers_configured(): coder.mcp_manager = MagicMock() coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result.to_dict()["result"] == "No MCP servers are configured." + assert result.to_dict()["result"][0]["content"] == "No MCP servers are configured." @pytest.mark.asyncio diff --git a/tests/unit/test_unit_remove_mcp_tool.py b/tests/unit/test_unit_remove_mcp_tool.py index 31c2fbd356c..a5ab9e49dc9 100644 --- a/tests/unit/test_unit_remove_mcp_tool.py +++ b/tests/unit/test_unit_remove_mcp_tool.py @@ -218,7 +218,7 @@ async def test_remove_mcp_tool_no_servers_configured(): result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result.to_dict()["result"] == "No MCP servers are configured." + assert result.to_dict()["result"][0]["content"] == "No MCP servers are configured." @pytest.mark.asyncio From b3fa8a67215efa950c078bfdcb1fdfb723826904 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 17:49:55 -0400 Subject: [PATCH 61/68] Change delegate format_output --- cecli/tools/delegate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cecli/tools/delegate.py b/cecli/tools/delegate.py index 80ede54cb79..bd27fef5901 100644 --- a/cecli/tools/delegate.py +++ b/cecli/tools/delegate.py @@ -184,11 +184,11 @@ def format_output(cls, coder, mcp_server, tool_response): for i, d in enumerate(delegations): name = d.get("name", "") prompt = d.get("prompt", "") + is_async = d.get("async", True) coder.io.tool_output(f"{color_start}delegation_{i + 1}:{color_end}") coder.io.tool_output(f"agent: {name}") - coder.io.tool_output(f"task: {prompt}") - is_async = d.get("async", True) coder.io.tool_output(f"mode: {'async' if is_async else 'sync'}") + coder.io.tool_output(f"task: {prompt}") if i < len(delegations) - 1: coder.io.tool_output("") From b6b2883dffdeb48cd6f2935cdc64f27394925ff6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 17:50:06 -0400 Subject: [PATCH 62/68] Remov duplicate print --- cecli/hooks/registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cecli/hooks/registry.py b/cecli/hooks/registry.py index d58a60b824c..cff8dfab3da 100644 --- a/cecli/hooks/registry.py +++ b/cecli/hooks/registry.py @@ -149,7 +149,6 @@ def load_hooks_from_config(self, config_file: Path) -> List[str]: loaded_hooks.append(hook.name) else: print(f"Warning: Could not register hook '{hook.name}': {e}") - print(f"Warning: Could not register hook '{hook.name}': {e}") return loaded_hooks From 898d1fac51b39c298f49969fc613017e5544712f Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 17:54:07 -0400 Subject: [PATCH 63/68] GitPython is not threadsafe so force it to be --- cecli/coders/base_coder.py | 14 +- cecli/commands/undo.py | 15 +- cecli/helpers/lock_detect.py | 54 +++ cecli/main.py | 32 +- cecli/repo.py | 670 +++++++++++++++++++++++------------ 5 files changed, 541 insertions(+), 244 deletions(-) create mode 100644 cecli/helpers/lock_detect.py diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index c0b64866153..3b8883f04a8 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -55,7 +55,7 @@ remove_reasoning_content, replace_reasoning_tags, ) -from cecli.repo import ANY_GIT_ERROR, GitRepo +from cecli.repo import ANY_GIT_ERROR, GitRepo, GitRepoProxy from cecli.repomap import RepoMap from cecli.report import update_error_prefix from cecli.run_cmd import run_cmd_async @@ -571,11 +571,13 @@ def __init__( self.repo = repo if use_git and self.repo is None: try: - self.repo = GitRepo( - self.io, - fnames, - None, - models=main_model.commit_message_models(), + self.repo = GitRepoProxy( + GitRepo( + self.io, + fnames, + None, + models=main_model.commit_message_models(), + ) ) except FileNotFoundError: pass diff --git a/cecli/commands/undo.py b/cecli/commands/undo.py index 66b02c24f11..60f9acad929 100644 --- a/cecli/commands/undo.py +++ b/cecli/commands/undo.py @@ -53,8 +53,12 @@ async def _raw_cmd_undo(cls, io, coder, args): ) return format_command_result(io, "undo", "Commit has multiple parents") - prev_commit = last_commit.parents[0] - changed_files_last_commit = [item.a_path for item in last_commit.diff(prev_commit)] + prev_sha = last_commit.parents[0] + # Get list of files changed between parent and current commit + diff_output = coder.repo.repo.git.diff("--name-only", prev_sha, last_commit.hexsha) + changed_files_last_commit = ( + [f for f in diff_output.splitlines() if f] if diff_output else [] + ) for fname in changed_files_last_commit: if coder.repo.repo.is_dirty(path=fname): @@ -64,9 +68,12 @@ async def _raw_cmd_undo(cls, io, coder, args): return format_command_result(io, "undo", f"File {fname} has uncommitted changes") # Check if the file was in the repo in the previous commit + # git ls-tree returns empty string if the file does not exist in that commit try: - prev_commit.tree[fname] - except KeyError: + tree_output = coder.repo.repo.git.ls_tree(prev_sha, "--", fname) + if not tree_output: + raise KeyError + except (KeyError, ANY_GIT_ERROR): io.tool_error( f"The file {fname} was not in the repository in the previous commit. Cannot" " undo safely." diff --git a/cecli/helpers/lock_detect.py b/cecli/helpers/lock_detect.py new file mode 100644 index 00000000000..6eecd852c44 --- /dev/null +++ b/cecli/helpers/lock_detect.py @@ -0,0 +1,54 @@ +import logging +import sys +import threading +import time +import traceback + +# Set up logging to catch asyncio framework logs +logging.basicConfig(level=logging.INFO) +logging.getLogger("asyncio").setLevel(logging.DEBUG) + + +def dump_stacks_to_file(filename="hang_dump.log", interval=5, max_prints=10): + """Periodically writes stack traces to a file, resetting it after max_prints.""" + print_count = 0 + + while True: + time.sleep(interval) + try: + # Determine mode: "w" to overwrite/clear on the 11th print, "a" to append + mode = "w" if print_count >= max_prints else "a" + + with open(filename, mode, encoding="utf-8") as f: + if mode == "w": + f.write(f"--- Log reset automatically after {max_prints} prints ---\n") + print_count = 0 # Reset the counter + + timestamp = time.strftime("%Y-%m-%d %H:%M:%S") + f.write( + f"\n=================== STACK TRACE DUMP ({timestamp}) ===================\n" + ) + + for thread_id, frame in sys._current_frames().items(): + thread_obj = threading._active.get(thread_id) + thread_name = thread_obj.name if thread_obj else "Unknown Thread" + + f.write(f"\nThread Name: {thread_name} (ID: {thread_id})\n") + f.write("-" * 40 + "\n") + traceback.print_stack(frame, file=f) + + f.write("=" * 70 + "\n") + + print_count += 1 # Increment after a successful write + + except Exception as e: + print(f"Error writing stack dump to file: {e}", file=sys.stderr) + + +# Start the monitor in a background daemon thread +monitor_thread = threading.Thread( + target=dump_stacks_to_file, + args=("program_hangs.log", 5, 10), # Clears every 10 prints + daemon=True, +) +monitor_thread.start() diff --git a/cecli/main.py b/cecli/main.py index 71e31e075fe..e49733b1934 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -554,7 +554,7 @@ async def main_async( from cecli.mcp import McpServerManager, load_mcp_servers from cecli.models import ModelSettings from cecli.onboarding import offer_openrouter_oauth, select_default_model - from cecli.repo import GitRepo + from cecli.repo import GitRepo, GitRepoProxy from cecli.report import report_uncaught_exceptions, set_args_error_data from cecli.versioncheck import check_version from cecli.watch import FileWatcher @@ -1041,20 +1041,22 @@ def get_io(pretty): repo = None if args.git: try: - repo = GitRepo( - io, - fnames, - git_dname, - args.cecli_ignore, - models=main_model.commit_message_models(), - attribute_author=args.attribute_author, - attribute_committer=args.attribute_committer, - attribute_commit_message_author=args.attribute_commit_message_author, - attribute_commit_message_committer=args.attribute_commit_message_committer, - commit_prompt=args.commit_prompt, - subtree_only=args.subtree_only, - git_commit_verify=args.git_commit_verify, - attribute_co_authored_by=args.attribute_co_authored_by, + repo = GitRepoProxy( + GitRepo( + io, + fnames, + git_dname, + args.cecli_ignore, + models=main_model.commit_message_models(), + attribute_author=args.attribute_author, + attribute_committer=args.attribute_committer, + attribute_commit_message_author=args.attribute_commit_message_author, + attribute_commit_message_committer=args.attribute_commit_message_committer, + commit_prompt=args.commit_prompt, + subtree_only=args.subtree_only, + git_commit_verify=args.git_commit_verify, + attribute_co_authored_by=args.attribute_co_authored_by, + ) ) except FileNotFoundError: pass diff --git a/cecli/repo.py b/cecli/repo.py index baec1592a99..d0170d762a9 100644 --- a/cecli/repo.py +++ b/cecli/repo.py @@ -17,6 +17,9 @@ git = None ANY_GIT_ERROR = [] +import concurrent.futures +import threading + import pathspec import cecli.prompts.utils.system as prompts @@ -52,6 +55,25 @@ def set_git_env(var_name, value, original_value): del os.environ[var_name] +class CommitInfo: + """ + Data-only container for extracted commit information. + + Wraps the raw git-python commit data into simple Python types + so that callers never receive a reference to the underlying + ``git.Commit`` object (which could otherwise be used outside + of the repository lock, leading to deadlocks). + """ + + def __init__(self, hexsha: str, message: str, parents: tuple[str, ...]): + self.hexsha = hexsha + self.message = message + self.parents = parents + + def __bool__(self): + return True + + class GitRepo: repo = None cecli_ignore_file = None @@ -104,6 +126,7 @@ def __init__( self.workspace_layout = "clone" self.workspace_ignore_specs = {} self.workspace_ignore_ts = {} + self._git_lock = threading.RLock() # Workspace detection and config loading occurs later in __init__ if git_dname: @@ -171,21 +194,21 @@ def __init__( self.cecli_ignore_file = Path(cecli_ignore_file) def init_repo(self): - if not self.repo: - self.repo = git.Repo(self._init_repo_path, odbt=git.GitCmdObjectDB) - self.root = utils.safe_abs_path(self.repo.working_tree_dir) - - if self.is_workspace: - self.root = self.workspace_path - - try: - commit = self.repo.head.commit - return commit - except ANY_GIT_ERROR: - if not self.is_workspace: + with self._git_lock: + if not self.repo: self.repo = git.Repo(self._init_repo_path, odbt=git.GitCmdObjectDB) self.root = utils.safe_abs_path(self.repo.working_tree_dir) + if self.is_workspace: + self.root = self.workspace_path + + try: + self.repo.head.commit # just access to check for errors, discard + except ANY_GIT_ERROR: + if not self.is_workspace: + self.repo = git.Repo(self._init_repo_path, odbt=git.GitCmdObjectDB) + self.root = utils.safe_abs_path(self.repo.working_tree_dir) + def _load_workspace_config(self) -> None: from cecli.helpers.monorepo.config import ( load_workspace_config, @@ -245,8 +268,9 @@ def _detect_workspace_path(self, start_path: str): return None def __del__(self): - if self.repo: - self.repo.close() + with self._git_lock: + if self.repo: + self.repo.close() async def commit(self, fnames=None, context=None, message=None, coder_edits=False, coder=None): """ @@ -321,12 +345,13 @@ async def commit(self, fnames=None, context=None, message=None, coder_edits=Fals if self.is_workspace and getattr(self, "workspace_layout", "clone") == "local": return await self._commit_local_workspace(fnames, context, message, coder_edits, coder) - if not fnames and not self.repo.is_dirty(): - return + with self._git_lock: + if not fnames and not self.repo.is_dirty(): + return - diffs = self.get_diffs(fnames) - if not diffs: - return + diffs = self.get_diffs(fnames) + if not diffs: + return if message: commit_message = message @@ -397,70 +422,72 @@ async def commit(self, fnames=None, context=None, message=None, coder_edits=Fals full_commit_message = commit_message + commit_message_trailer - cmd = ["-m", full_commit_message] - if not self.git_commit_verify: - cmd.append("--no-verify") - if fnames: - fnames = [str(self.abs_root_path(fn)) for fn in fnames] - added_fnames = [] - for fname in fnames: - try: - # Check if file is git-ignored before trying to add - if ( - coder - and hasattr(coder, "add_gitignore_files") - and coder.add_gitignore_files - ): - rel_fname = self.get_rel_fname(fname) - if self.git_ignored_file(rel_fname): - # Skip git-ignored files when add_gitignore_files is enabled - continue - self.repo.git.add(fname) - added_fnames.append(fname) - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to add {fname}: {err}") - if added_fnames: - cmd += ["--"] + added_fnames + with self._git_lock: + cmd = ["-m", full_commit_message] + if not self.git_commit_verify: + cmd.append("--no-verify") + if fnames: + fnames = [str(self.abs_root_path(fn)) for fn in fnames] + added_fnames = [] + for fname in fnames: + try: + # Check if file is git-ignored before trying to add + if ( + coder + and hasattr(coder, "add_gitignore_files") + and coder.add_gitignore_files + ): + rel_fname = self.get_rel_fname(fname) + if self.git_ignored_file(rel_fname): + # Skip git-ignored files when add_gitignore_files is enabled + continue + self.repo.git.add(fname) + added_fnames.append(fname) + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to add {fname}: {err}") + if added_fnames: + cmd += ["--"] + added_fnames + else: + # No files to commit (all were git-ignored or failed to add) + return else: - # No files to commit (all were git-ignored or failed to add) - return - else: - cmd += ["-a"] + cmd += ["-a"] - original_user_name = self.repo.git.config("--get", "user.name") - original_committer_name_env = os.environ.get("GIT_COMMITTER_NAME") - original_author_name_env = os.environ.get("GIT_AUTHOR_NAME") - committer_name = f"{original_user_name} (cecli)" + original_user_name = self.repo.git.config("--get", "user.name") + original_committer_name_env = os.environ.get("GIT_COMMITTER_NAME") + original_author_name_env = os.environ.get("GIT_AUTHOR_NAME") + committer_name = f"{original_user_name} (cecli)" - try: - # Use context managers to handle environment variables - with contextlib.ExitStack() as stack: - if use_attribute_committer: - stack.enter_context( - set_git_env( - "GIT_COMMITTER_NAME", committer_name, original_committer_name_env + try: + # Use context managers to handle environment variables + with contextlib.ExitStack() as stack: + if use_attribute_committer: + stack.enter_context( + set_git_env( + "GIT_COMMITTER_NAME", committer_name, original_committer_name_env + ) + ) + if use_attribute_author: + stack.enter_context( + set_git_env("GIT_AUTHOR_NAME", committer_name, original_author_name_env) ) - ) - if use_attribute_author: - stack.enter_context( - set_git_env("GIT_AUTHOR_NAME", committer_name, original_author_name_env) - ) - # Perform the commit - self.repo.git.commit(cmd) - commit_hash = self.get_head_commit_sha(short=True) - self.io.tool_success(f"Commit {commit_hash} {commit_message}") - return commit_hash, commit_message + # Perform the commit + self.repo.git.commit(cmd) + commit_hash = self.get_head_commit_sha(short=True) + self.io.tool_success(f"Commit {commit_hash} {commit_message}") + return commit_hash, commit_message - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to commit: {err}") - # No return here, implicitly returns None + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to commit: {err}") + # No return here, implicitly returns None def get_rel_repo_dir(self): - try: - return os.path.relpath(self.repo.git_dir, os.getcwd()) - except (ValueError, OSError): - return self.repo.git_dir + with self._git_lock: + try: + return os.path.relpath(self.repo.git_dir, os.getcwd()) + except (ValueError, OSError): + return self.repo.git_dir def get_rel_fname(self, fname): try: @@ -530,126 +557,129 @@ async def get_commit_message(self, diffs, context, user_language=None): def get_diffs(self, fnames=None): # We always want diffs of index and working dir - current_branch_has_commits = False - try: - active_branch = self.repo.active_branch + with self._git_lock: + current_branch_has_commits = False try: - commits = self.repo.iter_commits(active_branch) - current_branch_has_commits = any(commits) - except ANY_GIT_ERROR: + active_branch = self.repo.active_branch + try: + commits = self.repo.iter_commits(active_branch) + current_branch_has_commits = any(commits) + except ANY_GIT_ERROR: + pass + except (TypeError,) + ANY_GIT_ERROR: pass - except (TypeError,) + ANY_GIT_ERROR: - pass - if not fnames: - fnames = [] + if not fnames: + fnames = [] - diffs = "" - for fname in fnames: - if not self.path_in_repo(fname): - diffs += f"Added {fname}\n" + diffs = "" + for fname in fnames: + if not self.path_in_repo(fname): + diffs += f"Added {fname}\n" - try: - if current_branch_has_commits: - args = ["HEAD", "--"] + list(fnames) - diffs += self.repo.git.diff(*args, stdout_as_string=False).decode( + try: + if current_branch_has_commits: + args = ["HEAD", "--"] + list(fnames) + diffs += self.repo.git.diff(*args, stdout_as_string=False).decode( + self.io.encoding, "replace" + ) + return diffs + + wd_args = ["--"] + list(fnames) + index_args = ["--cached"] + wd_args + + diffs += self.repo.git.diff(*index_args, stdout_as_string=False).decode( + self.io.encoding, "replace" + ) + diffs += self.repo.git.diff(*wd_args, stdout_as_string=False).decode( self.io.encoding, "replace" ) + return diffs + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to diff: {err}") - wd_args = ["--"] + list(fnames) - index_args = ["--cached"] + wd_args + def diff_commits(self, pretty, from_commit, to_commit=None): + with self._git_lock: + args = [] + if pretty: + args += ["--color"] + else: + args += ["--color=never"] - diffs += self.repo.git.diff(*index_args, stdout_as_string=False).decode( - self.io.encoding, "replace" - ) - diffs += self.repo.git.diff(*wd_args, stdout_as_string=False).decode( + if to_commit is not None: + args += [from_commit, to_commit] + else: + args += [from_commit] + diffs = self.repo.git.diff(*args, stdout_as_string=False).decode( self.io.encoding, "replace" ) return diffs - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to diff: {err}") - - def diff_commits(self, pretty, from_commit, to_commit=None): - args = [] - if pretty: - args += ["--color"] - else: - args += ["--color=never"] - - if to_commit is not None: - args += [from_commit, to_commit] - else: - args += [from_commit] - diffs = self.repo.git.diff(*args, stdout_as_string=False).decode( - self.io.encoding, "replace" - ) - - return diffs def get_tracked_files(self): - if not self.repo: - return [] - - self.init_repo() + with self._git_lock: + if not self.repo: + return [] - try: - commit = self.repo.head.commit - except ValueError: - commit = None - except ANY_GIT_ERROR as err: - self.git_repo_error = err - self.io.tool_error(f"Unable to list files in git repo: {err}") - self.io.tool_output("Is your git repo corrupted?") - return [] + self.init_repo() - files = set() - if commit: - if self._tree_cache is not None and self._tree_cache[0] == commit.hexsha: - files = self._tree_cache[1] - else: - try: - iterator = commit.tree.traverse() - blob = None # Initialize blob - while True: - try: - blob = next(iterator) - if blob.type == "blob": # blob is a file - # Use sys.intern() to deduplicate path strings in memory - files.add(sys.intern(blob.path)) - except IndexError: - # Handle potential index error during tree traversal - # without relying on potentially unassigned 'blob' - self.io.tool_warning( - "GitRepo: Index error encountered while reading git tree object." - " Skipping." - ) - continue - except StopIteration: - break - except ANY_GIT_ERROR as err: - self.git_repo_error = err - self.io.tool_error(f"Unable to list files in git repo: {err}") - self.io.tool_output("Is your git repo corrupted?") - return [] - files = set(self.normalize_path(path) for path in files) - # Use single-entry cache (not per-commit dict) to limit memory growth - # Store only the SHA string (not the Commit object) to avoid retaining - # the entire git object graph (tree, blobs, parent commits, etc.) - self._tree_cache = (commit.hexsha, files) - - # Add staged files - index = self.repo.index - try: - staged_files = [path for path, _ in index.entries.keys()] - files.update(self.normalize_path(path) for path in staged_files) - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to read staged files: {err}") + try: + commit = self.repo.head.commit + except ValueError: + commit = None + except ANY_GIT_ERROR as err: + self.git_repo_error = err + self.io.tool_error(f"Unable to list files in git repo: {err}") + self.io.tool_output("Is your git repo corrupted?") + return [] + + files = set() + if commit: + if self._tree_cache is not None and self._tree_cache[0] == commit.hexsha: + files = self._tree_cache[1] + else: + try: + iterator = commit.tree.traverse() + blob = None # Initialize blob + while True: + try: + blob = next(iterator) + if blob.type == "blob": # blob is a file + # Use sys.intern() to deduplicate path strings in memory + files.add(sys.intern(blob.path)) + except IndexError: + # Handle potential index error during tree traversal + # without relying on potentially unassigned 'blob' + self.io.tool_warning( + "GitRepo: Index error encountered while reading git tree object." + " Skipping." + ) + continue + except StopIteration: + break + except ANY_GIT_ERROR as err: + self.git_repo_error = err + self.io.tool_error(f"Unable to list files in git repo: {err}") + self.io.tool_output("Is your git repo corrupted?") + return [] + files = set(self.normalize_path(path) for path in files) + # Use single-entry cache (not per-commit dict) to limit memory growth + # Store only the SHA string (not the Commit object) to avoid retaining + # the entire git object graph (tree, blobs, parent commits, etc.) + self._tree_cache = (commit.hexsha, files) + + # Add staged files + index = self.repo.index + try: + staged_files = [path for path, _ in index.entries.keys()] + files.update(self.normalize_path(path) for path in staged_files) + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to read staged files: {err}") - res = [fname for fname in files if not self.ignored_file(fname)] + res = [fname for fname in files if not self.ignored_file(fname)] - return res + return res async def _commit_local_workspace( self, fnames=None, context=None, message=None, coder_edits=False, coder=None @@ -1025,16 +1055,17 @@ def get_non_ignored_files_from_root(self): if not self.cecli_ignore_spec: return [] - try: - all_files = self.repo.git.ls_files( - "--others", "--cached", f"--exclude-from={str(self.cecli_ignore_file)}" - ).splitlines() - - return [f for f in all_files if not self.ignored_file(f)] - except Exception as e: - # Fall back to empty set if there's an error - self.io.tool_warning(f"Error getting ignored files from root: {e}") - return [] + with self._git_lock: + try: + all_files = self.repo.git.ls_files( + "--others", "--cached", f"--exclude-from={str(self.cecli_ignore_file)}" + ).splitlines() + + return [f for f in all_files if not self.ignored_file(f)] + except Exception as e: + # Fall back to empty set if there's an error + self.io.tool_warning(f"Error getting ignored files from root: {e}") + return [] def get_repo_files(self) -> list[str]: """ @@ -1070,19 +1101,20 @@ def get_cache_key(self) -> str: Returns: SHA-256 hex digest, or empty string if repo is unavailable """ - import hashlib + with self._git_lock: + import hashlib - if not self.repo: - return "" + if not self.repo: + return "" - sha = self.get_head_commit_sha() or "" - try: - staged = [item.a_path for item in self.repo.index.diff("HEAD")] - except ANY_GIT_ERROR: - staged = [] + sha = self.get_head_commit_sha() or "" + try: + staged = [item.a_path for item in self.repo.index.diff("HEAD")] + except ANY_GIT_ERROR: + staged = [] - combined = sha + "|" + "".join(sorted(staged)) - return hashlib.sha256(combined.encode()).hexdigest() + combined = sha + "|" + "".join(sorted(staged)) + return hashlib.sha256(combined.encode()).hexdigest() def path_in_repo(self, path): if not self.repo: @@ -1115,40 +1147,240 @@ def get_dirty_files(self): Returns a list of all files which are dirty (not committed), either staged or in the working directory. """ - dirty_files = set() + with self._git_lock: + dirty_files = set() - # Get staged files - staged_files = self.repo.git.diff("--name-only", "--cached").splitlines() - dirty_files.update(staged_files) + # Get staged files + staged_files = self.repo.git.diff("--name-only", "--cached").splitlines() + dirty_files.update(staged_files) - # Get unstaged files - unstaged_files = self.repo.git.diff("--name-only").splitlines() - dirty_files.update(unstaged_files) + # Get unstaged files + unstaged_files = self.repo.git.diff("--name-only").splitlines() + dirty_files.update(unstaged_files) - return list(dirty_files) + return list(dirty_files) def is_dirty(self, path=None): - if path and not self.path_in_repo(path): - return True + with self._git_lock: + if path and not self.path_in_repo(path): + return True - return self.repo.is_dirty(path=path) + return self.repo.is_dirty(path=path) def get_head_commit(self): + with self._git_lock: + try: + commit = self.repo.head.commit + return CommitInfo( + hexsha=commit.hexsha, + message=commit.message, + parents=tuple(p.hexsha for p in commit.parents), + ) + except (ValueError,) + ANY_GIT_ERROR: + return None + + def get_head_commit_sha(self, short=False): + with self._git_lock: + try: + commit = self.repo.head.commit + except (ValueError,) + ANY_GIT_ERROR: + return + if not commit: + return + if short: + return commit.hexsha[:7] + return commit.hexsha + + def get_head_commit_message(self, default=None): + with self._git_lock: + try: + commit = self.repo.head.commit + except (ValueError,) + ANY_GIT_ERROR: + return default + if not commit: + return default + return commit.message + + +class _GitObjectProxy: + """Routes all operations on a git sub-object through a shared single-thread executor. + + Wraps objects like ``git.Repo.git``, ``git.Repo.head``, ``git.Repo.index`` + so that every method call and attribute access is dispatched on the + executor thread, preventing deadlocks when the underlying ``git.Repo`` + is accessed from multiple asyncio workers. + + Iterators returned by git-python methods are eagerly materialised into + lists so that iteration happens on the executor thread, not the caller's. + """ + + def __init__(self, obj, executor): + object.__setattr__(self, "_obj", obj) + object.__setattr__(self, "_executor", executor) + + def __getattr__(self, name): + attr = getattr(self._obj, name) + + # Recursively wrap git-python objects (check FIRST since git.cmd.Git + # is callable via __call__, but we want chained access like + # ``repo.git.status(...)`` to stay on the executor thread). + cls = type(attr) + if cls.__module__ and cls.__module__.startswith("git"): + return _GitObjectProxy(attr, self._executor) + + if callable(attr): + + def proxy_fn(*args, **kwargs): + future = self._executor.submit(attr, *args, **kwargs) + result = future.result() + # Materialise iterators eagerly in the executor thread so + # iteration never touches git objects from the caller's thread. + if hasattr(result, "__iter__") and not isinstance(result, (str, bytes, dict)): + try: + return list(result) + except TypeError: + pass + return result + + return proxy_fn + + return attr + + +class GitRepoProxy: + """Thread-safe proxy for ``GitRepo`` that serialises all git operations + through a single dedicated thread. + + Prevents deadlocks by ensuring only one thread ever touches the + underlying ``git.Repo`` at any time. Follows the same delegation + pattern as ``IOProxy``: + + * Sync methods that access ``self.repo`` are explicitly dispatched + to the executor and block on the result. + * Non-git attributes and methods are forwarded transparently via + ``__getattr__`` / ``__setattr__``. + * Access to ``.repo`` (the raw ``git.Repo``) returns a + ``_GitObjectProxy`` that routes every attribute and call through + the same executor, so even callers that reach through to the + underlying git object stay thread-safe. + + Usage:: + + repo = GitRepoProxy(GitRepo(io, fnames, git_dname)) + files = repo.get_tracked_files() # runs on executor thread + sha = repo.get_head_commit_sha() # runs on executor thread + """ + + def __init__(self, target): + self._target = target + self._executor = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="git-repo" + ) + + # ------------------------------------------------------------------ + # Sync methods that access self.repo (the git.Repo object) + # ------------------------------------------------------------------ + + def init_repo(self): + return self._executor.submit(self._target.init_repo).result() + + @classmethod + def unwrap(cls, repo): + """Unwrap a ``GitRepoProxy`` to get the underlying ``GitRepo``. + + If *repo* is already a ``GitRepo`` (not a proxy) it is returned + unchanged. This mirrors ``IOProxy.unwrap()`` and prevents nested + proxy chains during coder switching (``SwitchCoderSignal``). + + Usage:: + + raw = GitRepoProxy.unwrap(maybe_proxied_repo) + """ + return repo._target if isinstance(repo, cls) else repo + + def __del__(self): try: - return self.repo.head.commit - except (ValueError,) + ANY_GIT_ERROR: - return None + if hasattr(self, "_target") and self._target is not None: + self._executor.submit(self._target.__del__).result(timeout=5) + except Exception: + pass + + def get_rel_repo_dir(self): + return self._executor.submit(self._target.get_rel_repo_dir).result() + + def get_diffs(self, fnames=None): + return self._executor.submit(self._target.get_diffs, fnames).result() + + def diff_commits(self, pretty, from_commit, to_commit=None): + return self._executor.submit( + self._target.diff_commits, pretty, from_commit, to_commit + ).result() + + def get_tracked_files(self): + return self._executor.submit(self._target.get_tracked_files).result() + + def path_in_repo(self, path): + return self._executor.submit(self._target.path_in_repo, path).result() + + def get_dirty_files(self): + return self._executor.submit(self._target.get_dirty_files).result() + + def is_dirty(self, path=None): + return self._executor.submit(self._target.is_dirty, path).result() + + def get_head_commit(self): + return self._executor.submit(self._target.get_head_commit).result() def get_head_commit_sha(self, short=False): - commit = self.get_head_commit() - if not commit: - return - if short: - return commit.hexsha[:7] - return commit.hexsha + return self._executor.submit(self._target.get_head_commit_sha, short).result() def get_head_commit_message(self, default=None): - commit = self.get_head_commit() - if not commit: - return default - return commit.message + return self._executor.submit(self._target.get_head_commit_message, default).result() + + def get_cache_key(self): + return self._executor.submit(self._target.get_cache_key).result() + + def git_ignored_file(self, path): + return self._executor.submit(self._target.git_ignored_file, path).result() + + def get_non_ignored_files_from_root(self): + return self._executor.submit(self._target.get_non_ignored_files_from_root).result() + + def get_repo_files(self): + return self._executor.submit(self._target.get_repo_files).result() + + def get_workspace_files(self): + return self._executor.submit(self._target.get_workspace_files).result() + + # ------------------------------------------------------------------ + # Async methods – called from the main asyncio task only, and + # already protected by ``_git_lock`` on the target; we do *not* + # proxy them through the executor because they mix git operations + # with LLM calls (``get_commit_message``) that must stay async. + # ------------------------------------------------------------------ + + # commit() and _commit_local_workspace() are forwarded via __getattr__ + + # ------------------------------------------------------------------ + # Access to the raw ``git.Repo`` – return a sub-proxy that routes + # every call through the same executor, so callers that reach + # through to the underlying git object stay thread-safe. + # ------------------------------------------------------------------ + + @property + def repo(self): + raw = self._target.repo + return _GitObjectProxy(raw, self._executor) if raw is not None else None + + # ------------------------------------------------------------------ + # Non-git attributes – transparent forwarding + # ------------------------------------------------------------------ + + def __getattr__(self, name): + return getattr(self._target, name) + + def __setattr__(self, name, value): + if name.startswith("_") or name in ("_target", "_executor"): + super().__setattr__(name, value) + else: + setattr(self._target, name, value) From c18a53fa79f73c6890551fe8c6aadb6559fa1556 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 18:46:47 -0400 Subject: [PATCH 64/68] Update keep alive tests --- tests/mcp/test_keepalive_integration.py | 35 ++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/tests/mcp/test_keepalive_integration.py b/tests/mcp/test_keepalive_integration.py index f94d343052c..47a6fb615ba 100644 --- a/tests/mcp/test_keepalive_integration.py +++ b/tests/mcp/test_keepalive_integration.py @@ -68,10 +68,18 @@ async def test_server_failure_triggers_unhealthy_state( running_mock_server.set_status(500) # Wait for failed ping - await asyncio.sleep(1.2) + # Wait for failed ping with polling for cross-platform reliability + for _ in range(30): # 30 * 0.2s = 6s timeout + if inspector.get_state(server) == ConnectionState.UNHEALTHY: + break + await asyncio.sleep(0.2) + else: + pytest.fail( + f"Server did not become UNHEALTHY after failures, state was " + f"{inspector.get_state(server).name}" + ) # Should transition to UNHEALTHY - assert inspector.get_state(server) == ConnectionState.UNHEALTHY assert inspector.get_failed_pings(server) == 1 await server.disconnect() @@ -115,14 +123,33 @@ async def test_successful_ping_after_failure_restores_healthy_state( # Cause a failure running_mock_server.set_status(500) - await asyncio.sleep(1.2) + for _ in range(30): # 30 * 0.2s = 6s timeout + if inspector.get_state(server) == ConnectionState.UNHEALTHY: + break + await asyncio.sleep(0.2) + else: + pytest.fail( + f"Server did not become UNHEALTHY after failures, state was " + f"{inspector.get_state(server).name}" + ) assert inspector.get_state(server) == ConnectionState.UNHEALTHY # Restore success running_mock_server.set_status(200) - await asyncio.sleep(1.2) + for _ in range(30): # 30 * 0.2s = 6s timeout + if inspector.get_state(server) == ConnectionState.CONNECTED: + break + await asyncio.sleep(0.2) + else: + pytest.fail( + f"Server did not become CONNECTED after success, state was " + f"{inspector.get_state(server).name}" + ) # Should be back to CONNECTED + + # Brief cooldown after state transition + await asyncio.sleep(0.2) assert inspector.get_state(server) == ConnectionState.CONNECTED assert inspector.get_failed_pings(server) == 0 From 45f98c5eb2f2cf6d3cbb0fd7e54482db3ba875d3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 19:00:30 -0400 Subject: [PATCH 65/68] Give LLM a measure of control over timeouts so it can work with longer running commands --- cecli/tools/command.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/cecli/tools/command.py b/cecli/tools/command.py index 06d694c97ed..e68404b9e13 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -88,6 +88,13 @@ class Tool(BaseTool): ), "default": False, }, + "timeout": { + "type": "integer", + "description": ( + "Timeout in seconds for command execution. " "Default is 30 seconds." + ), + "default": 30, + }, }, "required": [], }, @@ -114,6 +121,7 @@ async def execute( stdin=None, pty=False, user_input_required=False, + timeout=0, **kwargs, ): """ @@ -125,7 +133,7 @@ async def execute( or navigate terminal interfaces. For background interactions: provide 'background_key' + 'action' (stdin/stop). - Commands run with timeout based on agent_config['command_timeout'] (default: 30 seconds). + Commands run with timeout from agent_config['command_timeout'] (default: 30 seconds), """ response = ToolResponse( "command", @@ -181,10 +189,15 @@ async def execute( command = coder.format_command_with_prefix(command) - # Determine timeout from agent_config (default: 30 seconds) - timeout = 0 + # Determine timeout from agent_config (default: 30 seconds) as fallback + config_timeout = 0 if hasattr(coder, "agent_config"): - timeout = coder.agent_config.get("command_timeout", 30) + config_timeout = coder.agent_config.get("command_timeout", 30) + # Use LLM-specified timeout if provided, otherwise fallback to config + if timeout == 0: + timeout = config_timeout + # Clamp LLM timeout between config_timeout (minimum) and max(300, config_timeout) (maximum) + timeout = max(config_timeout, min(timeout, max(300, config_timeout))) if user_input_required: return await cls._execute_interactive(coder, command) @@ -641,6 +654,7 @@ def format_output(cls, coder, mcp_server, tool_response): action = params.get("action") stdin = params.get("stdin") pty = params.get("pty", False) + timeout = params.get("timeout", 30) user_input_required = params.get("user_input_required", False) coder.io.tool_output("") @@ -655,6 +669,8 @@ def format_output(cls, coder, mcp_server, tool_response): extras.append(f"action={action}") if pty: extras.append("pty=True") + if timeout != 30: + extras.append(f"timeout={timeout}s") if user_input_required: extras.append("user_input_required=True") From f38818ec39acb5761b6a924146039fdf0c2b7ccc Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 19:31:59 -0400 Subject: [PATCH 66/68] Integer cutoff for output limit --- cecli/tools/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cecli/tools/command.py b/cecli/tools/command.py index e68404b9e13..3a70a799277 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -403,7 +403,7 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non # Format output output_content = output or "" # Tokens are roughly 3-4 characters - output_limit = coder.large_file_token_threshold * 3.5 + output_limit = int(coder.large_file_token_threshold * 3.5) if coder.context_management_enabled and len(output_content) > output_limit * 1.25: # Save full output to paginated files instead of truncating From f2fbda8ef4fde6751216081b8e943c5bb96268fc Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 22:04:10 -0400 Subject: [PATCH 67/68] Patch the most low hanging sandbox escape vectors --- cecli/helpers/orchestration/agent_proxy.py | 10 ++++ cecli/helpers/orchestration/environment.py | 23 ++++++---- cecli/helpers/orchestration/safe_methods.py | 51 ++++++++++++++++++++- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py index 74d63cde862..6e20cd886ea 100644 --- a/cecli/helpers/orchestration/agent_proxy.py +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -151,6 +151,16 @@ def get_value(self, result: Any, path: str, default: Any = None) -> Any: """ from cecli.helpers import nested + # SECURITY: Reject paths that access private/dunder attributes + # (unless disable_security is active) + if not (self._env and self._env._orchestration_config.get("disable_security", False)): + for segment in path.replace("[", ".").replace("]", "").split("."): + segment = segment.strip() + if segment.startswith("_") and segment != "_": + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError(f"Access to private attribute '{segment}' is forbidden") + return nested.getter(result, path, default) @staticmethod diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index 67db6ff8f5e..fc9a5ec6828 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -26,6 +26,8 @@ _HelpfulBuiltins, _safe_dir, _safe_gather, + _safe_getattr, + _safe_hasattr, _safe_sleep, _safe_typeof, _safe_vars, @@ -225,6 +227,8 @@ def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = Non self.globals: dict[str, _Any] = {} self.locals: dict[str, _Any] = {} + _disable_sec = self._orchestration_config.get("disable_security", False) + self._safe_builtins: dict[str, _Any] = { "print": print, "range": range, @@ -242,8 +246,8 @@ def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = Non "vars": _safe_vars, "dir": _make_sandbox_dir(self.globals, self.locals), "isinstance": isinstance, - "hasattr": hasattr, - "getattr": getattr, + "hasattr": _safe_hasattr, + "getattr": _safe_getattr, "repr": repr, "enumerate": enumerate, "zip": zip, @@ -272,12 +276,12 @@ def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = Non "StopIteration": StopIteration, "ArithmeticError": ArithmeticError, "LookupError": LookupError, - "re": _SafeModuleProxy(re), - "math": _SafeModuleProxy(math), - "itertools": _SafeModuleProxy(itertools), - "collections": _SafeModuleProxy(collections), - "datetime": _SafeModuleProxy(datetime), - "traceback": _SafeModuleProxy(traceback), + "re": _SafeModuleProxy(re, disable_security=_disable_sec), + "math": _SafeModuleProxy(math, disable_security=_disable_sec), + "itertools": _SafeModuleProxy(itertools, disable_security=_disable_sec), + "collections": _SafeModuleProxy(collections, disable_security=_disable_sec), + "datetime": _SafeModuleProxy(datetime, disable_security=_disable_sec), + "traceback": _SafeModuleProxy(traceback, disable_security=_disable_sec), "pathlib": _SafePathlib, } @@ -296,6 +300,9 @@ def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = Non if self._orchestration_config.get("disable_security", False): for name in self._DANGEROUS_BUILTINS: self._safe_builtins[name] = __builtins__[name] + # Restore real getattr/hasattr when security is disabled + self._safe_builtins["getattr"] = getattr + self._safe_builtins["hasattr"] = hasattr # When allow_classes is enabled, expose __build_class__ needed by Python if self._orchestration_config.get("allow_classes", False): diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index 231c0029f18..cdfeaee524e 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -337,6 +337,35 @@ def _safe_dir(obj: Any) -> list: ] +def _safe_getattr(obj: Any, name: str, *args: Any) -> Any: + """Safe getattr() - blocks access to private/dunder attributes. + + Raises ``AttributeError`` (matching ``getattr``'s default behavior) when + the attribute name starts with ``_``, preventing sandboxed code from + bypassing the AST-level private-attribute filter. + + When ``disable_security`` is enabled in orchestration config, the real + ``getattr`` is restored so this restriction doesn't apply. + """ + if isinstance(name, str) and name.startswith("_") and name != "_": + raise AttributeError(f"Cannot access private attribute {name!r} in sandbox") + return getattr(obj, name, *args) + + +def _safe_hasattr(obj: Any, name: str) -> bool: + """Safe hasattr() - blocks probing of private/dunder attributes. + + Returns ``False`` for any attribute name starting with ``_`` so that + sandboxed code cannot detect the presence of private attributes. + + When ``disable_security`` is enabled in orchestration config, the real + ``hasattr`` is restored so this restriction doesn't apply. + """ + if isinstance(name, str) and name.startswith("_") and name != "_": + return False + return hasattr(obj, name) + + def _find_closing_quote(code: str, quote: str, start: int) -> int: """Find the position of the closing *quote* in *code*, skipping backslash-escaped characters so that ``\"`` (escaped quote) is not @@ -719,16 +748,31 @@ class _SafeModuleProxy: Setting an attribute on the proxy only affects the proxy, not the real module. This prevents sandbox code from monkey-patching standard library modules. + + The following methods are blocked and raise SecurityError: + - ``walk_stack()``, ``walk_tb()``, ``extract_stack()`` — frame introspection + that exposes unwrapped frame objects with access to parent-frame + ``f_locals`` / ``f_globals``. """ - def __init__(self, module: Any) -> None: + _BLOCKED_METHODS: frozenset[str] = frozenset({"walk_stack", "walk_tb", "extract_stack"}) + + def __init__(self, module: Any, *, disable_security: bool = False) -> None: object.__setattr__(self, "_module", module) + object.__setattr__(self, "_disable_security", disable_security) def __getattr__(self, name: str) -> Any: if name.startswith("_"): raise AttributeError( "Cannot access private attribute " + repr(name) + " on module proxy" ) + if name in self._BLOCKED_METHODS and not object.__getattribute__(self, "_disable_security"): + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + f"Cannot access '{name}' on module proxy. " + f"Frame introspection methods are blocked in the sandbox." + ) module = object.__getattribute__(self, "_module") return getattr(module, name) @@ -739,7 +783,10 @@ def __setattr__(self, name: str, value: Any) -> None: def __dir__(self) -> list: module = object.__getattribute__(self, "_module") - return [x for x in dir(module) if not x.startswith("_")] + blocked = ( + set() if object.__getattribute__(self, "_disable_security") else self._BLOCKED_METHODS + ) + return [x for x in dir(module) if not x.startswith("_") and x not in blocked] # --------------------------------------------------------------------------- From 9f297cdf776e8d04e590cb7648f69ab638ff489a Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 00:46:26 -0400 Subject: [PATCH 68/68] Rewrap sub module exports to prevent improper access, disable match statements --- cecli/helpers/orchestration/environment.py | 26 +- cecli/helpers/orchestration/safe_methods.py | 254 +++++++++++++++++++- cecli/helpers/orchestration/security.py | 13 + 3 files changed, 277 insertions(+), 16 deletions(-) diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py index fc9a5ec6828..ac8694a30ba 100644 --- a/cecli/helpers/orchestration/environment.py +++ b/cecli/helpers/orchestration/environment.py @@ -34,6 +34,8 @@ _SafeJson, _SafeModuleProxy, _SafePathlib, + _SafeRe, + _SafeTraceback, _strip_allowed_imports, ) from cecli.helpers.orchestration.security import ( @@ -276,12 +278,12 @@ def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = Non "StopIteration": StopIteration, "ArithmeticError": ArithmeticError, "LookupError": LookupError, - "re": _SafeModuleProxy(re, disable_security=_disable_sec), + "re": _SafeRe, "math": _SafeModuleProxy(math, disable_security=_disable_sec), "itertools": _SafeModuleProxy(itertools, disable_security=_disable_sec), "collections": _SafeModuleProxy(collections, disable_security=_disable_sec), "datetime": _SafeModuleProxy(datetime, disable_security=_disable_sec), - "traceback": _SafeModuleProxy(traceback, disable_security=_disable_sec), + "traceback": _SafeTraceback, "pathlib": _SafePathlib, } @@ -458,6 +460,16 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: code = f"Syntax Error in orchestration code: {e}" return {"results": code, "state_variables": self._state_snapshot()} + def _build_result(msg: str = "") -> dict: + print_output = "".join(captured_output) + parts = [] + if print_output: + parts.append(print_output.rstrip("\n")) + if msg: + parts.append(msg) + code = "\n".join(parts) if parts else "" + return {"results": code, "state_variables": self._state_snapshot()} + try: if not self._orchestration_config.get("disable_security", False): tree = SecurityFilter( @@ -488,16 +500,6 @@ def _capture_print(*args: Any, **kwargs: Any) -> None: code = f"Compilation Error: {e}" return {"results": code, "state_variables": self._state_snapshot()} - def _build_result(msg: str = "") -> dict: - print_output = "".join(captured_output) - parts = [] - if print_output: - parts.append(print_output.rstrip("\n")) - if msg: - parts.append(msg) - code = "\n".join(parts) if parts else "" - return {"results": code, "state_variables": self._state_snapshot()} - try: exec(compiled_code, self.globals, self.locals) runner_coro = self.locals["__agent_async_runner"]() diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py index cdfeaee524e..c7d5ea9da8c 100644 --- a/cecli/helpers/orchestration/safe_methods.py +++ b/cecli/helpers/orchestration/safe_methods.py @@ -10,6 +10,8 @@ import asyncio import json import pathlib +import re as _re_mod +import traceback as _tb_mod from typing import Any # --------------------------------------------------------------------------- @@ -264,8 +266,21 @@ def _safe_typeof(obj: Any) -> type: Unlike ``type(name, bases, dict)`` which can create new classes at runtime, ``_safe_typeof(obj)`` only returns the type of an object for inspection. + + Additionally, when *obj* is itself a type/class whose metaclass is ``type``, + the result would be ```` — the real ``type`` metaclass. + This is blocked because the metaclass can be used to dynamically create + arbitrary classes, bypassing the AST-level ``ClassDef`` filter. """ - return type(obj) + result = type(obj) + if result is type: + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + "Access to the 'type' metaclass is forbidden in the sandbox. " + "Use isinstance() for type checks instead of comparing type objects." + ) + return result def _safe_vars(obj: Any) -> dict: @@ -347,7 +362,9 @@ def _safe_getattr(obj: Any, name: str, *args: Any) -> Any: When ``disable_security`` is enabled in orchestration config, the real ``getattr`` is restored so this restriction doesn't apply. """ - if isinstance(name, str) and name.startswith("_") and name != "_": + if type(name) is not str: + raise TypeError(f"getattr() attribute name must be str, not {type(name).__name__}") + if name.startswith("_") and name != "_": raise AttributeError(f"Cannot access private attribute {name!r} in sandbox") return getattr(obj, name, *args) @@ -361,7 +378,9 @@ def _safe_hasattr(obj: Any, name: str) -> bool: When ``disable_security`` is enabled in orchestration config, the real ``hasattr`` is restored so this restriction doesn't apply. """ - if isinstance(name, str) and name.startswith("_") and name != "_": + if type(name) is not str: + raise TypeError(f"hasattr() attribute name must be str, not {type(name).__name__}") + if name.startswith("_") and name != "_": return False return hasattr(obj, name) @@ -743,6 +762,212 @@ def dumps(obj: Any, **kwargs) -> str: return json.dumps(obj, **safe_kwargs) +class _SafeTraceback: + """Drop-in ``traceback`` namespace exposing only safe print and format methods. + + Unlike ``_SafeModuleProxy`` which forwards all non-blocked attributes to the + real module, this class only exposes an explicit allowlist of print/format + methods. Module-level imports like ``traceback.sys`` and ``traceback.io`` + are not accessible through this wrapper, closing the ``sys.modules`` + sandbox-escape vector. + + Exposed methods: + + **Printing** (output to stderr or a given file): + - ``print_exc(limit=None, file=None)`` — print current exception + - ``print_exception(exc, limit=None, file=None)`` — print a given exception + - ``print_stack(f=None, limit=None, file=None)`` — print current call stack + - ``print_tb(tb, limit=None, file=None)`` — print a given traceback + - ``print_last(limit=None, file=None)`` — print the last exception + + **Formatting** (return strings for logging): + - ``format_exc(limit=None)`` — format current exception as string + - ``format_exception(exc, limit=None)`` — format a given exception to list of strings + - ``format_stack(f=None, limit=None)`` — format current stack to list of strings + - ``format_tb(tb, limit=None)`` — format a given traceback to list of strings + - ``format_list(extracted)`` — format extracted stack entries + - ``format_exception_only(exc)`` — format exception type + value only + + **Utility**: + - ``clear_frames(tb)`` — clear local variables from traceback frames + + **Classes**: + - ``TracebackException`` — programmatic traceback representation + - ``FrameSummary`` — single frame summary + - ``StackSummary`` — list of frame summaries + """ + + @classmethod + def format_exc(cls, limit=None): + import traceback + + return traceback.format_exc(limit=limit) + + @classmethod + def format_exception(cls, exc, limit=None): + import traceback + + return traceback.format_exception(exc, limit=limit) + + @classmethod + def format_stack(cls, f=None, limit=None): + import traceback + + return traceback.format_stack(f=f, limit=limit) + + @classmethod + def format_tb(cls, tb, limit=None): + import traceback + + return traceback.format_tb(tb, limit=limit) + + @classmethod + def format_list(cls, extracted): + import traceback + + return traceback.format_list(extracted) + + @classmethod + def format_exception_only(cls, exc): + import traceback + + return traceback.format_exception_only(exc) + + @classmethod + def print_exc(cls, limit=None, file=None): + import traceback + + traceback.print_exc(limit=limit, file=file) + + @classmethod + def print_exception(cls, exc, limit=None, file=None): + import traceback + + traceback.print_exception(exc, limit=limit, file=file) + + @classmethod + def print_stack(cls, f=None, limit=None, file=None): + import traceback + + traceback.print_stack(f=f, limit=limit, file=file) + + @classmethod + def print_tb(cls, tb, limit=None, file=None): + import traceback + + traceback.print_tb(tb, limit=limit, file=file) + + @classmethod + def print_last(cls, limit=None, file=None): + import traceback + + traceback.print_last(limit=limit, file=file) + + @classmethod + def clear_frames(cls, tb): + import traceback + + traceback.clear_frames(tb) + + # Classes from traceback — accessed via class attribute, not instance + TracebackException = None + FrameSummary = None + StackSummary = None + + +_SafeTraceback.TracebackException = _tb_mod.TracebackException +_SafeTraceback.FrameSummary = _tb_mod.FrameSummary +_SafeTraceback.StackSummary = _tb_mod.StackSummary + + +class _SafeRe: + """Drop-in ``re`` namespace exposing only safe regex operations. + + Unlike ``_SafeModuleProxy`` which forwards all non-blocked attributes to the + real module, this class only exposes an explicit allowlist of regex + functions, flags, and types. Module-level imports like ``re.enum``, + ``re.copyreg``, and ``re.functools`` are not accessible through this + wrapper, closing the ``re.enum.sys`` sandbox-escape vector. + + Exposed: + + **Compilation**: + - ``compile(pattern, flags=0)`` — compile a regex pattern + + **Searching / Matching**: + - ``search(pattern, string, flags=0)`` — scan through string + - ``match(pattern, string, flags=0)`` — match at start of string + - ``fullmatch(pattern, string, flags=0)`` — match whole string + - ``findall(pattern, string, flags=0)`` — all non-overlapping matches + - ``finditer(pattern, string, flags=0)`` — iterator over matches + + **Substitution / Splitting**: + - ``sub(pattern, repl, string, ...)`` — substitute + - ``subn(pattern, repl, string, ...)`` — substitute with count + - ``split(pattern, string, ...)`` — split by pattern + + **Utilities**: + - ``escape(string)`` — escape special characters + - ``purge()`` — clear the regex cache + + **Flags**: + - ``NOFLAG``, ``IGNORECASE`` / ``I``, ``MULTILINE`` / ``M``, + ``DOTALL`` / ``S``, ``VERBOSE`` / ``X``, ``ASCII`` / ``A`` + - ``RegexFlag`` — the flag enum type + + **Types / Exceptions**: + - ``Pattern`` — compiled pattern type + - ``Match`` — match result type + - ``error`` — regex exception class + """ + + # Compilation + compile = None + + # Searching / Matching + search = None + match = None + fullmatch = None + findall = None + finditer = None + + # Substitution / Splitting + sub = None + subn = None + split = None + + # Utilities + escape = None + purge = None + + # Flags + NOFLAG = None + IGNORECASE = None + I = None # noqa: E741 + MULTILINE = None + M = None + DOTALL = None + S = None + VERBOSE = None + X = None + ASCII = None + A = None + RegexFlag = None + + # Types / Exceptions + Pattern = None + Match = None + error = None + + +for _name in dir(_SafeRe): + if not _name.startswith("_"): + try: + setattr(_SafeRe, _name, getattr(_re_mod, _name)) + except AttributeError: + pass + + class _SafeModuleProxy: """Proxy that forwards attribute reads to a real module but prevents mutation. @@ -773,8 +998,29 @@ def __getattr__(self, name: str) -> Any: f"Cannot access '{name}' on module proxy. " f"Frame introspection methods are blocked in the sandbox." ) + if name == "modules" and not object.__getattribute__(self, "_disable_security"): + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + f"Cannot access '{name}' on module proxy. " + f"Direct module registry access is blocked in the sandbox." + ) module = object.__getattribute__(self, "_module") - return getattr(module, name) + attr = getattr(module, name) + + # Re-wrap sub-modules to prevent transitive sandbox escape. + # Without this, `re.enum.sys` would return the real `sys` module + # (since `re` imports `enum` and `enum` imports `sys`), giving + # attackers access to `sys.modules["builtins"].getattr` and from + # there unrestricted private-attribute access. + from types import ModuleType + + if isinstance(attr, ModuleType): + return _SafeModuleProxy( + attr, disable_security=object.__getattribute__(self, "_disable_security") + ) + + return attr def __setattr__(self, name: str, value: Any) -> None: if not name.startswith("_"): diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py index 97fd2d8e430..4a7580b2061 100644 --- a/cecli/helpers/orchestration/security.py +++ b/cecli/helpers/orchestration/security.py @@ -217,6 +217,19 @@ def visit_Call(self, node: ast.Call): return self.generic_visit(node) + def visit_Match(self, node: ast.Match) -> ast.Expr: + """Block all ``match``/``case`` statements. + + Class patterns destructure objects via runtime ``getattr()``, + bypassing the AST-level ``visit_Attribute`` guard entirely. + Rather than trying to cover every pattern variant, we reject + the ``match`` statement wholesale. + """ + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "The 'match'/'case' statement is disabled in the orchestration environment." + ) + class LoopYieldInjector(ast.NodeTransformer): """