diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 62c590f9adb..73f7222afbd 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -62,7 +62,6 @@ def __init__(self, *args, **kwargs): self.max_tool_vector_history = 20 self.read_tools = { "command", - "commandinteractive", "explorecode", "ls", "readfile", @@ -72,7 +71,6 @@ def __init__(self, *args, **kwargs): } self.write_tools = { "command", - "commandinteractive", "editfile", "undochange", } @@ -83,6 +81,7 @@ def __init__(self, *args, **kwargs): self.change_tracker = ChangeTracker() self.args = kwargs.get("args") self.files_added_in_exploration = set() + self.file_read_cache = set() self.tool_call_count = 0 self.turn_count = 0 self.max_reflections = 15 @@ -98,7 +97,7 @@ def __init__(self, *args, **kwargs): self.skip_cli_confirmations = False self.agent_finished = False self.agent_config = self._get_agent_config() - self.max_sub_agents = self.agent_config.get("max_sub_agents", 3) + self.max_sub_agents = self.agent_config.get("max_sub_agents", 30) self.sub_agent_paths = self.agent_config.get("subagent_paths", []) self._setup_agent() @@ -167,6 +166,11 @@ def _get_agent_config(self): config["hot_reload"] = nested.getter(config, "hot_reload", False) config["diff_colors"] = nested.getter(config, "diff_colors", True) config["allow_nested_delegation"] = nested.getter(config, "allow_nested_delegation", False) + config["allow_orchestration"] = nested.getter(config, "allow_orchestration", True) + config["max_sub_agents"] = nested.getter(config, "max_sub_agents", 30) + + if config["max_sub_agents"] == -1: + config["max_sub_agents"] = 2 ^ 31 - 1 config["tools_paths"] = nested.getter(config, ["tools_paths", "tool_paths"], []) config["tools_includelist"] = nested.getter( @@ -184,6 +188,7 @@ def _get_agent_config(self): config["servers_excludelist"] = nested.getter( config, ["servers_excludelist", "servers_blacklist"], [] ) + config["include_context_blocks"] = set( nested.getter( config, @@ -203,6 +208,10 @@ def _get_agent_config(self): ) config["exclude_context_blocks"] = set(nested.getter(config, "exclude_context_blocks", [])) + if config["allow_orchestration"]: + config["include_context_blocks"].add("orchestration") + config["exclude_context_blocks"].discard("orchestration") + self.large_file_token_threshold = config["large_file_token_threshold"] self.skip_cli_confirmations = config["skip_cli_confirmations"] self.hot_reload_enabled = config["hot_reload"] @@ -292,6 +301,7 @@ async def initialize_mcp_tools(self): async def _execute_mcp_tool(self, server, tool_name, params): """Helper to execute a single MCP tool call, created from legacy format.""" + from cecli.tools.utils.responses import ToolResponse async def _exec_async(): function_dict = {"name": tool_name, "arguments": json.dumps(params)} @@ -350,9 +360,14 @@ async def _exec_async(): result, interrupted = await interruptible(_exec_async(), self.interrupt_event) + response = ToolResponse(tool_name) if interrupted: - return "Tool execution interrupted by user." - return result + response.append_error("Tool execution interrupted by user.") + elif isinstance(result, str) and result.startswith("Error executing tool call"): + response.append_error(result) + else: + response.append_result(result) + return response def _calculate_context_block_tokens(self, force=False): """ @@ -383,6 +398,7 @@ def _calculate_context_block_tokens(self, force=False): "servers", "sub_agents", "loaded_skills", + "orchestration", ] for block_type in block_types: if block_type in self.allowed_context_blocks: @@ -419,6 +435,8 @@ def _generate_context_block(self, block_name): content = self.get_servers_context() elif block_name == "loaded_skills": content = self.get_skills_content() + elif block_name == "orchestration": + content = self.get_orchestration_context() elif block_name == "sub_agents" and ( not self.parent_uuid or self.agent_config.get("allow_nested_delegation", False) ): @@ -897,7 +915,7 @@ async def _execute_mcp_tools(self, server, tool_calls): { "role": "tool", "tool_call_id": tool_call.id, - "content": result, + "content": str(result), } ) return tool_responses @@ -1601,6 +1619,23 @@ def get_servers_context(self): self.io.tool_error(f"Error generating servers context: {str(e)}") return None + def get_orchestration_context(self): + """ + Generate a context block for the Orchestrate tool if allowed. + + Only returns content if ``allow_orchestration`` is enabled in the agent config. + """ + if not self.use_enhanced_context: + return None + + try: + from cecli.helpers.orchestration import build_orchestration_context_block + + return build_orchestration_context_block(self.agent_config) + except Exception as e: + self.io.tool_error(f"Error generating orchestration context: {str(e)}") + return None + def get_sub_agents_context(self): """ Generate a context block for registered sub-agents. diff --git a/cecli/coders/base_coder.py b/cecli/coders/base_coder.py index 629ec23d987..3b8883f04a8 100755 --- a/cecli/coders/base_coder.py +++ b/cecli/coders/base_coder.py @@ -55,7 +55,7 @@ remove_reasoning_content, replace_reasoning_tags, ) -from cecli.repo import ANY_GIT_ERROR, GitRepo +from cecli.repo import ANY_GIT_ERROR, GitRepo, GitRepoProxy from cecli.repomap import RepoMap from cecli.report import update_error_prefix from cecli.run_cmd import run_cmd_async @@ -571,11 +571,13 @@ def __init__( self.repo = repo if use_git and self.repo is None: try: - self.repo = GitRepo( - self.io, - fnames, - None, - models=main_model.commit_message_models(), + self.repo = GitRepoProxy( + GitRepo( + self.io, + fnames, + None, + models=main_model.commit_message_models(), + ) ) except FileNotFoundError: pass @@ -4266,9 +4268,12 @@ def check_added_files(self): if tokens < warn_number_of_tokens: return - self.io.tool_warning("Warning: it's best to only add files that need changes to the chat.") - self.io.tool_warning(urls.edit_errors) - self.warning_given = True + if self.context_compaction_current_ratio > 0.5: + self.io.tool_warning( + "Warning: it's best to only add files that need changes to the chat." + ) + self.io.tool_warning(urls.edit_errors) + self.warning_given = True async def prepare_to_edit(self, edits): res = [] diff --git a/cecli/commands/__init__.py b/cecli/commands/__init__.py index 7671804f33b..ef69f9053eb 100644 --- a/cecli/commands/__init__.py +++ b/cecli/commands/__init__.py @@ -5,6 +5,8 @@ BaseCommand pattern for modular, testable command execution. """ +from cecli.signals import ReloadProgramSignal, SwitchCoderSignal + from .add import AddCommand from .agent import AgentCommand from .agent_model import AgentModelCommand @@ -21,7 +23,7 @@ from .context_management import ContextManagementCommand from .copy import CopyCommand from .copy_context import CopyContextCommand -from .core import Commands, ReloadProgramSignal, SwitchCoderSignal +from .core import Commands from .diff import DiffCommand from .drop import DropCommand from .editor import EditCommand, EditorCommand diff --git a/cecli/commands/core.py b/cecli/commands/core.py index 29823d6cddd..a24bc16cc64 100644 --- a/cecli/commands/core.py +++ b/cecli/commands/core.py @@ -8,49 +8,7 @@ from cecli.helpers import nested, plugin_manager from cecli.helpers.file_searcher import handle_core_files from cecli.helpers.threading import ThreadSafeEvent -from cecli.repo import ANY_GIT_ERROR - - -class SwitchCoderSignal(BaseException): - """ - Signal to switch the current Coder instance to a new configuration. - - This is NOT an error - it's a control flow signal used to propagate - coder switching requests up through the async call stack. It carries - the kwargs needed to create a new Coder instance. - - Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) - to avoid being caught by generic `except Exception` handlers, making the - non-error nature of this signal explicit. - - Attributes: - kwargs: Configuration dict passed to Coder.create() for the new instance - placeholder: Optional placeholder text for the input prompt - """ - - def __init__(self, placeholder=None, **kwargs): - self.kwargs = kwargs - self.placeholder = placeholder - super().__init__() - - -class ReloadProgramSignal(BaseException): - """ - Signal to reload the entire program configuration. - - This is NOT an error - it's a control flow signal used to trigger - a full program reload, re-parsing config files and re-initializing - all components. Useful for hot-reloading when configuration files - change. - - Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) - to avoid being caught by generic `except Exception` handlers. - """ - - def __init__(self, message="Reloading program configuration...", **kwargs): - self.kwargs = kwargs - self.message = message - super().__init__(self.message) +from cecli.signals import SwitchCoderSignal class Commands: @@ -215,6 +173,8 @@ def get_commands(self): return sorted(commands) async def execute(self, cmd_name, args, coder=None, **kwargs): + from cecli.repo import ANY_GIT_ERROR + active_coder = coder or self.coder command_class = CommandRegistry.get_command(cmd_name) diff --git a/cecli/commands/hot_reload.py b/cecli/commands/hot_reload.py index acc8f2e0521..7a413c78d90 100644 --- a/cecli/commands/hot_reload.py +++ b/cecli/commands/hot_reload.py @@ -1,7 +1,7 @@ from typing import List -from cecli.commands.core import ReloadProgramSignal from cecli.commands.utils.base_command import BaseCommand +from cecli.signals import ReloadProgramSignal class HotReloadCommand(BaseCommand): diff --git a/cecli/commands/undo.py b/cecli/commands/undo.py index 66b02c24f11..60f9acad929 100644 --- a/cecli/commands/undo.py +++ b/cecli/commands/undo.py @@ -53,8 +53,12 @@ async def _raw_cmd_undo(cls, io, coder, args): ) return format_command_result(io, "undo", "Commit has multiple parents") - prev_commit = last_commit.parents[0] - changed_files_last_commit = [item.a_path for item in last_commit.diff(prev_commit)] + prev_sha = last_commit.parents[0] + # Get list of files changed between parent and current commit + diff_output = coder.repo.repo.git.diff("--name-only", prev_sha, last_commit.hexsha) + changed_files_last_commit = ( + [f for f in diff_output.splitlines() if f] if diff_output else [] + ) for fname in changed_files_last_commit: if coder.repo.repo.is_dirty(path=fname): @@ -64,9 +68,12 @@ async def _raw_cmd_undo(cls, io, coder, args): return format_command_result(io, "undo", f"File {fname} has uncommitted changes") # Check if the file was in the repo in the previous commit + # git ls-tree returns empty string if the file does not exist in that commit try: - prev_commit.tree[fname] - except KeyError: + tree_output = coder.repo.repo.git.ls_tree(prev_sha, "--", fname) + if not tree_output: + raise KeyError + except (KeyError, ANY_GIT_ERROR): io.tool_error( f"The file {fname} was not in the repository in the previous commit. Cannot" " undo safely." diff --git a/cecli/helpers/agents/service.py b/cecli/helpers/agents/service.py index 273b7e1e4b9..f229573a156 100644 --- a/cecli/helpers/agents/service.py +++ b/cecli/helpers/agents/service.py @@ -228,7 +228,7 @@ def __init__(self, coder) -> None: @property def max_sub_agents(self) -> int: """Return the max number of sub-agents allowed for this coder.""" - return getattr(self.coder, "max_sub_agents", 3) + return getattr(self.coder, "max_sub_agents", 30) # ------------------------------------------------------------------ # # Internal helpers @@ -495,7 +495,7 @@ async def _create_sub_agent_coder( ) new_coder = await Coder.create(**kwargs) - new_coder.max_sub_agents = getattr(self.coder, "max_sub_agents", 3) + new_coder.max_sub_agents = getattr(self.coder, "max_sub_agents", 30) # IOProxy wrapping is handled by base_coder.py's Coder.__init__ # Re-acquire the lock to register — we must re-check max agents since @@ -575,7 +575,7 @@ def start_generate_task(self, info: SubAgentInfo, user_message: str) -> asyncio. Args: - .. note:: + .. note: **Ordering dependency with mark_sub_agent_finished()** diff --git a/cecli/helpers/background_commands.py b/cecli/helpers/background_commands.py index 825dd09ad50..83fe53968cd 100644 --- a/cecli/helpers/background_commands.py +++ b/cecli/helpers/background_commands.py @@ -767,6 +767,8 @@ def save_paginated_output( Returns: Tuple of (folder path, rel file paths list, alias paths list), or None if output fits in one page. """ + import math + if not output or len(output) <= page_size: return None @@ -774,7 +776,7 @@ def save_paginated_output( abs_folder = abs_root_path_func(folder_path) os.makedirs(abs_folder, exist_ok=True) - num_pages = (len(output) + page_size - 1) // page_size + num_pages = math.ceil((len(output) + page_size - 1) // page_size) for i in range(num_pages): page_num = i + 1 diff --git a/cecli/helpers/conversation/files.py b/cecli/helpers/conversation/files.py index f28fd0f7ef9..ae5721b96e6 100644 --- a/cecli/helpers/conversation/files.py +++ b/cecli/helpers/conversation/files.py @@ -116,9 +116,7 @@ def add_file( try: content = coder.io.read_text(abs_fname, silent=True) if coder.hashlines: - content, json_str = hashline_formatted( - content, file_name=abs_fname, partial=False, expanded=False - ) + content, json_str = hashline_formatted(content, file_name=abs_fname) self._file_json_contents[abs_fname] = json_str except Exception: content = "" # Empty content for unreadable files @@ -236,9 +234,7 @@ def generate_diff(self, fname: str) -> Optional[str]: try: current_content = coder.io.read_text(abs_fname, silent=True) if coder.hashlines: - current_content, _ = hashline_formatted( - current_content, file_name=abs_fname, partial=False, expanded=False - ) + current_content, _ = hashline_formatted(current_content, file_name=abs_fname) except Exception: return None @@ -607,12 +603,30 @@ def get_file_context(self, file_path: str, all_ranges=False, check_versions=True ): continue - _, json_str = hashline_formatted( - range_content, + # Hash the full content first so that adjacent-line hashes + # are computed with proper surrounding context from the complete + # file, then extract only the range's lines. This ensures content + # IDs are consistent with other tools (e.g., ReadFile) that hash + # the full file content. + full_prefixed, _ = hashline_formatted( + content, file_name=abs_fname, - partial=True, - expanded=False, - start_line=start_line_adj, + total_lines=len(content_lines), + start_line=1, + ) + prefixed_lines = full_prefixed.splitlines() + range_prefixed_lines = prefixed_lines[start_line_adj - 1 : end_line_adj] + range_prefixed_content = "\n".join(range_prefixed_lines) + + json_str = json.dumps( + { + "file_name": abs_fname, + "start_line": start_line_adj, + "end_line": end_line_adj, + "total_lines": len(content_lines), + "prefixed_contents": range_prefixed_content, + }, + ensure_ascii=False, ) results.append(json.loads(json_str)) diff --git a/cecli/helpers/conversation/integration.py b/cecli/helpers/conversation/integration.py index 169d3676796..8888ab3b4ad 100644 --- a/cecli/helpers/conversation/integration.py +++ b/cecli/helpers/conversation/integration.py @@ -809,6 +809,10 @@ def add_static_context_blocks(self) -> None: block = coder._generate_context_block("servers") if block: message_blocks["servers"] = block + if "orchestration" in coder.allowed_context_blocks: + block = coder._generate_context_block("orchestration") + if block: + message_blocks["orchestration"] = block # Add static blocks to conversation manager with stable hash keys for block_type, block_content in message_blocks.items(): diff --git a/cecli/helpers/grep_ast/grep_ast.py b/cecli/helpers/grep_ast/grep_ast.py index 678a694a899..1b056f6195e 100644 --- a/cecli/helpers/grep_ast/grep_ast.py +++ b/cecli/helpers/grep_ast/grep_ast.py @@ -214,17 +214,24 @@ def format(self): # reset output += "\033[0m\n" - dots = not (0 in self.show_lines) + # dots = not (0 in self.show_lines) + in_gap = False + gap_start = -1 for i, line in enumerate(self.lines): if i not in self.show_lines: - if dots: - if self.line_number: - output += "...⋮...\n" - else: - output += "⋮\n" - dots = False + if not in_gap: + in_gap = True + gap_start = i continue + if in_gap: + omitted = i - gap_start + if self.line_number: + output += f"...⋮... [{omitted} lines omitted (L{gap_start + 1}–L{i})]\n" + else: + output += f"⋮ [{omitted} lines omitted]\n" + in_gap = False + if i in self.lines_of_interest and self.mark_lois: spacer = "█" if self.color: @@ -237,7 +244,7 @@ def format(self): line_output = f"{i + 1: 3}" + line_output output += line_output + "\n" - dots = True + # dots = True return output diff --git a/cecli/helpers/hashline.py b/cecli/helpers/hashline.py index 7064a41afd1..6a0222d4343 100644 --- a/cecli/helpers/hashline.py +++ b/cecli/helpers/hashline.py @@ -3,7 +3,11 @@ import re from collections import Counter -from cecli.helpers.hashpos.hashpos import HashPos +from cecli.helpers.hashpos.hashpos import ( # noqa + HASH_DELIMITER, + UNIQUE_HASH_DELIMITER, + HashPos, +) HASHLINE_PREFIX_RE = HashPos.HASH_PREFIX_RE @@ -30,7 +34,7 @@ def hashline(text: str, start_line: int = 1) -> str: def hashline_formatted( - text: str, file_name: str, partial: bool, expanded: bool, start_line: int = 1 + text: str, file_name: str, total_lines: int = 0, start_line: int = 1 ) -> tuple[str, str]: """ Generate hashline-formatted content and return it as both raw hashline text and a JSON structure. @@ -55,8 +59,7 @@ def hashline_formatted( "file_name": file_name, "start_line": start_line, "end_line": end_line, - "partial": partial, - "expanded": expanded, + "total_lines": total_lines if total_lines else hp.total, "prefixed_contents": prefixed, } @@ -73,14 +76,14 @@ def strip_hashline(text: str) -> str: return HashPos.strip_prefix(text) -def normalize_hashline(hashline_str: str) -> str: +def normalize_hashline(hashline_str: str, throw=True) -> str: """ Normalize a hashline string to the content id hash fragment. """ if hashline_str in ("@000", "000@"): return hashline_str try: - return HashPos.normalize(hashpos_str=hashline_str) + return HashPos.normalize(hashpos_str=hashline_str, throw=throw) except ValueError as e: raise ContentHashError(str(e)) @@ -276,6 +279,7 @@ def resolve_content_to_hashline_ids( original_content: str, start_value: str, end_value: str = None, + start_hint_line: int | None = None, ) -> tuple: """ Resolve potential line content values to proper hashline content IDs. @@ -296,6 +300,8 @@ def resolve_content_to_hashline_ids( original_content: Original file content (without hash prefixes) start_value: The start_line value from the edit end_value: The end_line value from the edit (optional) + start_hint_line: 0-based line number hint used to disambiguate + non-unique start_value matches (optional) Returns: tuple: (resolved_start, resolved_end) with hash IDs or original values @@ -332,8 +338,11 @@ def _find_multiline_match(lines, value): def _resolve_to_hash_id(lines, idx, hp): """Generate a hash ID for the line at the given index.""" - hash_id = hp.generate_public_id(lines[idx], idx) - return hash_id + "::" + line_text = lines[idx] + # Compute the occurrence number (1-based) of this line instance + occurrence = 1 + sum(1 for i in range(idx) if lines[i] == line_text) + hash_id = hp.generate_public_id(line_text, idx, occurrence) + return HashPos.get_wrapped_id(hash_id) lines = original_content.splitlines() hp = HashPos(original_content) @@ -355,6 +364,13 @@ def _resolve_to_hash_id(lines, idx, hp): if len(containing_indices) == 1: resolved_start_idx = containing_indices[0] resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) + elif len(containing_indices) > 1 and start_hint_line is not None: + # Multiple matches - pick closest to hint line + resolved_start_idx = min( + containing_indices, + key=lambda idx: abs(idx - start_hint_line), + ) + resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) elif start_value is not None and _looks_like_content_id(start_value): # Already a content ID - try to resolve it to find the line position # for proximity matching with end_value @@ -364,7 +380,22 @@ def _resolve_to_hash_id(lines, idx, hp): if candidates: resolved_start_idx = candidates[0] except (ContentHashError, ValueError): - pass + # The value looks like a content ID but normalized to UNIQUE_HASH_DELIMITER which can't + # be spatially resolved. Fall back: strip the UNIQUE_HASH_DELIMITER prefix and treat the + # remaining content as line content to match. + content = HashPos.strip_prefix(start_value) + if content != start_value and content.strip(): + containing_indices = _find_substring_matches(lines, content) + if len(containing_indices) == 1: + resolved_start_idx = containing_indices[0] + resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) + elif len(containing_indices) > 1 and start_hint_line is not None: + # Multiple matches - pick closest to hint line + resolved_start_idx = min( + containing_indices, + key=lambda idx: abs(idx - start_hint_line), + ) + resolved_start = _resolve_to_hash_id(lines, resolved_start_idx, hp) # Resolve end_value based on proximity to start position resolved_end = end_value @@ -390,6 +421,30 @@ def _resolve_to_hash_id(lines, idx, hp): key=lambda idx: abs(idx - resolved_start_idx), ) resolved_end = _resolve_to_hash_id(lines, closest_idx, hp) + elif end_value is not None and _looks_like_content_id(end_value): + # Already a content ID - try to resolve it + try: + normalized = normalize_hashline(end_value) + candidates = hp.resolve_to_lines(normalized) + if candidates and resolved_start_idx is not None: + # Pick candidate closest to start position + resolved_end = _resolve_to_hash_id(lines, candidates[0], hp) + except (ContentHashError, ValueError): + # The value looks like a content ID but normalized to UNIQUE_HASH_DELIMITER which can't + # be spatially resolved. Fall back: strip the UNIQUE_HASH_DELIMITER prefix and treat the + # remaining content as line content to match. + content = HashPos.strip_prefix(end_value) + if content != end_value and content.strip(): + containing_indices = _find_substring_matches(lines, content) + if len(containing_indices) == 1: + idx = containing_indices[0] + resolved_end = _resolve_to_hash_id(lines, idx, hp) + elif len(containing_indices) > 1 and resolved_start_idx is not None: + closest_idx = min( + containing_indices, + key=lambda idx: abs(idx - resolved_start_idx), + ) + resolved_end = _resolve_to_hash_id(lines, closest_idx, hp) return resolved_start, resolved_end @@ -1706,6 +1761,152 @@ def _has_indent_jump(test_lines: list, unit: int) -> bool: return safe_resolved_ops, rejected_ops +def _try_resolve_as_unique_line( + hp: HashPos, + value: str, +) -> str | None: + """ + Try to resolve a value by matching it against unique lines in the source content. + + If the value (after stripping) matches exactly one line in the source + AND that line appears only once (unique), returns a tilde-wrapped hash ID + that can be resolved to line indices by the HashPos engine. + + This serves as a preliminary resolution step that can salvage operations + whose hash references could not be normalized, by checking if the value + corresponds to unique line content in the file. + + Args: + hp: HashPos instance for the source content + value: The value to resolve (typically line content rather than a hash) + + Returns: + A tilde-wrapped hash ID if resolution succeeds, None otherwise + """ + if not hp or not value: + return None + + value_stripped = value.strip() + if not value_stripped: + return None + + # Find all lines that exactly match this value + matching_lines = [] + for i, line in enumerate(hp.lines): + if line.strip() == value_stripped: + matching_lines.append((i, line)) + + # Only resolve if it matches exactly one unique line + if len(matching_lines) == 1: + idx, matched_line = matching_lines[0] + # Verify the line is actually unique in the source (appears only once) + if hp.line_counts.get(matched_line, 0) == 1: + hash_id = hp.generate_public_id(matched_line, idx, 1) + return HashPos.get_wrapped_id(hash_id) + + return None + + +def _raise_clear_hash_error(hp, op, key): + """Raise a ContentHashError with a clear message explaining why a value couldn't be resolved. + + Checks if the value matches line content in the file and provides specific guidance + about how to fix the reference (use hashed ID prefix, check for duplicates, etc.). + """ + value = op.get(key, "") + value_stripped = value.strip() if value else "" + + if not hp or not value_stripped: + raise ContentHashError( + f"Could not resolve '{value}' as a content ID. " + "Use a hashed ID prefix (e.g., `~XXXX~`) to target the line." + ) + + # Count how many lines contain this value as a substring + matching_count = 0 + for line in hp.lines: + if value_stripped in line: + matching_count += 1 + + if matching_count > 1: + raise ContentHashError( + f"Line '{value_stripped}' appears {matching_count} times in the file. " + "Use the hashed ID prefix (e.g., `~XXXX~`) to disambiguate " + "which occurrence to target." + ) + elif matching_count == 1: + raise ContentHashError( + f"Line '{value_stripped}' was found once in the file but could not be " + "resolved as a unique content ID. Use the hashed ID prefix (e.g., `~XXXX~`) " + "to target it precisely." + ) + else: + raise ContentHashError( + f"Line '{value_stripped}' was not found in the file. " + "Use the exact line content or a hashed ID prefix (e.g., `~XXXX~`) " + "to target the desired line." + ) + + +def _detect_overlapping_ranges(resolved_ops): + """Check resolved operations for genuinely overlapping ranges. + + Detects operations where ranges overlap in a way that can't be resolved + by simple deduplication (same-start) or containment. These non-trivial + overlaps (e.g., a delete range partially overlapping a replace range) + are flagged and returned as failed operations with a clear message. + The overlapping operations' indices are also returned so the caller + can exclude them from `resolved_ops`. + + Returns a tuple of (failed_ops, indices_to_remove). + - failed_ops: list of failed operation dicts with overlap errors + - indices_to_remove: set of operation indices to remove from resolved_ops + """ + if len(resolved_ops) < 2: + return [], set() + + failed_ops = [] + indices_to_remove = set() + for i in range(len(resolved_ops)): + a = resolved_ops[i] + a_start = a["start_idx"] + a_end = a["end_idx"] + + for j in range(i + 1, len(resolved_ops)): + b = resolved_ops[j] + b_start = b["start_idx"] + b_end = b["end_idx"] + + # Skip trivial cases handled elsewhere: + # 1. Same start line - handled by _deduplicate_ranges + if a_start == b_start: + continue + + # 2. One range fully contained within the other - handled by _merged_contained_ranges + if (a_start <= b_start and b_end <= a_end) or (b_start <= a_start and a_end <= b_end): + continue + + # Check if ranges genuinely overlap (not just adjacent/contiguous) + # Two ranges [a_start, a_end] and [b_start, b_end] overlap if + # a_start <= b_end and b_start <= a_end + if a_start <= b_end and b_start <= a_end: + op_a = a["op"] + op_b = b["op"] + error_msg = ( + f"Operation {a['index'] + 1} ({op_a['operation']}, lines " + f"{a_start}-{a_end}) overlaps with operation {b['index'] + 1} " + f"({op_b['operation']}, lines {b_start}-{b_end}). " + "Edits with overlapping ranges are not supported. " + "Combine them into a single edit or use non-overlapping ranges." + ) + failed_ops.append({"index": a["index"], "error": error_msg, "operation": op_a}) + failed_ops.append({"index": b["index"], "error": error_msg, "operation": op_b}) + indices_to_remove.add(i) + indices_to_remove.add(j) + + return failed_ops, indices_to_remove + + def apply_hashline_operations( original_content: str, operations: list, @@ -1731,15 +1932,35 @@ def apply_hashline_operations( # Normalize hashline inputs in operations normalized_operations = [] failed_ops = [] + # Create a single HashPos instance for preliminary resolution and content hashing + hp = HashPos(original_content) if original_content else HashPos("") # Loop through each operation to normalize hashline strings for i, op in enumerate(operations): try: normalized_op = op.copy() # Normalize start line hash to ensure consistent format - normalized_op["start_line_hash"] = normalize_hashline(op["start_line_hash"]) + try: + normalized_op["start_line_hash"] = normalize_hashline(op["start_line_hash"]) + except (ContentHashError, ValueError): + # Preliminary resolution: check if the value matches a unique line + resolved = _try_resolve_as_unique_line(hp, op["start_line_hash"]) + if resolved is not None: + normalized_op["start_line_hash"] = resolved + else: + _raise_clear_hash_error(hp, op, "start_line_hash") + if "end_line_hash" in op: # Normalize end line hash if present - normalized_op["end_line_hash"] = normalize_hashline(op["end_line_hash"]) + try: + normalized_op["end_line_hash"] = normalize_hashline(op["end_line_hash"]) + except (ContentHashError, ValueError): + # Preliminary resolution: check if the value matches a unique line + resolved = _try_resolve_as_unique_line(hp, op["end_line_hash"]) + if resolved is not None: + normalized_op["end_line_hash"] = resolved + else: + _raise_clear_hash_error(hp, op, "end_line_hash") + normalized_operations.append(normalized_op) except Exception as e: failed_ops.append({"index": i, "error": str(e), "operation": op}) @@ -1756,7 +1977,7 @@ def apply_hashline_operations( # Apply hashline to original content once # This converts content to hashed lines for line tracking - hashed_content = hashline(original_content) + hashed_content = hp.format_content() hashed_lines = hashed_content.splitlines(keepends=True) # Resolve all operations to indices first @@ -1834,6 +2055,14 @@ def apply_hashline_operations( except Exception as e: failed_ops.append({"index": i, "error": str(e), "operation": op}) + # Check for overlapping ranges among resolved operations + overlap_failures, overlap_indices = _detect_overlapping_ranges(resolved_ops) + for overlap_failure in overlap_failures: + failed_ops.append(overlap_failure) + # Remove overlapping operations from resolved_ops so they don't get applied + if overlap_indices: + resolved_ops = [op for i, op in enumerate(resolved_ops) if i not in overlap_indices] + # Honor cancellations: remove operations that are cancelled by later cancel operations resolved_ops = _honor_cancellations(resolved_ops) # Deduplicate: if multiple operations start on the same line, keep only the latest one diff --git a/cecli/helpers/hashpos/hashpos.py b/cecli/helpers/hashpos/hashpos.py index 9caab1611cb..3a5698f98c8 100644 --- a/cecli/helpers/hashpos/hashpos.py +++ b/cecli/helpers/hashpos/hashpos.py @@ -2,182 +2,188 @@ import xxhash +# Delimiter used to wrap public hash IDs +HASH_DELIMITER = "~" +UNIQUE_HASH_DELIMITER = "~~" + class HashPos: - # ------------------------------------------------------------------------- - # TOKEN-OPTIMIZED PREFIX-FREE ENCODING SETUP - # ------------------------------------------------------------------------- - # flake8: noqa - # fmt: off - _TOKEN_LIST = [ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', #noqa - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', #noqa - 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'GA', 'GB', 'GC', 'GD', 'Ge', 'Gl', #noqa - 'Go', 'Gr', 'Gu', 'HA', 'HD', 'HE', 'HO', 'HP', 'HR', 'HT', 'Ha', 'He', 'Hi', 'Hy', 'IC', 'ID', #noqa - 'IE', 'IF', 'II', 'IL', 'IM', 'IN', 'IO', 'IP', 'IR', 'IS', 'IT', 'IV', 'IX', 'Id', 'If', 'Il', #noqa - 'Im', 'In', 'Ir', 'Is', 'It', 'JO', 'JS', 'Jo', 'Ke', 'Kn', 'LA', 'LE', 'LI', 'LL', 'LO', 'LP', #noqa - 'La', 'Le', 'Li', 'Lo', 'MA', 'MB', 'MC', 'MD', 'ME', 'MI', 'ML', 'MO', 'MP', 'MR', 'MS', 'MT', #noqa - 'MY', 'Ma', 'Mc', 'Me', 'Mi', 'Mo', 'Mr', 'Ms', 'My', 'NA', 'NC', 'NE', 'NL', 'NO', 'NS', 'NT', #noqa - 'NV', 'NY', 'Na', 'Ne', 'No', 'OB', 'OF', 'OK', 'ON', 'OP', 'OR', 'OS', 'Ob', 'Of', 'Oh', 'Ok', #noqa - 'On', 'Op', 'Or', 'Os', 'PA', 'PC', 'PD', 'PE', 'PG', 'PH', 'PI', 'PK', 'PL', 'PM', 'PO', 'PP', #noqa - 'PR', 'PS', 'PT', 'Pa', 'Pe', 'Ph', 'Pi', 'Pl', 'Po', 'Pr', 'Py', 'Qt', 'Qu', 'RC', 'RE', 'RF', #noqa - 'RO', 'RT', 'Ra', 'Re', 'Ro', 'SA', 'SB', 'SC', 'SD', 'SE', 'SF', 'SH', 'SI', 'SK', 'SL', 'SM', #noqa - 'SN', 'SO', 'SP', 'SR', 'SS', 'ST', 'SU', 'SW', 'SY', 'Sc', 'Se', 'Sh', 'Si', 'Sk', 'Sl', 'Sm', #noqa - 'Sn', 'So', 'Sp', 'St', 'Su', 'Sw', 'TD', 'TE', 'TF', 'TH', 'TL', 'TO', 'TR', 'TV', 'TX', 'Te', #noqa - 'Th', 'To', 'Tr', 'Tw', 'Ty', 'UE', 'UI', 'UK', 'UN', 'UP', 'US', 'UV', 'Un', 'Up', 'Ur', 'Us', #noqa - 'VA', 'VI', 'VM', 'VT', 'Va', 'Ve', 'WA', 'WE', 'WH', 'WM', 'WR', 'We', 'Wh', 'Wr', 'XX', 'Ye', #noqa - ] - # fmt: on - # flake8: qa - - # Quick lookups for the 256 bytes - ENCODE_MAP = {i: token for i, token in enumerate(_TOKEN_LIST)} - DECODE_MAP = {token: i for i, token in enumerate(_TOKEN_LIST)} - - # Because all 2-char tokens start with uppercase G-Y, which are never used as standalone - # 1-char tokens, we can cleanly split them. - _PREFIX_CHARS = set("GHIJKLMNOPQRSTUVWXY") - - # Regex building blocks dynamically matching the list logic. - # Single chars are 0-9, A-F, and a-z. Two-char tokens start with G-Y followed by any letter. - _BYTE_REGEX = r"(?:[0-9a-zA-F]|[GHIJKLMNOPQRSTUVWXY][A-Za-z])" - # Regex for HashPos format: {3 encoded bytes}:: - HASH_PREFIX_RE = re.compile(rf"^({_BYTE_REGEX}{{3}})::") - # Regex for normalization: 3 encoded bytes optionally followed by '::' - NORMALIZE_RE = re.compile(rf"^({_BYTE_REGEX}{{3}})(?:::.*)?$") - # Regex for a raw 3-byte encoded fragment - FRAGMENT_RE = re.compile(rf"^{_BYTE_REGEX}{{3}}$") + # 1024-character Base1024 corpus + B1024 = ( + "0123456789ABCDEFGHIJ" + "KLMNOPQRSTUVWXYZabcd" + "efghijklmnopqrstuvwx" + "yz¡£§©«®°±·»¿×ßæðøĐđ" + "ıłœəαβγδεηθικλμνοπρς" + "στυφχωАБВГДЕЗИКЛМНОП" + "РСТУФЦЧЭЯабвгдежзикл" + "мнопрстуфхцчшщъыьэюя" + "іאבדהוחילמנערשת،ابةت" + "ثجحخدذرزسشصضطظعغفقكل" + "منهوىيپکگیकतनपमरलसहন" + "রกขคงจชณดตถทนบปผพมยร" + "ลวสหอะาเแใไ‐–—―‘’“”„" + "†•′※€←→−─│█■●★☆♥♪、。《" + "》「」『』【】〜あいうえおかきくけこさし" + "すせそたちっつてとなにのはまみめもやよら" + "りるれろわをんアィイウェエオカキクコサシ" + "スセタチッテトナニフマムメャュョラリルレ" + "ロン・ー一万三上下不与专业东两个中为主么" + "义之也书了事二于五些交产享京人亿今介从他" + "付代以们件价任份企会传但位体何余作你使例" + "供価保信修倍值停像元先入全公共关其具内円" + "册再写出击分列则初利别到制前力功加务动動" + "包化北区十午华单南即历原去县参及友反发取" + "变口只可台右号司合同名后向否含听启告员周" + "命和品哈商問器四回因国图土在地场型城基報" + "場增声处备复外多大天失头女好如始子字存学" + "安完定实客家容密对导将小少尔就局展山岁州" + "工左已市布常平年并广序库应店度建开式引张" + "当录形影径待後得微心必志态思性总息您情意" + "感成我或户所手打技投报拉持指按换据排接推" + "提播支收改放政效数整文料断新方族无日时明" + "易星是時景更最月有服期木未本机权束条来板" + "构果查标样格案模次款止正此步歳段每比民気" + "水求江没治法注活流海消清游源火点無然片版" + "物特率环现球理生用由电男画界番登的目直相" + "省看県真知码确示社票私种科秒称移程税空立" + "站章端笑符第等简算管米类系素索约级线组经" + "结给统编网置美老考者而联能自至色节英藏行" + "表装西要見见规视角解言計記話読计认议记论" + "设证评试话该语误说请读调象责败货费资起超" + "路身车转载辑输达过运近还这进连述退送选通" + "速造連道部都配释里重量金错长開間関门闭问" + "间队阳陆限院除雅集雷需非面音页项题首验高" + "黑가간개거게결경고공과구그글기나내는능니" + "다당대도동되된드든들디라래러력로록료류른" + "를름리만메면명목문미버번보복부분비사산상" + "색생서성세션소수스습시식신아야어에여열오" + "와요용우운원위으은을음의이인일임입자작장" + "재적전정제져조주지진째체출치크태터턴트하" + "한할함해호화환회�ª²³´µ¹º¼½ÀÁ" + "ÂÃÄÇ" + ) + + _B1024_REGEX_SET = "".join(re.escape(c) for c in B1024) + + # Regex matches EITHER the exact string '~~' OR a tilde-wrapped 4-character Base1024 hash + HASH_PREFIX_RE = re.compile( + rf"^({UNIQUE_HASH_DELIMITER}|{HASH_DELIMITER}[{_B1024_REGEX_SET}]{{4}}{HASH_DELIMITER})" + ) + FRAGMENT_RE = re.compile( + rf"^({UNIQUE_HASH_DELIMITER}|{HASH_DELIMITER}[{_B1024_REGEX_SET}]{{4}}{HASH_DELIMITER})$" + ) + + # Loose prefix for robust stripping: Matches a tilde-wrapped 4-char string containing non-ASCII + _LOOSE_PREFIX_RE = re.compile( + rf"^{HASH_DELIMITER}(?=.{{0,3}}[^\x00-\x7f]).{{4}}{HASH_DELIMITER}" + ) def __init__(self, source_text: str = ""): self.lines = source_text.splitlines() self.total = len(self.lines) - def _get_region_val(self, line_idx: int) -> int: - """ - Maps the line to one of 16 proportional vertical buckets (4 bits). - This acts as a binary space partition: - - bit 3 is top/bottom half - - bit 2 is top/bottom quarter - - bit 1 is top/bottom eighth - - bit 0 is top/bottom sixteenth - """ - if self.total == 0: - return 0 - - # Calculate which 16th of the file the line falls into - region = (line_idx * 16) // self.total - - # Clamp to 15 to handle edge cases safely - return min(region, 15) - - def _get_neighborhood_hash(self, line_idx: int) -> int: - """ - Creates a 20-bit digest using the current line and the 2 lines - before and after it. - """ - start = max(0, line_idx - 2) - end = min(self.total, line_idx + 3) - - context_window = "\n".join(self.lines[start:end]) - full_hash = xxhash.xxh3_64_intdigest(context_window.encode("utf-8")) - - # Isolate exactly 20 bits - return full_hash & 0xFFFFF - - def generate_public_id(self, text: str, line_idx: int) -> str: - """ - Generates a 3-to-6 char ID using the token-optimized prefix-free encoding. - Layout: [20-bit Neighborhood Hash] [4-bit Region] = 24 bits total. - """ - neighborhood_hash = self._get_neighborhood_hash(line_idx) - region_val = self._get_region_val(line_idx) - - # Pack the 24-bit integer - packed = (neighborhood_hash << 4) | region_val - - # Encode 3 bytes using the prefix-free map - res = "" - for _ in range(3): - byte_segment = packed % 256 - res += self.ENCODE_MAP[byte_segment] - packed //= 256 + self.line_counts = {} + for line in self.lines: + if line.strip(): + self.line_counts[line] = self.line_counts.get(line, 0) + 1 - return res + def _get_line_hash(self, text: str) -> int: + return xxhash.xxh3_64_intdigest(text.encode("utf-8")) & 0xFFFFF - def unpack_public_id(self, public_id: str) -> tuple[int, int]: - """ - Reverses the Public ID back into its (Neighborhood Hash, Region Value) values. - Reads the prefix-free string left-to-right to unambiguously decode the bytes. - """ - packed = 0 - byte_shift = 0 - i = 0 + def generate_public_id(self, text: str, line_idx: int, occurrence: int) -> str: + line_hash = self._get_line_hash(text) - while i < len(public_id): - char = public_id[i] + # Explicit modulo for bounds wrapping + idx_bits = line_idx % 16384 + occ_bits = occurrence % 64 - # The G-Y characters explicitly signal a two-character sequence - if char in self._PREFIX_CHARS: - seq = public_id[i : i + 2] - i += 2 - else: - seq = char - i += 1 + packed = (line_hash << 20) | (idx_bits << 6) | occ_bits - byte_val = self.DECODE_MAP[seq] - packed |= byte_val << byte_shift - byte_shift += 8 + res = "" + for i in range(3, -1, -1): + res += self.B1024[(packed >> (10 * i)) & 0x3FF] + return res - # Extract the 20-bit hash (shift right by 4, mask 0xFFFFF) - neighborhood_hash = (packed >> 4) & 0xFFFFF + def unpack_public_id(self, public_id: str) -> tuple[int, int, int]: + packed = 0 + for i, char in enumerate(public_id): + packed |= self.B1024.index(char) << (10 * (3 - i)) - # Extract the 4-bit region from the lowest bits - region_val = packed & 0xF + occ_bits = packed & 0x3F + idx_bits = (packed >> 6) & 0x3FFF + line_hash = (packed >> 20) & 0xFFFFF - return neighborhood_hash, region_val + return line_hash, idx_bits, occ_bits def format_content(self, start_line: int = 1) -> str: formatted_lines = [] + seen = {} + for i, line in enumerate(self.lines): - prefix = self.generate_public_id(line, i) - if line.strip(): - formatted_lines.append(f"{prefix}::{line}") - else: + if not line.strip(): formatted_lines.append(f"{line}") + continue + + count = self.line_counts[line] + + if count == 1: + # Flush directly against code using the unique token + formatted_lines.append(f"{UNIQUE_HASH_DELIMITER}{line}") + else: + occ = seen.get(line, 0) + 1 + seen[line] = occ + prefix = self.generate_public_id(line, i, occ) + # Wrap the generated Base1024 hash in tildes + formatted_lines.append(f"{self.get_wrapped_id(prefix)}{line}") return "\n".join(formatted_lines) def resolve_to_lines(self, public_id: str, start_line: int = 1) -> list[int]: - target_hash, target_region = self.unpack_public_id(public_id) - matches = [] + if public_id == UNIQUE_HASH_DELIMITER: + raise ValueError( + f"Cannot spatially resolve the unique '{UNIQUE_HASH_DELIMITER}' identifier without line text." + ) + + # Strip the surrounding tildes to unpack the core 4 characters + clean_id = public_id.strip(HASH_DELIMITER) + if len(clean_id) != 4: + raise ValueError(f"Invalid public ID string for unpacking: {public_id}") + + target_line_hash, target_idx, target_occ = self.unpack_public_id(clean_id) - # Find all lines whose neighborhood hash matches our target + matches = [] for i, line in enumerate(self.lines): - if self._get_neighborhood_hash(i) == target_hash: - matches.append(i) + if self._get_line_hash(line) == target_line_hash: + matches.append((i, line)) if not matches: return [] - # If perfectly unique, return it immediately if len(matches) == 1: - return matches + return [matches[0][0]] + + current_seen = {} + scored_matches = [] + + for i, line in matches: + current_seen[line] = current_seen.get(line, 0) + 1 + current_occ = current_seen[line] - # Distance Heuristic: If multiple matches exist (e.g. repeated code blocks), - # prioritize the one whose current binary region is closest to the target region. - def region_distance(idx: int) -> int: - current_region = self._get_region_val(idx) - # Linear distance because proportional regions don't wrap around - return abs(current_region - target_region) + # Apply identical modulo to current spatial data before comparing + current_idx_mod = i % 16384 + current_occ_mod = current_occ % 64 - matches.sort(key=region_distance) + # Cartesian distance squared + distance_sq = ((current_idx_mod - target_idx) ** 2) + ( + (current_occ_mod - target_occ) ** 2 + ) + scored_matches.append((distance_sq, i)) - return matches + scored_matches.sort(key=lambda x: x[0]) + return [m[1] for m in scored_matches] def resolve_range(self, start_id: str, end_id: str) -> tuple[int, int]: - """ - Resolves a block range from two Public IDs. - """ starts = self.resolve_to_lines(start_id) ends = self.resolve_to_lines(end_id) @@ -190,48 +196,42 @@ def resolve_range(self, start_id: str, end_id: str) -> tuple[int, int]: return s, e raise ValueError( - f"Found matches for {start_id} and {end_id}, but no logically ordered range or unique matches." + f"Found matches for {start_id} and {end_id}, but no logically ordered range." ) + @staticmethod + def get_wrapped_id(public_id: str) -> str: + """Wrap a public ID with the HashPos delimiters for use in hashline content.""" + return f"{HASH_DELIMITER}{public_id}{HASH_DELIMITER}" + @staticmethod def strip_prefix(text: str) -> str: - """ - Remove HashPos prefixes from the start of every line. - """ lines = text.splitlines(keepends=True) result_lines = [] for line in lines: stripped_line = HashPos.HASH_PREFIX_RE.sub("", line, count=1) + if stripped_line == line: + stripped_line = HashPos._LOOSE_PREFIX_RE.sub("", line, count=1) result_lines.append(stripped_line) - return "".join(result_lines) @staticmethod def extract_prefix(line: str) -> str: - """ - Extract the hash prefix from a line if it has a HashPos prefix. - """ match = HashPos.HASH_PREFIX_RE.match(line) if match: return match.group(1) return "" @staticmethod - def normalize(hashpos_str: str) -> str: - """ - Normalize a HashPos string to the exact matched prefix fragment. - """ + def normalize(hashpos_str: str, throw=True) -> str: if hashpos_str is None: raise ValueError("HashPos string cannot be None") - if HashPos.FRAGMENT_RE.match(hashpos_str): - return hashpos_str - - match = HashPos.NORMALIZE_RE.match(hashpos_str) + match = HashPos.HASH_PREFIX_RE.match(hashpos_str) if match: return match.group(1) - raise ValueError( - f"Invalid HashPos format '{hashpos_str}'. " - r"Expected a valid content ID followed by `::`" - ) + if throw: + raise ValueError(f"Invalid HashPos format '{hashpos_str}'.") + else: + return False diff --git a/cecli/helpers/hashpos/transformations.py b/cecli/helpers/hashpos/transformations.py new file mode 100644 index 00000000000..162a1aca50b --- /dev/null +++ b/cecli/helpers/hashpos/transformations.py @@ -0,0 +1,476 @@ +""" +HashPos transformation utilities. + +Centralizes all operations related to searching, processing line hints, +contextual markers, and extracting content from HashPos-encoded files. +Previously scattered across read_file.py, edit_file.py, hashline.py, +and region_resolver.py. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from cecli.helpers.hashpos.hashpos import UNIQUE_HASH_DELIMITER + +if TYPE_CHECKING: + from cecli.helpers.hashpos.hashpos import HashPos + + +# ──────────────────────────────────────────────────────────── +# Pattern Matching +# ──────────────────────────────────────────────────────────── + + +def search_in_lines( + lines: list[str], + pattern: str, + *, + return_last_line: bool = False, +) -> list[int]: + """Search for a multiline pattern in lines. + + Each line of *pattern* must appear as a whole word in the corresponding + line in *lines* at the match position. A ``whole word`` means the + pattern is surrounded by non-word characters (or line boundaries) — + ``"def"`` matches ``"def foo()"`` but not ``"define"``. + + Args: + lines: Source lines to search. + pattern: Pattern string (may span multiple lines via ``\n``). + return_last_line: When True, returns the index of the **last** + line of each match instead of the first. + + Returns: + 0-based indices of each match. + """ + + pattern_lines = pattern.split("\n") + indices: list[int] = [] + offset = len(pattern_lines) - 1 if return_last_line else 0 + + for i in range(len(lines) - len(pattern_lines) + 1): + if all( + _is_whole_word_match(p_line, lines[i + j]) for j, p_line in enumerate(pattern_lines) + ): + indices.append(i + offset) + + return indices + + +def _is_whole_word_match(pattern: str, line: str) -> bool: + """Return True if *pattern* appears as a whole word in *line*. + + A match requires that the characters immediately before and after the + match position are **not** word characters (``[a-zA-Z0-9_]``) — or + that the match is at the start/end of the line respectively. + """ + + idx = line.find(pattern) + + while idx != -1: + before_ok = idx == 0 or not (line[idx - 1].isalnum() or line[idx - 1] == "_") + after_pos = idx + len(pattern) + after_ok = after_pos >= len(line) or not ( + line[after_pos].isalnum() or line[after_pos] == "_" + ) + + if before_ok and after_ok: + return True + + idx = line.find(pattern, idx + 1) + + return False + + +def find_substring_matches( + lines: list[str], + value: str, +) -> list[int]: + """Return 0-based indices of all lines containing *value* as a substring.""" + + value_stripped = value.strip() + return [i for i, line in enumerate(lines) if value_stripped in line] + + +def find_multiline_match( + lines: list[str], + value: str, +) -> int | None: + """Find the start index where the full multiline *value* matches consecutive lines. + + Returns None if *value* is a single line or no match is found. + """ + + value_lines = value.strip().splitlines() + + if len(value_lines) <= 1: + return None + + for i in range(len(lines) - len(value_lines) + 1): + if all(value_lines[j].strip() in lines[i + j] for j in range(len(value_lines))): + return i + + return None + + +# ──────────────────────────────────────────────────────────── +# Content ID Detection +# ──────────────────────────────────────────────────────────── + + +def is_content_id(value: str) -> bool: + """Return True if *value* appears to be a content ID rather than text.""" + + from cecli.helpers.hashline import ContentHashError, normalize_hashline + + if value in ("@000", "000@"): + return True + + try: + normalize_hashline(value) + + return True + except (ContentHashError, ValueError): + return False + + +# ──────────────────────────────────────────────────────────── +# @L Line Number Resolution +# ──────────────────────────────────────────────────────────── + + +def resolve_at_l( + line_spec: str, + hp: HashPos, + lines: list[str], +) -> str: + """Resolve ``@L{{num}}`` notation to a content ID using a HashPos index. + + Returns the input unchanged if it is not an @L{{num}} spec. + Raises ValueError if the line number is out of range. + """ + + if not ( + isinstance(line_spec, str) + and line_spec.startswith("@L") + and len(line_spec) > 2 + and line_spec[2:].isdigit() + ): + return line_spec + + line_num = int(line_spec[2:]) - 1 + + if line_num < 0 or line_num >= len(lines): + raise ValueError( + f"@L reference line {int(line_spec[2:])} is out of range " + f"(file has {len(lines)} lines)" + ) + + line_text = lines[line_num] + occurrence = 1 + sum(1 for i in range(line_num) if lines[i] == line_text) + + return hp.get_wrapped_id(hp.generate_public_id(line_text, line_num, occurrence)) + + +# ──────────────────────────────────────────────────────────── +# Hint Extraction (@L / @A / @B) +# ──────────────────────────────────────────────────────────── + + +def extract_hint( + pattern: str, + lines: list[str] | None = None, +) -> tuple[str, int | None, str | None]: + """Extract a hint suffix from a pattern string. + + Supports three hint types: + + * ``@L`` — direct line number (always resolvable). + * ``@A{{{{regex}}}}`` — keep only matches **after** the first regex hit. + * ``@B{{{{regex}}}}`` — keep only matches **before** the last regex hit. + + Returns a ``(stripped_pattern, hint_value, hint_type)`` tuple: + + * ``hint_value`` — 0-based line number, or None. + * ``hint_type`` — ``'L'``, ``'A'``, ``'B'``, or None. + """ + + # @L hint — direct line number, always resolvable + m = re.search(r"[ \t]+@L([0-9]+)[ \t]*$", pattern) + + if m: + return pattern[: m.start()], int(m.group(1)) - 1, "L" + + # @A{{regex}} hint — first regex match, filter to lines AFTER + m = re.search(r"[ \t]+@A\{\{(.+?)\}\}[ \t]*$", pattern) + + if m: + stripped = pattern[: m.start()] + + if lines is not None: + regex_str = m.group(1) + + try: + for i, line in enumerate(lines): + if re.search(regex_str, line): + return stripped, i, "A" + except re.error: + pass + + return stripped, None, None + + # @B{{regex}} hint — last regex match, filter to lines BEFORE + m = re.search(r"[ \t]+@B\{\{(.+?)\}\}[ \t]*$", pattern) + + if m: + stripped = pattern[: m.start()] + + if lines is not None: + regex_str = m.group(1) + + try: + last_match = None + + for i, line in enumerate(lines): + if re.search(regex_str, line): + last_match = i + + if last_match is not None: + return stripped, last_match, "B" + except re.error: + pass + + return stripped, None, None + + return pattern, None, None + + +# ──────────────────────────────────────────────────────────── +# Search Type Classification +# ──────────────────────────────────────────────────────────── + + +def classify_search_type( + range_start: str, + range_end: str, +) -> dict[str, bool]: + """Classify range markers into structured, text, mixed, or contextual types. + + The returned dict contains boolean flags usable by downstream resolvers + to decide how to find start/end line indices. + """ + + def _is_line_ref(s: str) -> bool: + return s.startswith("@L") and s[2:].isdigit() and len(s) > 2 + + start_is_lr = _is_line_ref(range_start) + end_is_lr = _is_line_ref(range_end) + start_is_sp = range_start in ("@000", "000@") + end_is_sp = range_end in ("@000", "000@") + + return { + "start_is_line_ref": start_is_lr, + "end_is_line_ref": end_is_lr, + "start_is_special": start_is_sp, + "end_is_special": end_is_sp, + "start_is_text": not start_is_lr and not start_is_sp, + "end_is_text": not end_is_lr and not end_is_sp, + "both_structured": (start_is_lr or start_is_sp) and (end_is_lr or end_is_sp), + "mixed_special": ( + (start_is_sp and not end_is_lr and not end_is_sp) + or (end_is_sp and not start_is_lr and not start_is_sp) + ), + "end_is_contextual": ( + range_end.startswith(("@C", "@P", "@N")) + and len(range_end) > 2 + and range_end[2:].isdigit() + ), + } + + +# ──────────────────────────────────────────────────────────── +# Contextual Markers (@C / @P / @N) +# ──────────────────────────────────────────────────────────── + + +def apply_contextual_marker( + start_indices: list[int], + range_start: str, + range_end: str, + num_lines: int, +) -> tuple[list[int], list[int]]: + """Expand range using @C{{num}}, @P{{num}}, or @N{{num}} contextual markers. + + Requires exactly one start match. Raises ValueError when the number + of start matches is not 1 (the caller is expected to format the error + message with its own context like file path / operation index). + + Returns ``(new_start_indices, new_end_indices)``. + """ + + if len(start_indices) != 1: + raise ValueError( + f"Start pattern '{range_start}' must match exactly one" + f" location when using @C/@P/@N end markers." + f" Found {len(start_indices)} matches." + ) + + ctx_s_idx = start_indices[0] + ctx_num = int(range_end[2:]) + marker_type = range_end[1] + + if marker_type == "C": + s_idx = max(0, ctx_s_idx - ctx_num) + e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) + elif marker_type == "P": + s_idx = max(0, ctx_s_idx - ctx_num) + e_idx = ctx_s_idx + else: # 'N' + s_idx = ctx_s_idx + e_idx = min(num_lines - 1, ctx_s_idx + ctx_num) + + return [s_idx], [e_idx] + + +# ──────────────────────────────────────────────────────────── +# Proximity / Narrowing +# ──────────────────────────────────────────────────────────── + + +def narrow_by_proximity( + indices: list[int], + target: int, + max_results: int = 5, +) -> list[int]: + """Return up to *max_results* indices closest to *target* (0-based). + + Results are sorted by ascending distance from *target*. + """ + + if not indices: + return [] + + scored = [(abs(i - target), i) for i in indices] + scored.sort(key=lambda x: x[0]) + + return [idx for _, idx in scored[:max_results]] + + +# ──────────────────────────────────────────────────────────── +# Pair / Range Computation +# ──────────────────────────────────────────────────────────── + + +def compute_best_pair( + start_indices: list[int], + end_indices: list[int], + *, + allow_inverted: bool = False, +) -> tuple[int, int] | None: + """Find the smallest-range valid (start, end) pair. + + When *allow_inverted* is True, also considers pairs where end < start + (the LLM may have swapped the order) and returns them in correct order. + """ + + min_dist = float("inf") + best_pair: tuple[int, int] | None = None + + for s in start_indices: + valid_ends = end_indices if allow_inverted else [e for e in end_indices if e >= s] + + for e in valid_ends: + dist = abs(e - s) + + if dist < min_dist: + min_dist = dist + best_pair = (s, e) + + if allow_inverted and best_pair and best_pair[1] < best_pair[0]: + best_pair = (best_pair[1], best_pair[0]) + + return best_pair + + +def reposition_indices( + target_idx: int, + start_idx: int, + end_idx: int, + total_lines: int = 20, +) -> tuple[int, int]: + """Calculate clamped start/end indices for a centered window. + + Returns ``(slice_start, slice_end)`` compatible with Python slicing + (i.e. *slice_end* is exclusive). + """ + + half_window = total_lines // 2 + + left = target_idx - half_window + right = target_idx + half_window + + if left < start_idx: + right += start_idx - left + left = start_idx + + if right > end_idx: + left -= right - end_idx + right = end_idx + + left = max(start_idx, left) + + return left, right + 1 + + +# ──────────────────────────────────────────────────────────── +# Prefix / Content ID Utilities +# ──────────────────────────────────────────────────────────── + + +def strip_hashline_prefix(value: str) -> str: + """Strip the ``~~`` virtual prefix from a ReadFile output line reference.""" + + if isinstance(value, str) and value.startswith(UNIQUE_HASH_DELIMITER): + return value[len(UNIQUE_HASH_DELIMITER) :].lstrip() + + return value + + +def try_resolve_as_unique_line( + hp: HashPos, + value: str, +) -> str | None: + """Try to resolve a value by matching it against unique lines in the source. + + If *value* (after stripping) matches exactly one line in the source + **and** that line appears only once (unique), returns a tilde-wrapped + hash ID that can be resolved to line indices by the HashPos engine. + + Returns None if resolution fails. + """ + + if not hp or not value: + return None + + value_stripped = value.strip() + + if not value_stripped: + return None + + matching_lines: list[tuple[int, str]] = [] + + for i, line in enumerate(hp.lines): + if line.strip() == value_stripped: + matching_lines.append((i, line)) + + if len(matching_lines) == 1: + idx, matched_line = matching_lines[0] + + if hp.line_counts.get(matched_line, 0) == 1: + hash_id = hp.generate_public_id(matched_line, idx, 1) + + return hp.get_wrapped_id(hash_id) + + return None diff --git a/cecli/helpers/leak_detect.py b/cecli/helpers/leak_detect.py index f4863844eb0..320bf217eb8 100644 --- a/cecli/helpers/leak_detect.py +++ b/cecli/helpers/leak_detect.py @@ -25,6 +25,19 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional +import psutil + +# Track the initial baseline memory at module load time +_BASELINE_MEM = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) + + +def check_memory(label="Current"): + """Prints absolute memory usage and growth since the script started.""" + current_mem = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024) + growth = current_mem - _BASELINE_MEM + print(f"[{label}] Total: {current_mem:.2f} MB | Growth: +{growth:.2f} MB") + + # ── Optional dependency wrappers ── diff --git a/cecli/helpers/lock_detect.py b/cecli/helpers/lock_detect.py new file mode 100644 index 00000000000..6eecd852c44 --- /dev/null +++ b/cecli/helpers/lock_detect.py @@ -0,0 +1,54 @@ +import logging +import sys +import threading +import time +import traceback + +# Set up logging to catch asyncio framework logs +logging.basicConfig(level=logging.INFO) +logging.getLogger("asyncio").setLevel(logging.DEBUG) + + +def dump_stacks_to_file(filename="hang_dump.log", interval=5, max_prints=10): + """Periodically writes stack traces to a file, resetting it after max_prints.""" + print_count = 0 + + while True: + time.sleep(interval) + try: + # Determine mode: "w" to overwrite/clear on the 11th print, "a" to append + mode = "w" if print_count >= max_prints else "a" + + with open(filename, mode, encoding="utf-8") as f: + if mode == "w": + f.write(f"--- Log reset automatically after {max_prints} prints ---\n") + print_count = 0 # Reset the counter + + timestamp = time.strftime("%Y-%m-%d %H:%M:%S") + f.write( + f"\n=================== STACK TRACE DUMP ({timestamp}) ===================\n" + ) + + for thread_id, frame in sys._current_frames().items(): + thread_obj = threading._active.get(thread_id) + thread_name = thread_obj.name if thread_obj else "Unknown Thread" + + f.write(f"\nThread Name: {thread_name} (ID: {thread_id})\n") + f.write("-" * 40 + "\n") + traceback.print_stack(frame, file=f) + + f.write("=" * 70 + "\n") + + print_count += 1 # Increment after a successful write + + except Exception as e: + print(f"Error writing stack dump to file: {e}", file=sys.stderr) + + +# Start the monitor in a background daemon thread +monitor_thread = threading.Thread( + target=dump_stacks_to_file, + args=("program_hangs.log", 5, 10), # Clears every 10 prints + daemon=True, +) +monitor_thread.start() diff --git a/cecli/helpers/nested.py b/cecli/helpers/nested.py index c77be72b717..3f70302f0e6 100644 --- a/cecli/helpers/nested.py +++ b/cecli/helpers/nested.py @@ -1,5 +1,10 @@ +import re from typing import Any, Dict, List, Union +# Precompiled regex for normalizing bracket notation to dot notation in paths +# e.g., "result[0]._.file_path" -> "result.0._.file_path" +_BRACKET_RE = re.compile(r"\[(\d+)\]") + def arg_resolver(obj: Union[List[Any], Dict[str, Any], Any], key: str, default: Any = None) -> Any: """ @@ -54,7 +59,12 @@ def arg_resolver(obj: Union[List[Any], Dict[str, Any], Any], key: str, default: def getter( data: Union[List[Any], Dict[str, Any], Any], path: Union[str, List[str]], default: Any = None ) -> Any: - """Safely access nested dicts, lists, and objects using normalized dot-notation.""" + """Safely access nested dicts, lists, and objects using normalized dot-notation. + + Supports both: + getter(data, "result.0._.file_path") # dot notation + getter(data, "result[0]._.file_path") # bracket notation + """ if data is None: return default @@ -65,6 +75,9 @@ def getter( else: paths = path + # Normalize bracket notation to dot notation + paths = [_BRACKET_RE.sub(r".\1", p) for p in paths] + # Try each path, return first valid result for path_str in paths: current = data diff --git a/cecli/helpers/orchestration/__init__.py b/cecli/helpers/orchestration/__init__.py new file mode 100644 index 00000000000..2ca920fd59c --- /dev/null +++ b/cecli/helpers/orchestration/__init__.py @@ -0,0 +1,35 @@ +""" +Orchestration module for programmatic tool calling in agent mode. + +Public API: +- AgentExecutionEnv: sandboxed execution environment +- AgentProxy / ToolProxy: proxies for calling tools from generated code +- build_orchestration_context_block: context block builder +- SecurityError / SecurityFilter / LoopYieldInjector: security components +""" + +from cecli.helpers.orchestration.environment import ( + AgentExecutionEnv, + AgentProxy, + ToolProxy, + build_orchestration_context_block, +) +from cecli.helpers.orchestration.security import ( + LoopYieldInjector, + SecurityError, + SecurityFilter, + _cooperative_yield, +) +from cecli.helpers.orchestration.service import OrchestrationService + +__all__ = [ + "AgentExecutionEnv", + "AgentProxy", + "ToolProxy", + "build_orchestration_context_block", + "SecurityError", + "SecurityFilter", + "LoopYieldInjector", + "_cooperative_yield", + "OrchestrationService", +] diff --git a/cecli/helpers/orchestration/agent_proxy.py b/cecli/helpers/orchestration/agent_proxy.py new file mode 100644 index 00000000000..6e20cd886ea --- /dev/null +++ b/cecli/helpers/orchestration/agent_proxy.py @@ -0,0 +1,566 @@ +""" +Singleton-like proxy injected into the orchestration environment. + +Usage in LLM-generated code: + + read_tool = Agent.get_tool("ReadFile") + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + +Supports both local tools (from ToolRegistry) and MCP tools (from connected +servers) using `ServerName--ToolName` or bare tool-name lookup. +""" + +from __future__ import annotations + +from typing import Any + +from cecli.helpers import nested, responses +from cecli.helpers.orchestration.tool_proxy import ToolProxy + + +class AgentProxy: + """ + Singleton-like proxy injected into the orchestration environment. + + Usage in LLM-generated code: + + read_tool = Agent.get_tool("ReadFile") + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + + Supports both local tools (from ToolRegistry) and MCP tools (from connected + servers) using `ServerName--ToolName` or bare tool-name lookup. + """ + + def __init__(self, coder: Any) -> None: + self._coder = coder + self._env = None # Set by AgentExecutionEnv after creation + + def get_tool(self, tool_name: str) -> ToolProxy: + from cecli.tools.utils.registry import ToolRegistry + + name_lower = tool_name.lower() + + # 1. Try local tools by exact name (unprefixed) + tool_module = ToolRegistry.get_tool(name_lower) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 2. Unprefix: "ServerName--ToolName" → (server_prefix, bare_name) + server_prefix, bare_name = responses.unprefix_tool_name(name_lower) + + # 3. If the prefix is "local", retry ToolRegistry with the bare name + if server_prefix == "local" and bare_name: + tool_module = ToolRegistry.get_tool(bare_name) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + # Also try with hyphen normalization on the bare name + bare_underscored = bare_name.replace("-", "_") + if bare_underscored != bare_name: + tool_module = ToolRegistry.get_tool(bare_underscored) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 3b. Replace hyphens with underscores and retry (kebab-case -> snake_case) + underscored = name_lower.replace("-", "_") + if underscored != name_lower: + tool_module = ToolRegistry.get_tool(underscored) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 3c. Strip underscores from the tool name and retry (snake_case fallback) + no_underscore = name_lower.replace("_", "") + if no_underscore != name_lower: + tool_module = ToolRegistry.get_tool(no_underscore) + if tool_module is not None: + return ToolProxy(tool_name, self._coder, tool_module=tool_module) + + # 4. Search MCP tools for the bare (unprefixed) name + for mcp_server_name, server_tools in self._coder.mcp_tools or []: + for tool_schema in server_tools: + schema_name = nested.getter(tool_schema, "function.name", "") + _schema_prefix, schema_unprefixed = responses.unprefix_tool_name( + schema_name.lower() + ) + if schema_unprefixed == bare_name: + server = self._find_mcp_server(mcp_server_name, server_prefix) + if server is not None: + return ToolProxy( + tool_name, + self._coder, + mcp_server=server, + mcp_tool_name=schema_name, + ) + + raise ValueError(f"Unknown tool: '{tool_name}'") + + def _find_mcp_server(self, server_name: str, server_prefix: str) -> Any: + if not hasattr(self._coder, "mcp_manager") or not self._coder.mcp_manager: + return None + for server in self._coder.mcp_manager: + if server.name == server_name and ( + not server_prefix or server.name.lower() == server_prefix.lower() + ): + return server + return None + + def peek(self, result: Any) -> str: + """Inspect the structure of a tool result and return a readable summary. + + Tool results have a standard shape: + + { + "result": [{"content": ..., "_": {"file_path": ..., ...}}], + "errors": [...], + "details": [...] + } + + This method unwraps the structure to show what keys are available, + helping the LLM navigate deeply nested tool outputs. Leaf values + include a short content snippet so you can see actual data. + + Example: + + output = await grep_tool.call(pattern="TODO", file_glob="*.py") + print(Agent.peek(output)) + # Shows: result[0].content: str = 'def foo():...' + # result[0]._.file_path: str = 'src/main.py' + + # Now the LLM can confidently access: + for item in output["result"]: + print(item["content"]) # or item["_"]["file_path"] + + Returns a multi-line string describing the available access paths + with short content previews for leaf values. + """ + + return self._inspect_structure(result) + + def get_value(self, result: Any, path: str, default: Any = None) -> Any: + """Safely access nested values in a tool result using dot-notation. + + Tool results have deeply nested dicts (e.g., ``result["result"][0]["_"]["file_path"]``). + ``get_value()`` provides a concise shorthand using ``nested.getter()``. + + Example: + + output = await grep_tool.call(pattern="TODO", file_glob="*.py") + file_path = Agent.get_value(output, "result.0._.file_path") + content = Agent.get_value(output, "result.0.content") + + Returns *default* if the path does not exist. + """ + from cecli.helpers import nested + + # SECURITY: Reject paths that access private/dunder attributes + # (unless disable_security is active) + if not (self._env and self._env._orchestration_config.get("disable_security", False)): + for segment in path.replace("[", ".").replace("]", "").split("."): + segment = segment.strip() + if segment.startswith("_") and segment != "_": + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError(f"Access to private attribute '{segment}' is forbidden") + + return nested.getter(result, path, default) + + @staticmethod + def _content_preview(value: Any, max_chars: int = 20) -> str: + """Return a short preview of a scalar value's stringified content.""" + + s = str(value) + if len(s) <= max_chars: + return repr(s) + + return repr(s[:max_chars]) + "..." + + @staticmethod + def _inspect_structure(obj: Any, prefix: str = "", depth: int = 0) -> str: + """Recursively inspect the structure of a tool result — paths, types, and content previews.""" + + max_depth = 3 + max_keys = 5 + lines: list[str] = [] + + if depth > max_depth: + return "" + + if isinstance(obj, dict): + for key, value in obj.items(): + path = f"{prefix}.{key}" if prefix else key + + if isinstance(value, dict): + keys = list(value.keys())[:max_keys] + suffix = "..." if len(value) > max_keys else "" + lines.append(f"{path}: dict[{' | '.join(keys)}{suffix}]") + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + elif isinstance(value, list): + if value: + first = value[0] + if isinstance(first, dict): + lines.append(f"{path}: list[{len(value)}] dict") + inner = AgentProxy._inspect_structure(first, f"{path}[0]", depth + 1) + if inner: + lines.append(inner) + else: + lines.append(f"{path}: list[{len(value)}] {type(first).__name__}") + else: + lines.append(f"{path}: list (empty)") + elif ( + hasattr(value, "keys") + and hasattr(value, "items") + and not isinstance(value, (str, bytes)) + ): + sub_keys = list(value.keys())[:max_keys] + sub_suffix = "..." if len(value) > max_keys else "" + lines.append( + f"{path}: {type(value).__name__}(keys: [{', '.join(sub_keys)}{sub_suffix}])" + ) + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + else: + preview = AgentProxy._content_preview(value) + lines.append(f"{path}: {type(value).__name__} = {preview}") + + elif isinstance(obj, list): + if obj: + first = obj[0] + if isinstance(first, dict): + lines.append(f"list[{len(obj)}] dict") + inner = AgentProxy._inspect_structure( + first, f"{prefix}[0]" if prefix else "[0]", depth + 1 + ) + if inner: + lines.append(inner) + else: + lines.append(f"list[{len(obj)}] {type(first).__name__}") + else: + lines.append("list (empty)") + + elif hasattr(obj, "keys") and hasattr(obj, "items") and not isinstance(obj, (str, bytes)): + keys = list(obj.keys()) + display_keys = keys[:max_keys] + suffix = "..." if len(keys) > max_keys else "" + lines.append(f"{type(obj).__name__}(keys: [{', '.join(display_keys)}{suffix}])") + for key in display_keys: + value = obj[key] + path = f"{prefix}.{key}" if prefix else key + + if isinstance(value, dict): + sub_keys = list(value.keys())[:max_keys] + sub_suffix = "..." if len(value) > max_keys else "" + lines.append(f"{path}: dict[{' | '.join(sub_keys)}{sub_suffix}]") + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + elif isinstance(value, list): + if value: + first = value[0] + if isinstance(first, dict): + lines.append(f"{path}: list[{len(value)}] dict") + inner = AgentProxy._inspect_structure(first, f"{path}[0]", depth + 1) + if inner: + lines.append(inner) + else: + lines.append(f"{path}: list[{len(value)}] {type(first).__name__}") + else: + lines.append(f"{path}: list (empty)") + elif ( + hasattr(value, "keys") + and hasattr(value, "items") + and not isinstance(value, (str, bytes)) + ): + sub_keys = list(value.keys())[:max_keys] + sub_suffix = "..." if len(value) > max_keys else "" + lines.append( + f"{path}: {type(value).__name__}(keys: [{', '.join(sub_keys)}{sub_suffix}])" + ) + inner = AgentProxy._inspect_structure(value, path, depth + 1) + if inner: + lines.append(inner) + else: + preview = AgentProxy._content_preview(value) + lines.append(f"{path}: {type(value).__name__} = {preview}") + + else: + lines.append(f"{type(obj).__name__}") + + return "\n".join(lines) + + def get_content_id(self, file_path: str, line_content: str) -> str: + """Resolve a content ID for use as start_line/end_line with EditFile. + + Supports three modes: + - **@L{number}**: e.g., `Agent.get_content_id("foo.py", "@L42")` + returns the content ID of line 42 (1-based). + - **content ID passthrough**: e.g., `Agent.get_content_id("foo.py", "~abcd~")` + verifies and returns an existing content ID string. + - **text match**: e.g., `Agent.get_content_id("foo.py", "def greet(")` + returns the content ID of the unique line containing that text. + """ + import os + import re + + from cecli.helpers.hashline import resolve_content_to_hashline_ids + from cecli.helpers.hashpos.hashpos import HASH_DELIMITER, HashPos + from cecli.tools.utils.helpers import resolve_paths + + abs_path, rel_path = resolve_paths(self._coder, file_path) + if not os.path.isfile(abs_path): + raise ValueError(f"File not found: {file_path}") + + content = self._coder.io.read_text(abs_path) + if content is None: + raise ValueError(f"Could not read file: {file_path}") + + lines = content.splitlines() + hp = HashPos(content) + + # @L{number} syntax + m = re.match(r"^@L(\d+)$", line_content.strip()) + if m: + line_num = int(m.group(1)) - 1 + if line_num < 0 or line_num >= len(lines): + raise ValueError(f"Line {m.group(1)} out of range (file has {len(lines)} lines)") + line_text = lines[line_num] + occurrence = 1 + sum(1 for i in range(line_num) if lines[i] == line_text) + return hp.get_wrapped_id(hp.generate_public_id(line_text, line_num, occurrence)) + + # Content ID passthrough: already looks like a content ID + if HashPos.FRAGMENT_RE.match(line_content): + from cecli.helpers.hashline import ContentHashError, normalize_hashline + + try: + normalized = normalize_hashline(line_content) + candidates = hp.resolve_to_lines(normalized) + if candidates: + return line_content + except (ContentHashError, ValueError): + pass + + # Fall back: value looks like a content ID (contains "~") but couldn't be + # resolved. Strip the prefix and try to match the remaining content. + stripped = HashPos.strip_prefix(line_content) + if stripped != line_content and stripped.strip(): + result, _ = resolve_content_to_hashline_ids(content, stripped, None) + if result != stripped and HASH_DELIMITER in str(result): + return result + + raise ValueError(f"Content ID '{line_content}' not found in {file_path}") + + # Text match via resolve_content_to_hashline_ids + result, _ = resolve_content_to_hashline_ids(content, line_content, None) + if result == line_content or HASH_DELIMITER not in str(result): + # Find all matching lines for a helpful error message + matching = [i + 1 for i, line in enumerate(lines) if line_content in line] + if len(matching) > 1: + line_nums = ", ".join(str(n) for n in matching[:10]) + suffix = f" ... ({len(matching)} total)" if len(matching) > 10 else "" + raise ValueError( + f"Pattern '{line_content}' matches {len(matching)} locations " + f"in {file_path} (lines {line_nums}{suffix}). " + f"Use ' @L' to disambiguate (e.g., '{line_content} @L{matching[0]}')." + ) + if matching: + raise ValueError( + f"Could not resolve content ID for '{line_content}' " + f"in {file_path} (line {matching[0]}). " + f"The match may not be unique enough." + ) + raise ValueError(f"Could not resolve content ID for '{line_content}' in {file_path}") + return result + + def resolve_regions( + self, + file_path: str, + regions: list[dict[str, str]], + ): + """ + Store named region patterns for lazy content-ID resolution. + + Content IDs are resolved *on access* via `.get_start(name)` / + `.get_end(name)`, so they are always fresh — even after + intervening edits shift hashline positions. + + Returns an :class:`AgentRegion` instance. + """ + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + return AgentRegion(file_path, self._coder, regions) + + async def edit_region( + self, + file_path: str, + edits: list[dict[str, object]], + change_id: str | None = None, + ): + """ + Thin wrapper around `EditFile` that accepts `{"start": content_id, "end": content_id}` region dicts. + + Use with `Agent.resolve_regions()` and `regions.get(name)`: + + regions = Agent.resolve_regions("foo.py", [ + {"name": "my_func", "start": "def my_func", "end": "return result"}, + ]) + await Agent.edit_region( + file_path="foo.py", + edits=[ + {"region": regions.get("my_func"), "text": "def my_func():\\n return 42"}, + ], + ) + """ + + edit_tool = self.get_tool("EditFile") + + edit_objects: list[dict[str, object]] = [] + for edit in edits: + region = edit["region"] + + edit_objects.append( + { + "file_path": file_path, + "operation": edit.get("operation", "replace"), + "start_line": region["start"], + "end_line": region["end"], + "text": edit["text"], + } + ) + + # self._validate_edit_regions(edits, edit_objects) + + return await edit_tool.call( + edits=edit_objects, + change_id=change_id, + ) + + async def sleep(self, seconds: float) -> None: + """Safe sleep - pauses execution (0-120 seconds max). + + Usage:: + + await Agent.sleep(1) # pause for 1 second + """ + from cecli.helpers.orchestration.safe_methods import _safe_sleep + + await _safe_sleep(seconds) + + def allowed_tools(self) -> list[str]: + """Return a sorted list of available tool names. + + Usage:: + + tools = Agent.allowed_tools() + """ + from cecli.helpers import nested + + tool_names = [] + tool_list = self._coder.get_tool_list() + for tool in tool_list: + name = nested.getter(tool, "function.name", "") + if name: + tool_names.append(name) + return sorted(tool_names) + + def allowed_methods(self) -> list[str]: + """Return a sorted list of all available functions and objects in the sandbox. + + Usage:: + + methods = Agent.allowed_methods() + """ + if self._env is None: + return [] + builtins = sorted(k for k in self._env._safe_builtins.keys() if not k.startswith("_")) + globals_list = sorted( + k + for k in self._env.globals.keys() + if not k.startswith("__") and k not in ("__builtins__", "NEWLINE") + ) + return builtins + globals_list + + @staticmethod + def _validate_edit_regions( + edits: list[dict[str, object]], + edit_objects: list[dict[str, object]], + ) -> None: + """Reject batches where any two edit regions overlap or are adjacent. + + Adjacent edits (within 8 lines of each other) corrupt content IDs + because the first edit regenerates IDs in the surrounding ~8-line + window, making the second edit's target IDs stale. + """ + _ADJACENCY_THRESHOLD = 8 # lines of ID-regeneration buffer + + if len(edits) <= 1: + return + + # Collect (start_line, end_line, idx) for each edit + indexed: list[tuple[int, int, int]] = [] + for i, (edit, eo) in enumerate(zip(edits, edit_objects)): + region = edit["region"] + sl = region.get("start_line") + el = region.get("end_line") + if sl is None or el is None: + # No line numbers available — skip validation (best-effort) + return + indexed.append((int(sl), int(el), i)) + + # Sort by start_line + indexed.sort(key=lambda x: x[0]) + + for j in range(len(indexed) - 1): + sl_a, el_a, idx_a = indexed[j] + sl_b, el_b, idx_b = indexed[j + 1] + + if sl_b <= el_a: + raise ValueError( + f"Overlapping edit regions detected: " + f"edit {idx_a + 1} (lines {sl_a}-{el_a}) overlaps with " + f"edit {idx_b + 1} (lines {sl_b}-{el_b}). " + f"Split edits into separate calls." + ) + if sl_b <= el_a + _ADJACENCY_THRESHOLD: + raise ValueError( + f"Adjacent edit regions detected: " + f"edit {idx_a + 1} (lines {sl_a}-{el_a}) is within " + f"{_ADJACENCY_THRESHOLD} lines of edit {idx_b + 1} (lines {sl_b}-{el_b}). " + f"Content IDs within ~8 lines of an edit are regenerated, " + f"so adjacent edits corrupt each other's targets. " + f"Split edits into separate calls." + ) + + +# --------------------------------------------------------------------------- +# Main execution environment +# --------------------------------------------------------------------------- + + +class _HelpfulBuiltins(dict): + """Custom __builtins__ dict that provides helpful hints for missing functions.""" + + _HINTS: dict[str, str] = { + "open": "Filesystem access is not available. Use the Command tool instead.", + "eval": "eval() is disabled for security.", + "exec": "exec() is disabled for security.", + "__import__": "Imports are disabled. Use only the primitives provided.", + "compile": "compile() is disabled for security.", + "breakpoint": "breakpoint() is disabled in the sandbox.", + "globals": "globals() is disabled. Use state or shared_state for persistence.", + "locals": "locals() is disabled. Use state or shared_state for persistence.", + "vars": "Use vars(obj) instead of vars() — vars(obj) returns non-dunder attrs of obj.", + "getattr": "getattr() is disabled. Access attributes directly.", + "setattr": "setattr() is disabled. Assign attributes directly.", + "delattr": "delattr() is disabled. Use del obj.attr instead.", + } + + def __missing__(self, key: str): + hint = self._HINTS.get(key) + if hint: + raise NameError(f"'{key}' is not available. {hint}") + raise NameError(f"name '{key}' is not defined") + + +# --------------------------------------------------------------------------- diff --git a/cecli/helpers/orchestration/environment.py b/cecli/helpers/orchestration/environment.py new file mode 100644 index 00000000000..ac8694a30ba --- /dev/null +++ b/cecli/helpers/orchestration/environment.py @@ -0,0 +1,695 @@ +""" +Sandboxed execution environment for LLM-generated orchestration code. + +Provides AgentExecutionEnv, ToolProxy, AgentProxy, and the context block builder. +""" + +from __future__ import annotations + +import ast +import asyncio +import collections +import datetime +import itertools +import json +import logging +import math +import re +import traceback +from typing import Any + +from cecli.helpers import nested, responses +from cecli.helpers.orchestration.agent_proxy import AgentProxy +from cecli.helpers.orchestration.safe_methods import ( + GatherResult, + _escape_newlines_in_strings, + _HelpfulBuiltins, + _safe_dir, + _safe_gather, + _safe_getattr, + _safe_hasattr, + _safe_sleep, + _safe_typeof, + _safe_vars, + _SafeJson, + _SafeModuleProxy, + _SafePathlib, + _SafeRe, + _SafeTraceback, + _strip_allowed_imports, +) +from cecli.helpers.orchestration.security import ( + LoopYieldInjector, + SecurityError, + SecurityFilter, + _cooperative_yield, + _security_raise, +) +from cecli.helpers.orchestration.tool_proxy import ToolProxy + +logger = logging.getLogger(__name__) + + +class TrackedDict(dict): + """A dict subclass that records mutations (set/delete/clear/pop) on a parent env. + + Tracks both adds (new keys) and modifications (existing key value changes) + so the environment can report only changed variables after execution. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._owner: AgentExecutionEnv | None = None + self._is_shared: bool = False + + def _set_owner(self, owner: AgentExecutionEnv, is_shared: bool = False) -> None: + self._owner = owner + self._is_shared = is_shared + + def __getattr__(self, key: str): + """Route attribute reads through dict lookup. + + Internal attrs (_owner, _is_shared) are accessed via object.__getattribute__ + to avoid recursion. All other names are looked up in the dict. + """ + if key.startswith("_"): + raise AttributeError(f"Cannot access private attribute {key!r} on TrackedDict") + try: + return self[key] + except KeyError: + raise AttributeError( + f"TrackedDict has no key {key!r}. " + f"Available keys: {', '.join(sorted(self.keys()))}" + ) from None + + def __setattr__(self, key: str, value) -> None: + """Route attribute writes through dict __setitem__. + + Internal attrs (_owner, _is_shared) are set directly on the instance + to avoid polluting dict state. + """ + if key.startswith("_"): + object.__setattr__(self, key, value) + else: + self[key] = value + + def __delattr__(self, key: str) -> None: + """Route attribute deletes through dict __delitem__.""" + if key.startswith("_"): + object.__delattr__(self, key) + else: + del self[key] + + def __setitem__(self, key, value): + is_new = key not in self + super().__setitem__(key, value) + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared, is_new=is_new) + + def __delitem__(self, key): + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared) + super().__delitem__(key) + + def clear(self): + if self._owner is not None: + for key in list(self): + self._owner._record_mutation(key, is_shared=self._is_shared) + super().clear() + else: + super().clear() + + def pop(self, key, *args): + if key in self: + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared) + return super().pop(key, *args) + if args: + return args[0] + raise KeyError(key) + + def popitem(self): + if not self: + raise KeyError("popitem(): dictionary is empty") + key = next(iter(self)) + if self._owner is not None: + self._owner._record_mutation(key, is_shared=self._is_shared) + return super().popitem() + + def get(self, key, default=None): + """Return value for *key*, falling through to ``shared_state`` if not found locally. + + When called on a local state dict (i.e., ``state``, not ``shared_state`` + itself), keys not present locally are looked up in the shared state + before returning *default*. This makes it easy to read globally-set + values without an explicit fallback:: + + val = state.get("my_global") # checks local, then shared_state + """ + if key in self: + return self[key] + # Don't fall through when this IS the shared_state (avoid infinite loop) + if self._is_shared: + return default + if self._owner is not None: + shared = type(self._owner)._shared_state + if key in shared: + return shared[key] + return default + + +# --------------------------------------------------------------------------- +# Safe primitives exposed to the sandbox + + +def _make_sandbox_dir(globals_dict, locals_dict): + """Create a safe dir() callable that handles both no-arg and obj modes. + + When called without arguments (like the builtin ``dir()``), returns a sorted + list of public (non-dunder) names from the sandbox globals, locals, and + builtins (including pre-imported modules like ``re``, ``math``, etc.). + When called with an object, delegates to ``_safe_dir(obj)``. + """ + _SENTINEL = object() + + def sandbox_dir(obj=_SENTINEL): + if obj is _SENTINEL: + names = set(globals_dict.keys()) | set(locals_dict.keys()) + builtins = globals_dict.get("__builtins__", {}) + if isinstance(builtins, dict): + names |= set(builtins.keys()) + return sorted(n for n in names if not n.startswith("_")) + + return _safe_dir(obj) + + return sandbox_dir + + +class AgentExecutionEnv: + """ + Sandboxed REPL environment for executing LLM-generated orchestration code. + + Provides: + - `Agent` : proxy to look up and call tools and other core agentic utility methods + - `gather` : safe parallel execution helper + - `state` : persistent dict (survives across Orchestrate calls) + - `print` : captured output + - `range`, `len`, `int`, `str`, `list`, `dict`, `bool`, `Exception` + + Security guarantees: + - AST security filtering before compilation + - Loop yield injection + - Timeout via asyncio.wait_for + - No imports, no private attributes, no dangerous builtins + """ + + # Shared across all AgentExecutionEnv instances — any agent can read/write + _shared_state: TrackedDict = TrackedDict() + + def __init__(self, coder: Any, orchestration_config: dict[str, Any] | None = None) -> None: + from typing import Any as _Any + + self._orchestration_config: dict[str, _Any] = orchestration_config or {} + self._DANGEROUS_BUILTINS: set[str] = { + "eval", + "exec", + "open", + "__import__", + "compile", + "breakpoint", + "globals", + "locals", + } + + self.state = TrackedDict() + self.state._set_owner(self) + self._shared_state._set_owner(self, is_shared=True) + + # Initialize early so _safe_builtins can reference them + self.globals: dict[str, _Any] = {} + self.locals: dict[str, _Any] = {} + + _disable_sec = self._orchestration_config.get("disable_security", False) + + self._safe_builtins: dict[str, _Any] = { + "print": print, + "range": range, + "len": len, + "int": int, + "str": str, + "float": float, + "list": list, + "dict": dict, + "bool": bool, + "tuple": tuple, + "set": set, + "typeof": _safe_typeof, + "type": _safe_typeof, + "vars": _safe_vars, + "dir": _make_sandbox_dir(self.globals, self.locals), + "isinstance": isinstance, + "hasattr": _safe_hasattr, + "getattr": _safe_getattr, + "repr": repr, + "enumerate": enumerate, + "zip": zip, + "sorted": sorted, + "reversed": reversed, + "min": min, + "max": max, + "sum": sum, + "abs": abs, + "round": round, + "any": any, + "all": all, + "filter": filter, + "map": map, + "chr": chr, + "next": next, + "Exception": Exception, + "ValueError": ValueError, + "TypeError": TypeError, + "KeyError": KeyError, + "IndexError": IndexError, + "AttributeError": AttributeError, + "RuntimeError": RuntimeError, + "NameError": NameError, + "ZeroDivisionError": ZeroDivisionError, + "StopIteration": StopIteration, + "ArithmeticError": ArithmeticError, + "LookupError": LookupError, + "re": _SafeRe, + "math": _SafeModuleProxy(math, disable_security=_disable_sec), + "itertools": _SafeModuleProxy(itertools, disable_security=_disable_sec), + "collections": _SafeModuleProxy(collections, disable_security=_disable_sec), + "datetime": _SafeModuleProxy(datetime, disable_security=_disable_sec), + "traceback": _SafeTraceback, + "pathlib": _SafePathlib, + } + + # Task 3: allowed_builtins — extend _safe_builtins with user-requested builtins + allowed_builtins: list[str] = self._orchestration_config.get("allowed_builtins", []) + for name in allowed_builtins: + if name.startswith("_"): + raise ValueError(f"Cannot allow private builtin '{name}'") + if name in self._DANGEROUS_BUILTINS and not self._orchestration_config.get( + "disable_security" + ): + raise ValueError(f"Cannot allow dangerous builtin '{name}'") + self._safe_builtins[name] = __builtins__[name] + + # When security is fully disabled, expose dangerous builtins + if self._orchestration_config.get("disable_security", False): + for name in self._DANGEROUS_BUILTINS: + self._safe_builtins[name] = __builtins__[name] + # Restore real getattr/hasattr when security is disabled + self._safe_builtins["getattr"] = getattr + self._safe_builtins["hasattr"] = hasattr + + # When allow_classes is enabled, expose __build_class__ needed by Python + if self._orchestration_config.get("allow_classes", False): + self._safe_builtins["__build_class__"] = __builtins__["__build_class__"] + + # When allowed_imports is configured, expose __import__ so imports work, + # and add __name__ since imported modules reference it. + if self._orchestration_config.get("allowed_imports"): + self._safe_builtins["__import__"] = __builtins__["__import__"] + self._safe_builtins["__name__"] = "__sandbox__" + + self.globals.clear() + _agent = AgentProxy(coder) + _agent._env = self + + self.globals.update( + { + "__builtins__": _HelpfulBuiltins(self._safe_builtins), + "Agent": _agent, + "gather": _safe_gather, + "json": _SafeJson, + "state": self.state, + "shared_state": AgentExecutionEnv._shared_state, + "__yield": _cooperative_yield, + "__security_raise": _security_raise, + "NEWLINE": "\n", + } + ) + self.locals.clear() + + def _make_reset(env_locals, env_state): + def reset_func(local_vars: bool = True, state: bool = False) -> None: + """Clear execution namespaces. + + Args: + local_vars: Clear the ephemeral locals namespace (default True). + state: Also clear per-agent state keys (default False). + Shared state is never affected. + """ + if local_vars: + env_locals.clear() + if state: + env_state.clear() + + return reset_func + + self.globals["reset"] = _make_reset(self.locals, self.state) + + @staticmethod + def _size_of(value: Any) -> int: + """Return a meaningful size metric for a state variable.""" + if value is None: + return 0 + if isinstance(value, (str, list, tuple, set, dict)): + return len(value) + return len(str(value)) + + @staticmethod + def _value_preview(value: Any) -> str: + """Return a short preview of a value for state snapshot display.""" + if value is None: + return "None" + s = str(value) + if len(s) > 80: + return s[:77] + "..." + return s + + def _record_mutation(self, key: object, is_shared: bool = False, is_new: bool = False) -> None: + """Record that a state key was modified during this execution. + + Called automatically by TrackedDict on set/delete/clear/pop. + """ + if not hasattr(self, "_modified_keys"): + self._modified_keys: set = set() + if not hasattr(self, "_modified_shared_keys"): + self._modified_shared_keys: set = set() + if is_shared: + self._modified_shared_keys.add(key) + else: + self._modified_keys.add(key) + + def _state_snapshot(self) -> list: + """Build a list of state variable descriptors. + + For local state, only modified keys are reported (with `modified: True`). + For shared state, all keys are reported so that cross-agent writes are + visible even when the current agent didn't modify them. Keys modified + by the current execution are marked `modified: True`; all others are + `modified: False`. + """ + modified_keys = getattr(self, "_modified_keys", set()) + modified_shared_keys = getattr(self, "_modified_shared_keys", set()) + entries = [] + + for key, value in self.state.items(): + if key not in modified_keys: + continue + entries.append( + { + "name": key, + "type": type(value).__name__, + "size": self._size_of(value), + "modified": True, + "scope": "local", + "preview": self._value_preview(value), + } + ) + + for key, value in self._shared_state.items(): + entries.append( + { + "name": key, + "type": type(value).__name__, + "size": self._size_of(value), + "modified": key in modified_shared_keys, + "scope": "shared", + "preview": self._value_preview(value), + } + ) + + return entries + + async def execute(self, code_str: str) -> dict: + + code_str = code_str.strip() + code_str = _escape_newlines_in_strings(code_str) + extra_allowed = frozenset(self._orchestration_config.get("allowed_imports", [])) + code_str, extra_globals = _strip_allowed_imports(code_str, extra_allowed=extra_allowed) + self.globals.update(extra_globals) + if not code_str: + return {"results": "", "state_variables": self._state_snapshot()} + + captured_output: list[str] = [] + + # Clear mutation tracking for this execution + self._modified_keys = set() + self._modified_shared_keys = set() + + def _capture_print(*args: Any, **kwargs: Any) -> None: + sep = kwargs.pop("sep", " ") + end = kwargs.pop("end", "\n") + kwargs.pop("file", None) # silently ignore + kwargs.pop("flush", None) # silently ignore + if kwargs: + raise TypeError(f"print() got unexpected keyword arguments: {list(kwargs.keys())}") + captured_output.append(sep.join(str(a) for a in args) + end) + + self.globals["__builtins__"]["print"] = _capture_print + + try: + tree = ast.parse(code_str, filename="", mode="exec") + except SyntaxError as e: + code = f"Syntax Error in orchestration code: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + + def _build_result(msg: str = "") -> dict: + print_output = "".join(captured_output) + parts = [] + if print_output: + parts.append(print_output.rstrip("\n")) + if msg: + parts.append(msg) + code = "\n".join(parts) if parts else "" + return {"results": code, "state_variables": self._state_snapshot()} + + try: + if not self._orchestration_config.get("disable_security", False): + tree = SecurityFilter( + allowed_imports=extra_allowed, + allow_classes=self._orchestration_config.get("allow_classes", False), + ).visit(tree) + if not self._orchestration_config.get("disable_loop_protection", False): + tree = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(tree) + except SecurityError as e: + return _build_result(f"Security Error: {e}") + except Exception as e: + return _build_result(f"AST Transform Error: {e}") + + wrapper_func = ast.AsyncFunctionDef( + name="__agent_async_runner", + args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), + body=tree.body, + decorator_list=[], + ) + + ast.fix_missing_locations(wrapper_func) + mod = ast.Module(body=[wrapper_func], type_ignores=[]) + + try: + compiled_code = compile(mod, filename="", mode="exec") + except Exception as e: + code = f"Compilation Error: {e}" + return {"results": code, "state_variables": self._state_snapshot()} + + try: + exec(compiled_code, self.globals, self.locals) + runner_coro = self.locals["__agent_async_runner"]() + result = await runner_coro + except asyncio.CancelledError: + return _build_result("Execution Error: Script was cancelled.") + except SecurityError as e: + return _build_result(f"Security Error: {e}") + except Exception as e: + tb = traceback.format_exc() + logger.warning("Orchestration execution error: %s\n%s", e, tb) + return _build_result(f"Execution Error: {type(e).__name__}: {e}") + finally: + self.locals.pop("__agent_async_runner", None) + self.globals["__builtins__"]["print"] = print + # Mutation tracking is handled automatically by TrackedDict + + return _build_result() + + +# flake8: noqa +# fmt: off + +def build_orchestration_context_block(agent_config: dict[str, Any]) -> str | None: + """ + Build the orchestration context block that explains calling conventions. + + Only returns content if `allow_orchestration` is enabled in agent_config. + """ + if not agent_config.get("allow_orchestration", True): + return None + + orchestration_config = agent_config.get("orchestration", {}) + + context = """ +The `Orchestrate` tool lets you batch multiple tool calls in a single step by writing Python code in a limited, secure sandbox. +This is much more efficient than making individual tool calls for loop-heavy workflows. +Variables and methods defined in a script are persisted in subsequent turns. +As such, results from previous calls can be reused and helper methods can be defined to enhance ease of use within the environment. + +### Primitives + +| Primitive | Description | +|-----------|-------------| +| `Agent.allowed_methods()` | List all available builtin function names | +| `Agent.allowed_tools()` | List all available tool names | + +| `Agent.get_tool(name)` | Get a tool proxy (case-insensitive, accepts `Local--` or `{{Server}}--` prefix) | +| `await tool.call(**params)` | Execute a tool; returns `{"result": [...], "errors": [...], "details": [...]}` — each result item is `{"content": ..., "_": {...}}` | + +| `Agent.peek(result)` | Inspect a tool result's structure and leaf content — returns a string; use `print(Agent.peek(result))` to see it | +| `Agent.get_value(result, path, default?)` | Safely access nested values in tool results using dot-notation (e.g. `"result.0.content"`) | + +| `Agent.get_content_id(path, text)` | Resolve a content ID from `@L{num}` or line text for EditFile | +| `Agent.resolve_regions(path, regions)` | Batch-resolve text patterns to content IDs. Use `start_line_hint` / `end_line_hint` in region spec entries for disambiguation (supports `@L`, `@A{{regex}}`, `@B{{regex}}` — same hint syntax as ReadFile). Ambiguous patterns raise immediately. The returned `AgentRegion` has `.get_start(name)`, `.get_end(name)`, `.names()`, `.get(name)` | +| `Agent.edit_region(path, edits)` | Thin wrapper around EditFile that accepts pre-resolved region dicts `{"start": content_id, "end": content_id}`. Use with `Agent.resolve_regions()` and `regions.get(name)` | + +| `await Agent.sleep(seconds)` | Pause execution (0-120s max) | + +| `gather(**named_tasks)` | Run tasks concurrently; returns an iterable with `.key` / `["key"]` access | +| `state` / `shared_state` | `state` persists across *all* `Orchestrate` calls within the same agent session (not just one call). `state.get(key)` falls through to ``shared_state`` when the key is not local. `shared_state` persists across *all* agent sessions globally | +| `print(...)` / `reset(local_vars=True, state=False)` | Output messages; clear local namespace (and optionally state) | +| `typeof(x)` / `isinstance(x, t)` / `hasattr(x, n)` / `repr(x)` / `vars(obj)` | Type inspection and debugging | + +### Available Modules + +Pre-imported, read-only standard library modules: + +| Module | Common uses | +|--------|-------------| +| `re` | Regular expressions: `re.search(r"pat", s)`, `re.findall(...)` | +| `math` | Math functions: `math.ceil(n)`, `math.sqrt(n)` | +| `itertools` | Combinatorics: `itertools.chain(a, b)`, `itertools.product(...)` | +| `collections` | Container helpers: `collections.Counter(...)`, `collections.defaultdict(...)` | +| `datetime` | Date/time: `datetime.datetime.now()`, `datetime.timedelta(...)` | +| `pathlib` | Safe filesystem paths: `pathlib.Path("/tmp/foo")`, `.parent`, `.name`, `/` joining. I/O methods (``read_text``, ``write_text``, etc.) blocked | +| `json` | Parse / serialize: `json.loads(s)`, `json.dumps(obj, indent=2)` | +| `traceback` | Traceback formatting: `traceback.format_exc()`, `traceback.format_tb(...)` | + +All other module imports will fail. + +### Usage + +```python +tool = Agent.get_tool("delegate") +tool_outputs = await gather( + a=tool.call(prompt="A"), + b=tool.call(prompt="B"), +) +print(tool_outputs.a) # attribute access +print(tool_outputs["b"]) # key access +``` + +### Working with Results + +Use `Agent.peek()` to discover a result's structure, then `Agent.get_value()` +to extract specific fields by dot-path:: + + result = await read_tool.call(file_path="foo.py", range_start="@000", range_end="000@") + print(Agent.peek(result)) + # Shows: result[0].content: str = '...' + # result[0]._.file_path: str = 'foo.py' + + content = Agent.get_value(result, "result.0.content") + file_path = Agent.get_value(result, "result.0._.file_path", "unknown") + +Result list items are plain dicts — use item['content'] / item.get('content') +and item['_'] / item.get('_') to access data and metadata respectively. + +### Creating New Files + +Use `ResourceManager` to create an empty file, then `EditFile` to write +initial content (use `@000` for start/end on empty files):: + + rm = Agent.get_tool("ResourceManager") + edit = Agent.get_tool("EditFile") + + await rm.call(create=["path/to/new_file.py"]) + await edit.call(edits=[{ + "file_path": "path/to/new_file.py", + "operation": "replace", + "start_line": "@000", + "end_line": "@000", + "text": "def greet(name):\n return f\"Hello, {name}!\"\n", + }]) + +After creation, use `resolve_regions` and `edit_region` for targeted edits. + +### Editing with Regions + +Use `Agent.resolve_regions()` to convert text patterns into content IDs, then `Agent.edit_region()` to apply edits using the resolved IDs. + +#### Step 1 — resolve region boundaries once + +```python +regions = Agent.resolve_regions("foo.py", [ + {"name": "my_func", "start": "def my_func", "end": "return result"}, + {"name": "init", "start": "def __init__", "end": "self.x = x"}, +]) +``` + +#### Step 2 — Use `regions.get(name)` with `Agent.edit_region()` (recommended shorthand) + +```python +await Agent.edit_region( + file_path="foo.py", + edits=[ + {"region": regions.get("my_func"), "text": "def my_func():\n return 42"}, + ], +) +``` + +#### Alternative: Call `EditFile` directly with `regions.get_start()` / `regions.get_end()` + +### Gotchas + +- **Types**: compare with `typeof(x) == dict` or `isinstance(x, dict)` — NOT `typeof(x) == "dict"` +- **Args**: use keyword args only — `tool.call(file_path="f", ...)` +- **gather**: always use named `gather(x=a, y=b)` — positional args are not supported + +### Rules + +1. No imports - use only the primitives and modules above +2. Do not access attributes starting with `_` (private/dunder) +3. All tool calls must be awaited +4. Use `print(...)` to output results — only printed output is returned +""" + + # Task 6: Append sandbox configuration overrides when non-empty + if orchestration_config: + overrides: list[str] = [] + if orchestration_config.get("allowed_imports"): + overrides.append( + f"- Allowed extra imports: {orchestration_config['allowed_imports']}" + ) + if orchestration_config.get("allowed_builtins"): + overrides.append( + f"- Allowed extra builtins: {orchestration_config['allowed_builtins']}" + ) + if orchestration_config.get("allow_classes"): + overrides.append("- Class definitions are allowed (__init__, __str__, etc.)") + if orchestration_config.get("disable_security"): + overrides.append("- ⚠ Security filtering is DISABLED") + if orchestration_config.get("disable_loop_protection"): + overrides.append("- ⚠ Loop protection is DISABLED") + if overrides: + context += "\n### Sandbox Configuration Overrides\n" + "\n".join(overrides) + "\n" + + return context + + +# flake8: noqa +# fmt: on diff --git a/cecli/helpers/orchestration/region_resolver.py b/cecli/helpers/orchestration/region_resolver.py new file mode 100644 index 00000000000..f19fcbe543b --- /dev/null +++ b/cecli/helpers/orchestration/region_resolver.py @@ -0,0 +1,473 @@ +""" +Region resolution helpers for the orchestration sandbox. + +Provides AgentRegion for storing named region boundary *patterns* that are +resolved to content IDs on-demand at access time. This ensures content IDs +are always fresh — important after intervening edits shift hashline positions. +""" + +from __future__ import annotations + +from typing import Any + +from cecli.helpers.hashpos.transformations import ( + extract_hint, + is_content_id, + narrow_by_proximity, + resolve_at_l, + search_in_lines, +) + + +class AgentRegion: + """ + Stores named region boundary *patterns* resolved to content IDs on access. + + Content IDs can shift after edits, so ``get_start(name)`` and + ``get_end(name)`` re-read the file and re-resolve patterns at each call. + Use these directly in ``EditFile`` calls for always-fresh IDs. + + When a region spec uses a **content ID** (e.g. `"~abcd~"`) instead of + text, the referenced line content is snapshotted on first resolution. + If the ID goes stale after intervening edits, subsequent resolutions + fall back to content matching against the snapshotted line text. + + Example usage in orchestration code: + + regions = Agent.resolve_regions("foo.py", [ + {"name": "helper", "start": "def helper", "end": "return x"}, + ]) + edit_tool = Agent.get_tool("EditFile") + await edit_tool.call(edits=[{ + "file_path": "foo.py", + "operation": "replace", + "start_line": regions.get_start("helper"), + "end_line": regions.get_end("helper"), + "text": "def helper():\\n return 42", + }]) + """ + + def __init__( + self, + file_path: str, + coder: Any, + region_specs: list[dict[str, str]], + ) -> None: + self._file_path = file_path + self._coder = coder + self._specs: dict[str, dict[str, str]] = {} + + for spec in region_specs: + name = spec["name"] + entry: dict[str, object] = { + "start": spec["start"], + "end": spec["end"], + } + if "start_line_hint" in spec: + entry["start_line_hint"] = spec["start_line_hint"] + if "end_line_hint" in spec: + entry["end_line_hint"] = spec["end_line_hint"] + self._specs[name] = entry + + # Eagerly validate uniqueness for all regions at creation time. + # This catches ambiguous patterns immediately with clear error messages. + self._eager_validate() + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def get_start(self, name: str) -> str: + """Re-read file and return the current content ID for the start of *name*.""" + + return self._resolve(name)[0] + + def get_end(self, name: str) -> str: + """Re-read file and return the current content ID for the end of *name*.""" + + return self._resolve(name)[1] + + def get_start_line(self, name: str) -> int: + """Re-read file and return the current 1-based start line for *name*.""" + + return self._resolve(name)[2] + + def get_end_line(self, name: str) -> int: + """Re-read file and return the current 1-based end line for *name*.""" + + return self._resolve(name)[3] + + def __contains__(self, name: str) -> bool: + return name in self._specs + + def __len__(self) -> int: + return len(self._specs) + + def names(self) -> list[str]: + """Return the list of region names.""" + + return sorted(self._specs.keys()) + + def get(self, name: str) -> dict[str, object]: + """Return ``{"start": ..., "end": ..., "start_line": N, "end_line": N}`` for *name*. + + The returned dict can be passed directly as the ``region`` value + in ``Agent.edit_region()`` edits. ``start_line`` / ``end_line`` are + 1-based for readability and enable adjacent-edit detection. + """ + + start_id, end_id, start_line, end_line = self._resolve(name) + return { + "start": start_id, + "end": end_id, + "start_line": start_line + 1, + "end_line": end_line + 1, + } + + def __repr__(self) -> str: + names = ", ".join(sorted(self._specs.keys())) + return f"AgentRegion({len(self._specs)} regions on " f"{self._file_path!r}: {names})" + + # ------------------------------------------------------------------ + # Resolution internals + # ------------------------------------------------------------------ + + def _eager_validate(self) -> None: + """Eagerly resolve and validate all region patterns at init time. + + Raises ValueError immediately for ambiguous patterns so the LLM + gets clear feedback without waiting for the first access. + """ + + for name in list(self._specs.keys()): + self._resolve(name) + + def _resolve(self, name: str) -> tuple[str, str, int, int]: + """ + Re-read file and resolve *name* to + (start_id, end_id, start_line, end_line). + + When a pattern is a content ID the referenced line is snapshotted + so future resolutions can fall back to content matching if the + original ID goes stale. + """ + + import os + + from cecli.helpers.hashline import ( + ContentHashError, + normalize_hashline, + resolve_content_to_hashline_ids, + ) + from cecli.helpers.hashpos.hashpos import HashPos + from cecli.tools.utils.helpers import resolve_paths + + spec = self._specs[name] + + # Read explicit line hints from spec (preferred over @L in patterns). + # 1-based in the spec, converted to 0-based internally. + explicit_start = spec.get("start_line_hint") + explicit_end = spec.get("end_line_hint") + + abs_path, rel_path = resolve_paths(self._coder, self._file_path) + + if not os.path.isfile(abs_path): + raise ValueError(f"File not found: {self._file_path}") + + content = self._coder.io.read_text(abs_path) + + if content is None: + raise ValueError(f"Could not read file: {self._file_path}") + + lines = content.splitlines() + hp = HashPos(content) + + start_pattern = self._resolve_pattern( + hp, lines, spec, "start", normalize_hashline, ContentHashError + ) + end_pattern = self._resolve_pattern( + hp, lines, spec, "end", normalize_hashline, ContentHashError + ) + + # Always strip @L hints from patterns — they are metadata, not literal text. + # Explicit hints (start_line_hint / end_line_hint) override any @L in patterns. + start_pattern, extracted_start, start_hint_type = self._extract_l_hint(start_pattern, lines) + end_pattern, extracted_end, end_hint_type = self._extract_l_hint(end_pattern, lines) + + # Handle explicit hints (start_line_hint / end_line_hint) + # Integers are treated as @L (1-based line numbers). + # Strings support the full @L, @A, @B syntax (same as ReadFile). + if explicit_start is not None: + if isinstance(explicit_start, str): + # String hint — parse through _extract_l_hint + _, start_hint, start_hint_type = self._extract_l_hint(explicit_start, lines) + if start_hint is None: + raise ValueError( + f"start_line_hint '{explicit_start}' for region " + f"'{name}' could not be resolved" + ) + else: + # Integer hint — treat as @L (1-based, converted to 0-based) + start_hint = explicit_start - 1 + start_hint_type = "L" + else: + start_hint = extracted_start + + if explicit_end is not None: + if isinstance(explicit_end, str): + _, end_hint, end_hint_type = self._extract_l_hint(explicit_end, lines) + if end_hint is None: + raise ValueError( + f"end_line_hint '{explicit_end}' for region " + f"'{name}' could not be resolved" + ) + else: + end_hint = explicit_end - 1 + end_hint_type = "L" + else: + end_hint = extracted_end + + # Strip hashline prefixes from text patterns — the LLM may have copied + # content-ID-prefixed lines from a ReadFile response (e.g. ~XYZ12::text). + # Content-ID patterns and special markers are left untouched. + if not self._looks_like_content_id(start_pattern) and start_pattern not in ("@000", "000@"): + start_pattern = HashPos.strip_prefix(start_pattern) + if not self._looks_like_content_id(end_pattern) and end_pattern not in ("@000", "000@"): + end_pattern = HashPos.strip_prefix(end_pattern) + + # Validate uniqueness for text-based patterns (not content IDs or special markers). + self._validate_pattern_uniqueness( + start_pattern, start_hint, "start", name, lines, start_hint_type + ) + self._validate_pattern_uniqueness(end_pattern, end_hint, "end", name, lines, end_hint_type) + + start_id, end_id = resolve_content_to_hashline_ids( + content, + start_pattern, + end_pattern, + start_hint_line=start_hint if start_hint_type == "L" else None, + ) + + # Resolve line numbers from content IDs + def _line_from_id(content_id: str, default_if_not_found: int) -> int: + if content_id == "@000": + return 1 + + if content_id == "000@": + return len(lines) + + try: + normalized = normalize_hashline(content_id) + candidates = hp.resolve_to_lines(normalized) + + if candidates: + return candidates[0] + 1 + except (ContentHashError, ValueError): + pass + + return default_if_not_found + + start_line = _line_from_id(start_id, -1) + end_line = _line_from_id(end_id, -1) + + if start_line < 0 or end_line < 0: + parts = [] + if start_line >= 0: + parts.append(f" Start pattern resolved to: {start_pattern!r} (line {start_line})") + else: + parts.append(f" Start pattern NOT FOUND: {start_pattern!r}") + if end_line >= 0: + parts.append(f" End pattern resolved to: {end_pattern!r} (line {end_line})") + else: + parts.append(f" End pattern NOT FOUND: {end_pattern!r}") + raise ValueError( + f"Could not resolve line numbers for region " + f"'{name}' in {self._file_path}\n" + "\n".join(parts) + ) + + return start_id, end_id, start_line, end_line + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + @staticmethod + def _search_in_lines(lines: list[str], pattern: str) -> list[int]: + """Return 0-based indices of all lines where *pattern* matches. + + Supports multiline patterns (each line of *pattern* must appear + as a whole word in the corresponding line of *lines*). + + Delegates to the core ``search_in_lines`` in + ``cecli.helpers.hashpos.transformations`` for unified matching. + """ + + return search_in_lines(lines, pattern) + + @staticmethod + def _extract_l_hint( + pattern: str, lines: list[str] | None = None + ) -> tuple[str, int | None, str | None]: + """Extract a hint suffix from a pattern string. + + Supports @L (direct line number), @A{{regex}} (filter to matches AFTER + the first regex match), and @B{{regex}} (filter to matches BEFORE the last + regex match) hints. + + Returns (stripped_pattern, hint_value, hint_type) where: + - hint_value is a 0-based line number or None + - hint_type is 'L', 'A', 'B', or None + """ + + return extract_hint(pattern, lines) + + @staticmethod + def _narrow_by_proximity(indices: list[int], target: int, max_results: int = 5) -> list[int]: + """Return up to *max_results* indices closest to *target* (0-based).""" + + return narrow_by_proximity(indices, target, max_results=max_results) + + def _validate_pattern_uniqueness( + self, + pattern: str, + hint: int | None, + boundary: str, + name: str, + lines: list[str], + hint_type: str | None = None, + ) -> None: + """Raise ValueError if *pattern* matches multiple locations. + + Only validates text patterns — content IDs and special markers + (@000 / 000@) are inherently unique and are skipped. + """ + + if self._looks_like_content_id(pattern): + return + if pattern in ("@000", "000@"): + return + + matches = self._search_in_lines(lines, pattern) + + # Apply @A/@B directional filtering — keep only closest match in direction + if hint_type == "A" and hint is not None: + after = [m for m in matches if m > hint] + matches = [min(after)] if after else [] + elif hint_type == "B" and hint is not None: + before = [m for m in matches if m < hint] + matches = [max(before)] if before else [] + + if len(matches) <= 1: + return + + # Try proximity narrowing — find the unique closest match (only for @L hints) + if hint is not None and hint_type == "L": + best = min(matches, key=lambda i: abs(i - hint)) + best_dist = abs(best - hint) + conflicts = [i for i in matches if i != best and abs(i - hint) == best_dist] + if not conflicts: + return # Unique closest match found + matches_for_display = [best] + conflicts + else: + matches_for_display = matches + + # Build detailed match display with line content + max_preview = 10 + sorted_matches = sorted(matches_for_display) + all_line_nums = [str(i + 1) for i in sorted_matches] + line_nums_display = ", ".join(all_line_nums) + if len(all_line_nums) > 50: + line_nums_display = ( + ", ".join(all_line_nums[:50]) + f", ... ({len(all_line_nums)} total)" + ) + + # Only show content previews for the first 10 matches + match_lines = [] + for i in sorted_matches[:max_preview]: + line_content = lines[i] + if len(line_content) > 120: + line_content = line_content[:117] + "..." + match_lines.append(f" @L{i + 1}: {line_content}") + match_details = "\n".join(match_lines) + if len(sorted_matches) > max_preview: + match_details += f"\n ... and {len(sorted_matches) - max_preview} more match(es)" + + if hint is not None: + raise ValueError( + f"{boundary.capitalize()} pattern '{pattern}' for region " + f"'{name}' has {len(matches)} matches (lines {line_nums_display}); " + f"@L{hint + 1} hint ties between " + f"{len(matches_for_display)} equally-close locations:\n" + f"{match_details}\n" + f"Use a more specific pattern." + ) + + raise ValueError( + f"{boundary.capitalize()} pattern '{pattern}' for region " + f"'{name}' matches {len(matches)} locations (lines {line_nums_display}):\n" + f"{match_details}\n" + f"Use a more specific pattern or append ' @L' to " + f"disambiguate (e.g., '{pattern} @L{all_line_nums[0]}')." + ) + + @staticmethod + def _looks_like_content_id(value: str) -> bool: + """Return True if *value* appears to be a content ID rather than text.""" + + return is_content_id(value) + + def _resolve_pattern( + self, + hp, + lines: list[str], + spec: dict[str, str], + key: str, + normalize_hashline, + ContentHashError, + ) -> str: + """ + Resolve a single boundary pattern. + + Content-ID patterns have their referenced line content snapshotted + so stale IDs can be recovered via content matching. + """ + + pattern = spec[key] + + # Special markers never go stale + if pattern in ("@000", "000@"): + return pattern + + # @L{num} notation — resolve via shared utility + import re as _re + + _m = _re.match(r"^@L(\d+)$", pattern) + if _m: + return resolve_at_l(pattern, hp, lines) + + if not self._looks_like_content_id(pattern): + return pattern + + # Content ID — try to resolve and snapshot the line content + content_key = f"_{key}_content" + + try: + normalized = normalize_hashline(pattern) + candidates = hp.resolve_to_lines(normalized) + + if candidates and candidates[0] < len(lines): + spec[content_key] = lines[candidates[0]] + + return pattern + except (ContentHashError, ValueError): + pass + + # Content ID may be stale — fall back to snapshotted line content + cached = spec.get(content_key) + + if cached: + return cached + + # No fallback available; return the (stale) ID and let the + # caller's resolution logic handle the error + return pattern diff --git a/cecli/helpers/orchestration/safe_methods.py b/cecli/helpers/orchestration/safe_methods.py new file mode 100644 index 00000000000..c7d5ea9da8c --- /dev/null +++ b/cecli/helpers/orchestration/safe_methods.py @@ -0,0 +1,1064 @@ +""" +Safe primitives and shim classes for the orchestration sandbox. + +Provides helper functions, container classes, and module proxies +that are injected into the sandbox globals. +""" + +from __future__ import annotations + +import asyncio +import json +import pathlib +import re as _re_mod +import traceback as _tb_mod +from typing import Any + +# --------------------------------------------------------------------------- +# Safe Path wrapper +# --------------------------------------------------------------------------- + + +class _SafePath: + """Wrapper around pathlib.Path that blocks dangerous I/O methods. + + The following methods are blocked and raise SecurityError: + - ``read_text()``, ``read_bytes()`` — file reads + - ``write_text()``, ``write_bytes()`` — file writes + - ``open()`` — file handle access + + All other Path properties and methods (``.parent``, ``/`` joining, + ``.exists()``, ``.is_dir()``, ``.glob()``, etc.) work normally. + Path-like return values are automatically wrapped to maintain safety. + """ + + _BLOCKED_METHODS: frozenset[str] = frozenset( + {"read_text", "read_bytes", "write_text", "write_bytes", "open"} + ) + + def __init__(self, *args, **kwargs): + import pathlib as _pathlib + + object.__setattr__(self, "_path", _pathlib.Path(*args, **kwargs)) + + def __getattr__(self, name: str): + if name.startswith("_"): + raise AttributeError(f"Cannot access private attribute {name!r} on Path wrapper") + if name in self._BLOCKED_METHODS: + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + f"Cannot call '{name}' on Path objects in the sandbox. " + f"Filesystem I/O is restricted to the Command tool." + ) + import pathlib as _pathlib + + attr = getattr(self._path, name) + if callable(attr): + + def _wrapped(*a, **kw): + result = attr(*a, **kw) + if isinstance(result, _pathlib.PurePath): + return _SafePath._from_path(result) + return result + + return _wrapped + if isinstance(attr, _pathlib.PurePath): + return _SafePath._from_path(attr) + return attr + + def __truediv__(self, other): + return _SafePath._from_path(self._path / other) + + def __rtruediv__(self, other): + import pathlib as _pathlib + + return _SafePath._from_path(_pathlib.Path(other) / self._path) + + def __repr__(self): + return repr(self._path) + + def __str__(self): + return str(self._path) + + def __fspath__(self): + import os + + return os.fspath(self._path) + + def __eq__(self, other): + if isinstance(other, _SafePath): + return self._path == other._path + return self._path == other + + def __hash__(self): + return hash(self._path) + + @staticmethod + def _from_path(p): + sp = object.__new__(_SafePath) + object.__setattr__(sp, "_path", p) + return sp + + +class _SafePathlib: + """Drop-in ``pathlib`` module proxy with Path I/O methods blocked. + + Usage in sandbox code:: + + p = pathlib.Path("/tmp/foo") + parent = p.parent # works, returns wrapped Path + name = p.name # works + content = p.read_text() # raises SecurityError + + ``pathlib.Path()`` returns ``_SafePath`` wrappers. Pure path classes + (``PurePath``, ``PurePosixPath``, ``PureWindowsPath``) are safe to pass + through directly — they have no I/O methods. Concrete ``PosixPath`` and + ``WindowsPath`` are intentionally excluded; use ``Path()`` instead. + """ + + PurePath = pathlib.PurePath + PurePosixPath = pathlib.PurePosixPath + PureWindowsPath = pathlib.PureWindowsPath + + @staticmethod + def Path(*args, **kwargs): + return _SafePath(*args, **kwargs) + + +# Safe primitives exposed to the sandbox +# --------------------------------------------------------------------------- + + +async def _safe_sleep(seconds: float) -> None: + """Safe sleep wrapper for the orchestration environment.""" + if seconds < 0: + raise ValueError("sleep() requires a non-negative value") + if seconds > 120: + raise ValueError("sleep() is limited to 120 seconds maximum") + await asyncio.sleep(seconds) + + +class GatherResult: + """Result container for named ``gather()`` calls. + + Supports both attribute access (``results.my_task``) and + key access (``results["my_task"]``), plus ``len()`` and + iteration for unpacking. + """ + + def __init__(self, results: dict[str, Any]) -> None: + object.__setattr__(self, "_results", results) + + def __getattr__(self, name: str) -> Any: + if name.startswith("_"): + raise AttributeError( + "Cannot access private attribute " + repr(name) + " on GatherResult" + ) + results = object.__getattribute__(self, "_results") + if name not in results: + raise AttributeError( + "GatherResult has no key " + + repr(name) + + ". Available keys: " + + ", ".join(sorted(results.keys())) + ) + return results[name] + + def __getitem__(self, key: str) -> Any: + results = object.__getattribute__(self, "_results") + if not isinstance(key, str): + raise TypeError( + f"GatherResult keys must be strings, got {type(key).__name__}. " + f"Use attribute access: results.my_key, " + f"or string keys: results['my_key']. " + f"Available keys: {', '.join(sorted(results.keys()))}" + ) + if key not in results: + raise KeyError( + f"GatherResult has no key {key!r}. " + f"Available keys: {', '.join(sorted(results.keys()))}" + ) + return results[key] + + def __setattr__(self, name: str, value: Any) -> None: + if not name.startswith("_"): + raise AttributeError("GatherResult is read-only: cannot set " + repr(name)) + object.__setattr__(self, name, value) + + def __len__(self) -> int: + results = object.__getattribute__(self, "_results") + return len(results) + + def __iter__(self): + results = object.__getattribute__(self, "_results") + return iter(results.items()) + + def keys(self) -> Any: + results = object.__getattribute__(self, "_results") + return results.keys() + + def values(self) -> Any: + results = object.__getattribute__(self, "_results") + return results.values() + + def items(self) -> Any: + results = object.__getattribute__(self, "_results") + return results.items() + + def __dir__(self): + results = object.__getattribute__(self, "_results") + return sorted(set(super().__dir__()) | set(results.keys())) + + def __contains__(self, key: str) -> bool: + results = object.__getattribute__(self, "_results") + return key in results + + def __repr__(self) -> str: + results = object.__getattribute__(self, "_results") + inner = ", ".join(f"{k}={type(v).__name__}" for k, v in results.items()) + return f"GatherResult({inner})" + + +async def _safe_gather(*args: Any, **named_awaitables: Any): + """ + Safely execute multiple awaitables concurrently. + + All awaitables must be passed as keyword arguments. Results are returned + as a ``GatherResult`` with attribute and key access: + + results = await gather(read_a=task_a, grep_b=task_b) + print(results.read_a) # attribute access + print(results["grep_b"]) # key access + len(results) # number of results + + Forces ``return_exceptions=True`` so that failures in one task + do not crash the entire batch. Exceptions are converted to + structured error dicts. + """ + if args: + raise TypeError( + "gather() requires keyword arguments. " + "Use named arguments like gather(a=task1, b=task2), " + "not positional arguments like gather(task1, task2)." + ) + if not named_awaitables: + return GatherResult({}) + + keys = list(named_awaitables.keys()) + coros = list(named_awaitables.values()) + results = await asyncio.gather(*coros, return_exceptions=True) + processed = {} + for key, r in zip(keys, results): + if isinstance(r, BaseException): + processed[key] = { + "result": [], + "errors": [f"{type(r).__name__}: {r}"], + "details": [], + } + else: + processed[key] = r + return GatherResult(processed) + + +def _safe_typeof(obj: Any) -> type: + """Safe type inspection — only accepts 1 argument, cannot create classes. + + Unlike ``type(name, bases, dict)`` which can create new classes at runtime, + ``_safe_typeof(obj)`` only returns the type of an object for inspection. + + Additionally, when *obj* is itself a type/class whose metaclass is ``type``, + the result would be ```` — the real ``type`` metaclass. + This is blocked because the metaclass can be used to dynamically create + arbitrary classes, bypassing the AST-level ``ClassDef`` filter. + """ + result = type(obj) + if result is type: + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + "Access to the 'type' metaclass is forbidden in the sandbox. " + "Use isinstance() for type checks instead of comparing type objects." + ) + return result + + +def _safe_vars(obj: Any) -> dict: + """Safe vars() - only accepts 1 argument, returns non-dunder attributes. + + Unlike ``vars()`` with no arguments (which returns the local namespace), + ``vars(obj)`` returns the non-dunder instance attributes of a single object. + For objects without ``__dict__`` (e.g. basic types, ``__slots__``-only objects), + returns an empty dict. + + Private and dunder attributes are silently excluded from the result. + Objects with problematic ``__dict__`` or ``__getattr__`` implementations + are handled gracefully (empty dict returned) rather than crashing. + """ + try: + if hasattr(obj, "__dict__"): + d = obj.__dict__ + if isinstance(d, dict): + return {k: v for k, v in d.items() if not k.startswith("_")} + except Exception: + pass + + # Handle __slots__-only objects + try: + for cls in type(obj).__mro__: + slots = getattr(cls, "__slots__", ()) + if slots: + result = {} + for slot in slots: + if not slot.startswith("_") and hasattr(obj, slot): + try: + result[slot] = getattr(obj, slot) + except Exception: + pass + if result: + return result + except Exception: + pass + + return {} + + +def _safe_dir(obj: Any) -> list: + """Safe dir() - returns public attributes only, without builtins or dunders. + + Filters out names starting with ``_`` and common builtin attributes + inherited from ``object`` (e.g., ``__class__``, ``__dict__``, etc.). + + Unlike the builtin ``dir()`` which includes everything, this returns + only the user-facing attributes and methods of the object. + """ + all_attrs = dir(obj) + + return [ + a + for a in all_attrs + if not a.startswith("_") + and a + not in ( + "__builtins__", + "__cached__", + "__doc__", + "__file__", + "__loader__", + "__name__", + "__package__", + "__spec__", + ) + ] + + +def _safe_getattr(obj: Any, name: str, *args: Any) -> Any: + """Safe getattr() - blocks access to private/dunder attributes. + + Raises ``AttributeError`` (matching ``getattr``'s default behavior) when + the attribute name starts with ``_``, preventing sandboxed code from + bypassing the AST-level private-attribute filter. + + When ``disable_security`` is enabled in orchestration config, the real + ``getattr`` is restored so this restriction doesn't apply. + """ + if type(name) is not str: + raise TypeError(f"getattr() attribute name must be str, not {type(name).__name__}") + if name.startswith("_") and name != "_": + raise AttributeError(f"Cannot access private attribute {name!r} in sandbox") + return getattr(obj, name, *args) + + +def _safe_hasattr(obj: Any, name: str) -> bool: + """Safe hasattr() - blocks probing of private/dunder attributes. + + Returns ``False`` for any attribute name starting with ``_`` so that + sandboxed code cannot detect the presence of private attributes. + + When ``disable_security`` is enabled in orchestration config, the real + ``hasattr`` is restored so this restriction doesn't apply. + """ + if type(name) is not str: + raise TypeError(f"hasattr() attribute name must be str, not {type(name).__name__}") + if name.startswith("_") and name != "_": + return False + return hasattr(obj, name) + + +def _find_closing_quote(code: str, quote: str, start: int) -> int: + """Find the position of the closing *quote* in *code*, skipping + backslash-escaped characters so that ``\"`` (escaped quote) is not + mistaken for the string terminator. + + Returns -1 if no unescaped closing quote is found. + """ + + i = start + while i < len(code): + if code[i] == chr(92): # backslash — skip the next char + i += 2 + continue + if code[i : i + len(quote)] == quote: + return i + i += 1 + + return -1 + + +def _naive_escape_newlines(code: str, NL: str, BSN: str) -> str: + """Replace literal newlines with ``\\n`` inside all string literals. + + Simple quote-matching — does not understand f-string expression boundaries. + Callers should validate the result with ``compile()`` and fall back to the + f-string-aware variant when this produces invalid code.""" + + result = [] + idx = 0 + while idx < len(code): + ch = code[idx] + if ch == chr(34) or ch == chr(39): + quote = ch + if idx + 2 < len(code) and code[idx : idx + 3] == quote * 3: + quote = quote * 3 + result.append(quote) + idx += len(quote) + end = _find_closing_quote(code, quote, idx) + if end == -1: + result.append(code[idx:]) + idx = len(code) + break + inner = code[idx:end] + inner = json.dumps(inner, ensure_ascii=False)[1:-1] + result.append(inner) + result.append(quote) + idx = end + len(quote) + elif ch == chr(35): + nl = code.find(NL, idx) + if nl == -1: + result.append(code[idx:]) + idx = len(code) + else: + result.append(code[idx:nl]) + idx = nl + else: + result.append(ch) + idx += 1 + return "".join(result) + + +def _process_fstring_body(code: str, idx: int, quote: str, NL: str, BSN: str, result: list) -> int: + """Process the body of an f-string, escaping literal newlines only in + string-literal portions (brace-depth 0). Expression portions (brace-depth + >= 1) are appended verbatim so that multi-line expressions survive.""" + + literal_start = idx + pos = idx + brace_depth = 0 + + while pos < len(code): + c = code[pos] + + # Closing quote when not inside an expression + if brace_depth == 0 and c == quote: + if len(quote) == 1: + break + # triple-quoted — need three consecutive quote chars + if pos + 2 < len(code) and code[pos : pos + 3] == quote: + break + pos += 1 + continue + + # Backslash — skip the escaped character + if c == chr(92): + pos += 2 + continue + + # Opening brace + if c == chr(123): + if pos + 1 < len(code) and code[pos + 1] == chr(123): + pos += 2 # {{ — literal brace + continue + if brace_depth == 0: + # Flush the string-literal portion up to { + inner = code[literal_start:pos] + inner = json.dumps(inner, ensure_ascii=False)[1:-1] + result.append(inner) + result.append(c) + literal_start = pos + 1 + brace_depth += 1 + pos += 1 + continue + + # Closing brace + if c == chr(125): + if pos + 1 < len(code) and code[pos + 1] == chr(125): + pos += 2 # }} — literal brace + continue + if brace_depth > 0: + brace_depth -= 1 + if brace_depth == 0: + # Expression block ended — append verbatim + result.append(code[literal_start:pos]) + result.append(c) + literal_start = pos + 1 + pos += 1 + continue + + # Nested string literal inside an expression — skip over it + if brace_depth > 0 and (c == chr(34) or c == chr(39)): + nq = c + if pos + 2 < len(code) and code[pos : pos + 3] == nq * 3: + nq = nq * 3 + pos += len(nq) + ne = _find_closing_quote(code, nq, pos) + pos = (ne + len(nq)) if ne != -1 else len(code) + continue + + pos += 1 + + # Flush the final string-literal portion + if literal_start < pos: + inner = code[literal_start:pos] + inner = json.dumps(inner, ensure_ascii=False)[1:-1] + result.append(inner) + + if pos < len(code): + result.append(quote) + pos += len(quote) + + return pos + + +def _fstring_aware_escape_newlines(code: str, NL: str, BSN: str) -> str: + """Like ``_naive_escape_newlines``, but f-string expression blocks + + (``{...}``) have their newlines preserved verbatim.""" + + result = [] + idx = 0 + while idx < len(code): + ch = code[idx] + if ch == chr(34) or ch == chr(39): + quote = ch + if idx + 2 < len(code) and code[idx : idx + 3] == quote * 3: + quote = quote * 3 + + # Detect f-string prefix + prev = idx - 1 + is_fstring = prev >= 0 and code[prev] in "fF" + if not is_fstring and prev >= 1: + if code[prev] in "rR" and code[prev - 1] in "fF": + is_fstring = True + + result.append(quote) + idx += len(quote) + + if is_fstring: + idx = _process_fstring_body(code, idx, quote, NL, BSN, result) + else: + end = _find_closing_quote(code, quote, idx) + if end == -1: + result.append(code[idx:]) + idx = len(code) + break + inner = code[idx:end] + inner = json.dumps(inner, ensure_ascii=False)[1:-1] + result.append(inner) + result.append(quote) + idx = end + len(quote) + elif ch == chr(35): + nl = code.find(NL, idx) + if nl == -1: + result.append(code[idx:]) + idx = len(code) + else: + result.append(code[idx:nl]) + idx = nl + else: + result.append(ch) + idx += 1 + return "".join(result) + + +def _escape_newlines_in_strings(code: str) -> str: + """Pre-process code to escape literal newlines inside string literals. + + So that LLMs can write backslash-n literally in strings without + Python interpreting it as an actual newline causing SyntaxError. + + Uses a two-pass strategy: + 1. Try naive quote-matching (fast, handles regular strings). + 2. If ``compile`` rejects the result, fall back to an f-string-aware + pass that preserves newlines inside ``{...}`` expression blocks. + """ + NL = chr(10) + + # Fast path — no literal newlines, nothing to do + if NL not in code: + return code + + # Fast path — code is already valid Python + try: + compile(code, "", "exec") + return code + except SyntaxError: + pass + + BSN = chr(92) + chr(110) + + # Pass 1 — naive quote-matching (works for regular strings) + escaped = _naive_escape_newlines(code, NL, BSN) + try: + compile(escaped, "", "exec") + return escaped + except SyntaxError: + pass + + # Pass 2 — f-string aware (preserves newlines in expression blocks) + return _fstring_aware_escape_newlines(code, NL, BSN) + + +# Modules that are pre-imported in the sandbox. Import statements for +# these can be safely commented out instead of triggering SecurityError. +_PREIMPORTED_MODULES: frozenset[str] = frozenset( + { + "re", + "math", + "itertools", + "collections", + "datetime", + "traceback", + "json", + "pathlib", + } +) + + +def _strip_allowed_imports( + code: str, + extra_allowed: frozenset[str] | None = None, +) -> tuple[str, dict[str, object]]: + """Strip import lines for modules already pre-imported in the sandbox. + + The sandbox provides ``re``, ``math``, ``itertools``, ``collections``, + ``datetime``, ``traceback``, and ``json`` as read-only proxies. + + When *extra_allowed* is provided, import statements for those modules + are also allowed through (neither stripped nor commented out). The caller + is responsible for ensuring those names are resolvable at execution time + (either by pre-importing them into the sandbox globals or letting the + ``SecurityFilter`` allow the ``import`` statement through). + + Returns ``(code, extra_globals)`` where *extra_globals* maps imported + names to their resolved values. The caller must inject these into the + execution namespace before running the code. + + ``import module`` lines are commented out — the module name is already + available as a global. + + ``from module import name [as alias]`` lines are commented out and the + corresponding entries are added to *extra_globals* so the names are + available without Python scoping issues. + """ + + import re as _re + + preimported = _PREIMPORTED_MODULES + skip_allowed = extra_allowed or frozenset() + + lines = code.splitlines() + + result = [] + extra_globals: dict[str, object] = {} + + for line in lines: + stripped = line.lstrip() + indent = line[: len(line) - len(stripped)] + + # "import module" or "import module as alias" + m = _re.match( + r"import\s+(\w+)(?:\s+as\s+(\w+))?\s*$", + stripped, + ) + if m and m.group(1) in skip_allowed: + result.append(line) + continue + + if m and m.group(1) in preimported: + mod_name = m.group(1) + alias = m.group(2) or mod_name + + result.append(f"{indent}# {stripped} " f"# auto-removed: {mod_name} is pre-imported") + # Only inject if aliased (otherwise the module name is already + # available as-is) + if alias != mod_name: + import importlib + + extra_globals[alias] = importlib.import_module(mod_name) + continue + + # "from module import name1 [as alias1], ..." + m = _re.match( + r"from\s+(\w+)\s+import\s+(.*)$", + stripped, + ) + if m and m.group(1) in skip_allowed: + result.append(line) + continue + + if m and m.group(1) in preimported: + mod_name = m.group(1) + names_clause = m.group(2) + + result.append(f"{indent}# {stripped} " f"# auto-removed: {mod_name} is pre-imported") + + for name_item in names_clause.split(","): + name_item = name_item.strip() + as_match = _re.match( + r"(\w+)\s+as\s+(\w+)\s*$", + name_item, + ) + if as_match: + extra_globals[as_match.group(2)] = _resolve_module_attr( + mod_name, as_match.group(1) + ) + else: + extra_globals[name_item] = _resolve_module_attr(mod_name, name_item) + + continue + + result.append(line) + + return "\n".join(result), extra_globals + + +def _resolve_module_attr(mod_name: str, attr: str) -> object: + """Return ``mod_name.attr`` by importing the real module.""" + + import importlib + + module = importlib.import_module(mod_name) + return getattr(module, attr) + + +class _SafeJson: + """Drop-in ``json`` namespace with ``loads``, ``dumps``, and ``JSONDecodeError``.""" + + JSONDecodeError = json.JSONDecodeError + + @staticmethod + def loads(s: str) -> Any: + import json + + return json.loads(s) + + @staticmethod + def dumps(obj: Any, **kwargs) -> str: + import json + + # Whitelist of safe kwargs for formatting + allowed = {"indent", "sort_keys", "default", "separators", "ensure_ascii", "allow_nan"} + safe_kwargs = {k: v for k, v in kwargs.items() if k in allowed} + return json.dumps(obj, **safe_kwargs) + + +class _SafeTraceback: + """Drop-in ``traceback`` namespace exposing only safe print and format methods. + + Unlike ``_SafeModuleProxy`` which forwards all non-blocked attributes to the + real module, this class only exposes an explicit allowlist of print/format + methods. Module-level imports like ``traceback.sys`` and ``traceback.io`` + are not accessible through this wrapper, closing the ``sys.modules`` + sandbox-escape vector. + + Exposed methods: + + **Printing** (output to stderr or a given file): + - ``print_exc(limit=None, file=None)`` — print current exception + - ``print_exception(exc, limit=None, file=None)`` — print a given exception + - ``print_stack(f=None, limit=None, file=None)`` — print current call stack + - ``print_tb(tb, limit=None, file=None)`` — print a given traceback + - ``print_last(limit=None, file=None)`` — print the last exception + + **Formatting** (return strings for logging): + - ``format_exc(limit=None)`` — format current exception as string + - ``format_exception(exc, limit=None)`` — format a given exception to list of strings + - ``format_stack(f=None, limit=None)`` — format current stack to list of strings + - ``format_tb(tb, limit=None)`` — format a given traceback to list of strings + - ``format_list(extracted)`` — format extracted stack entries + - ``format_exception_only(exc)`` — format exception type + value only + + **Utility**: + - ``clear_frames(tb)`` — clear local variables from traceback frames + + **Classes**: + - ``TracebackException`` — programmatic traceback representation + - ``FrameSummary`` — single frame summary + - ``StackSummary`` — list of frame summaries + """ + + @classmethod + def format_exc(cls, limit=None): + import traceback + + return traceback.format_exc(limit=limit) + + @classmethod + def format_exception(cls, exc, limit=None): + import traceback + + return traceback.format_exception(exc, limit=limit) + + @classmethod + def format_stack(cls, f=None, limit=None): + import traceback + + return traceback.format_stack(f=f, limit=limit) + + @classmethod + def format_tb(cls, tb, limit=None): + import traceback + + return traceback.format_tb(tb, limit=limit) + + @classmethod + def format_list(cls, extracted): + import traceback + + return traceback.format_list(extracted) + + @classmethod + def format_exception_only(cls, exc): + import traceback + + return traceback.format_exception_only(exc) + + @classmethod + def print_exc(cls, limit=None, file=None): + import traceback + + traceback.print_exc(limit=limit, file=file) + + @classmethod + def print_exception(cls, exc, limit=None, file=None): + import traceback + + traceback.print_exception(exc, limit=limit, file=file) + + @classmethod + def print_stack(cls, f=None, limit=None, file=None): + import traceback + + traceback.print_stack(f=f, limit=limit, file=file) + + @classmethod + def print_tb(cls, tb, limit=None, file=None): + import traceback + + traceback.print_tb(tb, limit=limit, file=file) + + @classmethod + def print_last(cls, limit=None, file=None): + import traceback + + traceback.print_last(limit=limit, file=file) + + @classmethod + def clear_frames(cls, tb): + import traceback + + traceback.clear_frames(tb) + + # Classes from traceback — accessed via class attribute, not instance + TracebackException = None + FrameSummary = None + StackSummary = None + + +_SafeTraceback.TracebackException = _tb_mod.TracebackException +_SafeTraceback.FrameSummary = _tb_mod.FrameSummary +_SafeTraceback.StackSummary = _tb_mod.StackSummary + + +class _SafeRe: + """Drop-in ``re`` namespace exposing only safe regex operations. + + Unlike ``_SafeModuleProxy`` which forwards all non-blocked attributes to the + real module, this class only exposes an explicit allowlist of regex + functions, flags, and types. Module-level imports like ``re.enum``, + ``re.copyreg``, and ``re.functools`` are not accessible through this + wrapper, closing the ``re.enum.sys`` sandbox-escape vector. + + Exposed: + + **Compilation**: + - ``compile(pattern, flags=0)`` — compile a regex pattern + + **Searching / Matching**: + - ``search(pattern, string, flags=0)`` — scan through string + - ``match(pattern, string, flags=0)`` — match at start of string + - ``fullmatch(pattern, string, flags=0)`` — match whole string + - ``findall(pattern, string, flags=0)`` — all non-overlapping matches + - ``finditer(pattern, string, flags=0)`` — iterator over matches + + **Substitution / Splitting**: + - ``sub(pattern, repl, string, ...)`` — substitute + - ``subn(pattern, repl, string, ...)`` — substitute with count + - ``split(pattern, string, ...)`` — split by pattern + + **Utilities**: + - ``escape(string)`` — escape special characters + - ``purge()`` — clear the regex cache + + **Flags**: + - ``NOFLAG``, ``IGNORECASE`` / ``I``, ``MULTILINE`` / ``M``, + ``DOTALL`` / ``S``, ``VERBOSE`` / ``X``, ``ASCII`` / ``A`` + - ``RegexFlag`` — the flag enum type + + **Types / Exceptions**: + - ``Pattern`` — compiled pattern type + - ``Match`` — match result type + - ``error`` — regex exception class + """ + + # Compilation + compile = None + + # Searching / Matching + search = None + match = None + fullmatch = None + findall = None + finditer = None + + # Substitution / Splitting + sub = None + subn = None + split = None + + # Utilities + escape = None + purge = None + + # Flags + NOFLAG = None + IGNORECASE = None + I = None # noqa: E741 + MULTILINE = None + M = None + DOTALL = None + S = None + VERBOSE = None + X = None + ASCII = None + A = None + RegexFlag = None + + # Types / Exceptions + Pattern = None + Match = None + error = None + + +for _name in dir(_SafeRe): + if not _name.startswith("_"): + try: + setattr(_SafeRe, _name, getattr(_re_mod, _name)) + except AttributeError: + pass + + +class _SafeModuleProxy: + """Proxy that forwards attribute reads to a real module but prevents mutation. + + Setting an attribute on the proxy only affects the proxy, not the real module. + This prevents sandbox code from monkey-patching standard library modules. + + The following methods are blocked and raise SecurityError: + - ``walk_stack()``, ``walk_tb()``, ``extract_stack()`` — frame introspection + that exposes unwrapped frame objects with access to parent-frame + ``f_locals`` / ``f_globals``. + """ + + _BLOCKED_METHODS: frozenset[str] = frozenset({"walk_stack", "walk_tb", "extract_stack"}) + + def __init__(self, module: Any, *, disable_security: bool = False) -> None: + object.__setattr__(self, "_module", module) + object.__setattr__(self, "_disable_security", disable_security) + + def __getattr__(self, name: str) -> Any: + if name.startswith("_"): + raise AttributeError( + "Cannot access private attribute " + repr(name) + " on module proxy" + ) + if name in self._BLOCKED_METHODS and not object.__getattribute__(self, "_disable_security"): + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + f"Cannot access '{name}' on module proxy. " + f"Frame introspection methods are blocked in the sandbox." + ) + if name == "modules" and not object.__getattribute__(self, "_disable_security"): + from cecli.helpers.orchestration.security import SecurityError + + raise SecurityError( + f"Cannot access '{name}' on module proxy. " + f"Direct module registry access is blocked in the sandbox." + ) + module = object.__getattribute__(self, "_module") + attr = getattr(module, name) + + # Re-wrap sub-modules to prevent transitive sandbox escape. + # Without this, `re.enum.sys` would return the real `sys` module + # (since `re` imports `enum` and `enum` imports `sys`), giving + # attackers access to `sys.modules["builtins"].getattr` and from + # there unrestricted private-attribute access. + from types import ModuleType + + if isinstance(attr, ModuleType): + return _SafeModuleProxy( + attr, disable_security=object.__getattribute__(self, "_disable_security") + ) + + return attr + + def __setattr__(self, name: str, value: Any) -> None: + if not name.startswith("_"): + raise AttributeError("Module proxy is read-only — cannot set " + repr(name)) + object.__setattr__(self, name, value) + + def __dir__(self) -> list: + module = object.__getattribute__(self, "_module") + blocked = ( + set() if object.__getattribute__(self, "_disable_security") else self._BLOCKED_METHODS + ) + return [x for x in dir(module) if not x.startswith("_") and x not in blocked] + + +# --------------------------------------------------------------------------- +# Helpful builtins +# --------------------------------------------------------------------------- + + +class _HelpfulBuiltins(dict): + """Custom __builtins__ dict that provides helpful hints for missing functions.""" + + _HINTS: dict[str, str] = { + "open": "Filesystem access is not available. Use the Command tool instead.", + "eval": "eval() is disabled for security.", + "exec": "exec() is disabled for security.", + "__import__": "Imports are disabled. Use only the primitives provided.", + "compile": "compile() is disabled for security.", + "breakpoint": "breakpoint() is disabled in the sandbox.", + "globals": "globals() is disabled. Use state or shared_state for persistence.", + "locals": "locals() is disabled. Use state or shared_state for persistence.", + "vars": "Use vars(obj) instead of vars() — vars(obj) returns non-dunder attrs of obj.", + "setattr": "setattr() is disabled. Assign attributes directly.", + "delattr": "delattr() is disabled. Use del obj.attr instead.", + } + + def __missing__(self, key: str): + hint = self._HINTS.get(key) + if hint: + raise NameError(f"'{key}' is not available. {hint}") + raise NameError(f"name '{key}' is not defined") diff --git a/cecli/helpers/orchestration/security.py b/cecli/helpers/orchestration/security.py new file mode 100644 index 00000000000..4a7580b2061 --- /dev/null +++ b/cecli/helpers/orchestration/security.py @@ -0,0 +1,267 @@ +""" +AST-level security transforms for the orchestration sandbox. + +- SecurityFilter: blocks imports, dunder attributes, dangerous builtins +- LoopYieldInjector: injects cooperative yields into loops +- _cooperative_yield: the injected yield function +""" + +import ast +import asyncio + + +class SecurityError(Exception): + """Raised when generated code violates security constraints.""" + + +def _security_raise(message: str): + """Runtime raise helper injected into sandbox globals. + + Called by AST-rewritten forbidden expressions so that private-access + violations raise at *runtime* rather than rejecting the entire script + during the pre-execution AST walk. This allows try/except blocks to + gracefully handle unreachable code paths. + """ + raise SecurityError(message) + + +class SecurityFilter(ast.NodeTransformer): + """ + AST node transformer that rewrites dangerous constructs into runtime + ``__security_raise(...)`` calls. + + Instead of rejecting the entire script during the pre-execution walk, + forbidden constructs are replaced with a call that raises + ``SecurityError`` at *runtime*. This means code inside ``try/except`` + blocks can gracefully handle unreachable paths while the security + boundary is preserved — any actual execution of private access still + fails. + + Rewrites: + - All import statements (import X, from X import Y) + - Access to private/dunder attributes (__class__, __subclasses__, etc.) + - Calls to eval, exec, open, __import__, compile, breakpoint + - global / nonlocal statements + """ + + _DANGEROUS_BUILTINS: set[str] = { + "eval", + "exec", + "open", + "__import__", + "compile", + "breakpoint", + "globals", + "locals", + "setattr", + "delattr", + } + + _SAFE_DUNDER: set[str] = {"__name__", "__doc__"} + + _SAFE_CLASS_DUNDERS: frozenset[str] = frozenset( + { + "__init__", + "__str__", + "__repr__", + "__iter__", + "__next__", + "__len__", + "__getitem__", + "__setitem__", + "__contains__", + "__enter__", + "__exit__", + "__aenter__", + "__aexit__", + "__await__", + "__anext__", + "__aiter__", + "__del__", + "__init_subclass__", + "__set_name__", + "__class_getitem__", + } + ) + + def __init__( + self, + allowed_imports: frozenset[str] | None = None, + allow_classes: bool = False, + ): + super().__init__() + self._allowed_imports = allowed_imports or frozenset() + self._allow_classes = allow_classes + self._class_depth = 0 + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + @staticmethod + def _make_raise_expr(message: str) -> ast.Call: + """Return an AST Call node that invokes ``_security_raise(message)``. + + The function ``_security_raise`` is injected into the sandbox + globals at execution time. + """ + return ast.Call( + func=ast.Name(id="__security_raise", ctx=ast.Load()), + args=[ast.Constant(value=message)], + keywords=[], + ) + + @staticmethod + def _make_raise_stmt(message: str) -> ast.Expr: + """Return an AST Expr statement wrapping ``_security_raise(message)``. + + Used when the forbidden construct is itself a statement + (import / global / nonlocal) rather than an expression. + """ + return ast.Expr(value=SecurityFilter._make_raise_expr(message)) + + # ------------------------------------------------------------------ + # Statement visitors (import / global / nonlocal) + # ------------------------------------------------------------------ + + def visit_Import(self, node: ast.Import) -> ast.Expr | ast.Import: + for alias in node.names: + if alias.name not in self._allowed_imports: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + f"Import '{alias.name}' is not in allowed_imports." + ) + + return node + + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.Expr | ast.ImportFrom: + if node.module is None or node.module not in self._allowed_imports: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + f"Import from '{node.module}' is not in allowed_imports." + ) + + return node + + def visit_Global(self, node: ast.Global) -> ast.Expr: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "The 'global' statement is disabled in the orchestration environment." + ) + + def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.Expr: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "The 'nonlocal' statement is disabled in the orchestration environment." + ) + + # ------------------------------------------------------------------ + def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef | ast.Expr: + if self._allow_classes: + self._class_depth += 1 + self.generic_visit(node) + self._class_depth -= 1 + return node + + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "Class definitions are disabled in the orchestration environment." + ) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef | ast.Expr: + if self._allow_classes and self._class_depth > 0 and node.name.startswith("__"): + if node.name not in self._SAFE_CLASS_DUNDERS: + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + f"Method name '{node.name}' is not allowed. " + f"Only safe dunder methods are permitted inside class bodies." + ) + + return self.generic_visit(node) + + # ------------------------------------------------------------------ + # Expression visitors (attribute access / dangerous calls) + + def visit_Attribute(self, node: ast.Attribute): + if node.attr.startswith("_"): + if node.attr in self._SAFE_DUNDER: + return self.generic_visit(node) + + if ( + self._allow_classes + and self._class_depth > 0 + and node.attr in self._SAFE_CLASS_DUNDERS + ): + return self.generic_visit(node) + + if isinstance(node.ctx, (ast.Store, ast.Del)): + verb = "assign to" if isinstance(node.ctx, ast.Store) else "delete" + return self._make_raise_expr( + f"Security filter error at line {node.lineno}: " + f"cannot {verb} private attribute '{node.attr}'" + ) + + return self._make_raise_expr( + f"Security filter error at line {node.lineno}: " + f"Access to private/dunder attribute '{node.attr}' is forbidden." + ) + + return self.generic_visit(node) + + def visit_Call(self, node: ast.Call): + if isinstance(node.func, ast.Name) and node.func.id in self._DANGEROUS_BUILTINS: + return self._make_raise_expr( + f"Security filter error at line {node.lineno}: " + f"Calling '{node.func.id}' is forbidden." + ) + + return self.generic_visit(node) + + def visit_Match(self, node: ast.Match) -> ast.Expr: + """Block all ``match``/``case`` statements. + + Class patterns destructure objects via runtime ``getattr()``, + bypassing the AST-level ``visit_Attribute`` guard entirely. + Rather than trying to cover every pattern variant, we reject + the ``match`` statement wholesale. + """ + return self._make_raise_stmt( + f"Security filter error at line {node.lineno}: " + "The 'match'/'case' statement is disabled in the orchestration environment." + ) + + +class LoopYieldInjector(ast.NodeTransformer): + """ + Injects ``await __yield()`` at the top of every ``for`` and ``while`` loop body. + + This forces cooperative multitasking so that infinite loops can be cancelled + via ``asyncio.wait_for`` timeout. + """ + + def __init__(self) -> None: + super().__init__() + self._yield_stmt = ast.Expr( + value=ast.Await( + value=ast.Call( + func=ast.Name(id="__yield", ctx=ast.Load()), + args=[], + keywords=[], + ) + ) + ) + + def visit_While(self, node: ast.While) -> ast.While: + self.generic_visit(node) + node.body.insert(0, self._yield_stmt) + return node + + def visit_For(self, node: ast.For) -> ast.For: + self.generic_visit(node) + node.body.insert(0, self._yield_stmt) + return node + + +async def _cooperative_yield() -> None: + """Force the current coroutine to briefly yield to the event loop.""" + await asyncio.sleep(0) diff --git a/cecli/helpers/orchestration/service.py b/cecli/helpers/orchestration/service.py new file mode 100644 index 00000000000..9856da3091e --- /dev/null +++ b/cecli/helpers/orchestration/service.py @@ -0,0 +1,34 @@ +import weakref + +from cecli.helpers.orchestration.environment import AgentExecutionEnv + + +class OrchestrationService: + """ + Singleton-per-coder registry for AgentExecutionEnv instances. + + Uses weak references (keyed by coder and uuid) so that envs are + automatically cleaned up when their owning coder is garbage collected. + """ + + _instances = weakref.WeakKeyDictionary() + _uuid_index = weakref.WeakValueDictionary() + + @classmethod + def get_instance(cls, coder) -> AgentExecutionEnv: + if coder in cls._instances: + return cls._instances[coder] + + if coder.uuid in cls._uuid_index: + instance = cls._uuid_index[coder.uuid] + cls._instances[coder] = instance + return instance + + orchestration_config = {} + if hasattr(coder, "agent_config"): + orchestration_config = coder.agent_config.get("orchestration", {}) + + instance = AgentExecutionEnv(coder, orchestration_config=orchestration_config) + cls._instances[coder] = instance + cls._uuid_index[coder.uuid] = instance + return instance diff --git a/cecli/helpers/orchestration/tool_proxy.py b/cecli/helpers/orchestration/tool_proxy.py new file mode 100644 index 00000000000..4eb9b098316 --- /dev/null +++ b/cecli/helpers/orchestration/tool_proxy.py @@ -0,0 +1,274 @@ +""" +Proxy for a single tool — local or MCP. + +The LLM code calls ``tool.call(**params)`` and this proxy routes it +through the appropriate execution path. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + + +class ToolResult(dict): + """A result dict that supports both key and attribute access. + + Wraps tool call results so LLM code can use either style: + + result = await tool.call(...) + print(result["result"]) # key access (dict-compatible) + print(result.result) # attribute access (convenient) + print(result.errors) # attribute access + + Since ToolResult is a dict subclass, all dict methods + (``.get()``, ``.keys()``, ``in``, etc.) work as expected. + """ + + def __getattr__(self, key: str): + if key.startswith("_"): + raise AttributeError(f"Cannot access private attribute {key!r} on ToolResult") + try: + return self[key] + except KeyError: + raise AttributeError( + f"ToolResult has no key {key!r}. " + f"Available keys: {', '.join(sorted(self.keys()))}" + ) from None + + def __setattr__(self, key: str, value) -> None: + self[key] = value + + def __delattr__(self, key: str) -> None: + try: + del self[key] + except KeyError: + raise AttributeError(f"ToolResult has no key {key!r}") from None + + +class ToolProxy: + """ + Proxy for a single tool — local or MCP. + + The LLM code calls ``tool.call(**params)`` and this proxy routes it + through the appropriate execution path. + """ + + def __init__( + self, + tool_name: str, + coder: Any, + *, + tool_module: Any = None, + mcp_server: Any = None, + mcp_tool_name: str = "", + ) -> None: + # Respect the per-coder tool includelist/excludelist filters + incl = getattr(coder, "registered_tools", {}).get("included", set()) + excl = getattr(coder, "registered_tools", {}).get("excluded", set()) + name_lower = tool_name.lower() + if incl and name_lower not in incl: + raise ValueError(f"Tool '{tool_name}' is not in the allowed tools list.") + if name_lower in excl: + raise ValueError(f"Tool '{tool_name}' has been excluded.") + + self._tool_name = tool_name + self._coder = coder + self._tool_module = tool_module + self._mcp_server = mcp_server + self._mcp_tool_name = mcp_tool_name + + async def __call__(self, *args: Any, **kwargs: Any): + """Make the proxy directly callable. + + Supports both ``await tool(key=val)`` and ``await tool("val")``. + Positional arguments are mapped to parameter names using the + tool's schema (when available). + """ + if args and kwargs: + raise TypeError( + f"Tool '{self._tool_name}': cannot mix positional and keyword arguments" + ) + + if args: + param_names = self._get_param_names() + if not param_names: + if len(args) == 1: + # Fallback: try common first-param names + for guess in ( + "path", + "read", + "searches", + "edits", + "queries", + "tasks", + "delegations", + "code", + "command_string", + "command", + "summary", + ): + kwargs = {guess: args[0]} + break + else: + raise TypeError( + f"Tool '{self._tool_name}': cannot resolve positional " + f"argument – no schema available" + ) + else: + raise TypeError( + f"Tool '{self._tool_name}': cannot resolve positional " + f"arguments – no schema available" + ) + elif len(args) > len(param_names): + raise TypeError( + f"Tool '{self._tool_name}': too many positional arguments " + f"({len(args)} for {len(param_names)} parameter(s): {param_names})" + ) + else: + kwargs = dict(zip(param_names, args)) + + return await self.call(**kwargs) + + def _get_param_names(self) -> list: + """Extract ordered parameter names from the tool's JSON Schema.""" + if self._tool_module is None: + return [] + try: + props = self._tool_module.SCHEMA["function"]["parameters"]["properties"] + return list(props.keys()) + except (KeyError, TypeError, AttributeError): + return [] + + async def call(self, **kwargs: Any): + """Execute the tool with the given keyword arguments. + + Tool results are normalized to a dict with ``result`` (list), + ``errors`` (list), and ``details`` (list) keys, matching the + documented orchestration contract. + """ + + if self._tool_module is not None: + result = self._tool_module.process_response(self._coder, kwargs, _convert=False) + if asyncio.iscoroutine(result): + result = await result + result = self._tool_module.ptc_format(result) + return self._normalize_result(result) + + if self._mcp_server is not None: + result = await self._coder._execute_mcp_tool( + self._mcp_server, self._mcp_tool_name, kwargs + ) + return self._normalize_result(result) + + raise ValueError(f"No executor for tool '{self._tool_name}'") + + @staticmethod + def _normalize_result(result: Any) -> "ToolResult": + """Normalize a tool result into a ToolResult with ``result``, ``errors``, ``details`` keys. + + Each item in the ``result`` list has the shape ``{"content": ..., "_": {...}}``. + Supports both key access (``result["result"]``) and + attribute access (``result.result``). + """ + + from cecli.tools.utils.responses import ToolResponse + + if isinstance(result, ToolResponse): + data = result.to_dict() + return ToolResult( + { + "result": data.get("result", []), + "errors": data.get("errors", []), + "details": data.get("details", []), + } + ) + + if isinstance(result, str): + # Attempt to auto-parse a JSON-stringified ToolResponse + try: + parsed = json.loads(result) + if isinstance(parsed, dict): + if "result" in parsed: + result_list = parsed["result"] + if not isinstance(result_list, list): + result_list = [result_list] if result_list else [] + return ToolResult( + { + "result": [ + ( + item + if isinstance(item, dict) and "content" in item + else {"content": item, "_": {}} + ) + for item in result_list + ], + "errors": parsed.get("errors", []), + "details": parsed.get("details", []), + } + ) + return ToolResult( + { + "result": [{"content": parsed, "_": {}}], + "errors": [], + "details": [], + } + ) + except (json.JSONDecodeError, TypeError): + pass + + # Error strings from handle_tool_error + if result.startswith("Error in "): + return ToolResult( + { + "result": [], + "errors": [result], + "details": [], + } + ) + + # Plain string result + return ToolResult( + { + "result": [{"content": result, "_": {}}], + "errors": [], + "details": [], + } + ) + + if isinstance(result, dict): + if "result" in result: + result_list = result["result"] + if not isinstance(result_list, list): + result_list = [result_list] if result_list else [] + + out = dict(result) + out["result"] = [ + ( + item + if isinstance(item, dict) and "content" in item + else {"content": item, "_": {}} + ) + for item in result_list + ] + out.setdefault("errors", []) + out.setdefault("details", []) + return ToolResult(out) + + return ToolResult( + { + "result": [{"content": result, "_": {}}], + "errors": [], + "details": [], + } + ) + + # Fallback: wrap anything else + return ToolResult( + { + "result": [{"content": str(result), "_": {}}] if result is not None else [], + "errors": [], + "details": [], + } + ) diff --git a/cecli/helpers/similarity.py b/cecli/helpers/similarity.py index 37e43b640ca..4e7f0725d9c 100644 --- a/cecli/helpers/similarity.py +++ b/cecli/helpers/similarity.py @@ -1,42 +1,38 @@ -import numpy as np - - def normalize_vector(vector): """Normalize a vector to unit length (L2 norm). Args: - vector (np.ndarray or list): Input vector + vector (list): Input vector Returns: - np.ndarray: Normalized vector with length 1 + list: Normalized vector with length 1 """ - vector = np.asarray(vector, dtype=np.float64) - magnitude = np.linalg.norm(vector) + import math + + magnitude = math.sqrt(sum(x * x for x in vector)) if magnitude == 0: - return vector # Return original if zero vector - return vector / magnitude + return list(vector) # Return copy if zero vector + return [x / magnitude for x in vector] def cosine_similarity(vector1, vector2): """Calculate cosine similarity between two vectors. Args: - vector1 (np.ndarray or list): First vector - vector2 (np.ndarray or list): Second vector + vector1 (list): First vector + vector2 (list): Second vector Returns: float: Cosine similarity between the vectors (range: -1 to 1) """ - vector1 = np.asarray(vector1, dtype=np.float64) - vector2 = np.asarray(vector2, dtype=np.float64) + import math if len(vector1) != len(vector2): raise ValueError("Vectors must have the same length") - # Use NumPy's optimized dot product and norm functions - dot_product = np.dot(vector1, vector2) - magnitude1 = np.linalg.norm(vector1) - magnitude2 = np.linalg.norm(vector2) + dot_product = sum(a * b for a, b in zip(vector1, vector2)) + magnitude1 = math.sqrt(sum(x * x for x in vector1)) + magnitude2 = math.sqrt(sum(x * x for x in vector2)) if magnitude1 == 0 or magnitude2 == 0: return 0.0 # Return 0 if either vector is zero @@ -45,16 +41,13 @@ def cosine_similarity(vector1, vector2): def create_bigram_vector(texts): - """Create a bigram frequency vector using optimized NumPy operations. - - This version uses pre-computed bigram indices and NumPy's bincount - for maximum performance on large datasets. + """Create a bigram frequency vector. Args: texts (tuple): Tuple of strings to process Returns: - np.ndarray: Vector of bigram frequencies + list: Vector of bigram frequencies """ # Pre-compute bigram indices (0 for 'aa', 1 for 'ab', ..., 675 for 'zz') bigram_indices = {} @@ -66,7 +59,7 @@ def create_bigram_vector(texts): idx += 1 # Initialize frequency vector - vector = np.zeros(26 * 26, dtype=np.int32) + vector = [0] * (26 * 26) # Process all texts for text in texts: @@ -74,25 +67,11 @@ def create_bigram_vector(texts): if len(text_lower) < 2: continue - # Extract bigrams using NumPy sliding window view - # Convert string to character array for efficient slicing - chars = np.array(list(text_lower)) - # Create bigrams by combining consecutive characters - bigrams = np.char.add(chars[:-1], chars[1:]) - - # Filter only alphabetic bigrams - mask = np.array([bg.isalpha() for bg in bigrams]) - valid_bigrams = bigrams[mask] - - # Count bigrams using bincount with pre-computed indices - indices = [] - for bg in valid_bigrams: - if bg in bigram_indices: - indices.append(bigram_indices[bg]) - - if indices: - counts = np.bincount(indices, minlength=26 * 26) - vector += counts + for i in range(len(text_lower) - 1): + bg = text_lower[i : i + 2] + # Filter only alphabetic bigrams + if bg.isalpha() and bg in bigram_indices: + vector[bigram_indices[bg]] += 1 return vector diff --git a/cecli/hooks/helpers.py b/cecli/hooks/helpers.py index 7b8ce539c1a..0d0677c2c11 100644 --- a/cecli/hooks/helpers.py +++ b/cecli/hooks/helpers.py @@ -3,7 +3,7 @@ Provides a higher-level API for accessing conversation history, making model calls, and working with sub-agents from within hook implementations. -Typical usage:: +Typical usage: from cecli.hooks import HookHelpers diff --git a/cecli/hooks/registry.py b/cecli/hooks/registry.py index d58a60b824c..cff8dfab3da 100644 --- a/cecli/hooks/registry.py +++ b/cecli/hooks/registry.py @@ -149,7 +149,6 @@ def load_hooks_from_config(self, config_file: Path) -> List[str]: loaded_hooks.append(hook.name) else: print(f"Warning: Could not register hook '{hook.name}': {e}") - print(f"Warning: Could not register hook '{hook.name}': {e}") return loaded_hooks diff --git a/cecli/io.py b/cecli/io.py index 9d03c9be88a..89e02462773 100644 --- a/cecli/io.py +++ b/cecli/io.py @@ -11,7 +11,6 @@ import subprocess import sys import time -import webbrowser from collections import defaultdict from dataclasses import dataclass from datetime import datetime @@ -360,7 +359,7 @@ def __init__( encoding="utf-8", line_endings="platform", dry_run=False, - editingmode=EditingMode.EMACS, + editingmode="EMACS", fancy_input=True, file_watcher=None, multiline_mode=False, @@ -526,10 +525,10 @@ def __init__( "input": self.input, "output": self.output, "lexer": PygmentsLexer(MarkdownLexer), - "editing_mode": self.editingmode, + "editing_mode": EditingMode(self.editingmode), "bottom_toolbar": self.get_bottom_toolbar, } - if self.editingmode == EditingMode.VI: + if self.editingmode == "VI": session_kwargs["cursor"] = ModalCursorShapeConfig() if self.input_history_file is not None: session_kwargs["history"] = FileHistory(self.input_history_file) @@ -1235,6 +1234,8 @@ async def offer_url( self, url, prompt="Open URL for more info?", allow_never=True, acknowledge=False ): """Offer to open a URL in the browser, returns True if opened.""" + import webbrowser + if url in self.never_prompts: return False if await self.confirm_ask( diff --git a/cecli/main.py b/cecli/main.py index aeaf49f1acd..e49733b1934 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -10,23 +10,19 @@ try: if not os.getenv("CECLI_DEFAULT_TLS"): - import truststore + from truststore import inject_into_ssl - truststore.inject_into_ssl() + inject_into_ssl() except Exception as e: print(e) pass import asyncio import json -import os import re import shutil -import threading import time import traceback -import webbrowser -from dataclasses import fields from pathlib import Path try: @@ -54,7 +50,6 @@ def new_event_loop(self): return asyncio.SelectorEventLoop(selectors.SelectSelector()) asyncio.set_event_loop_policy(_SelectSelectorPolicy()) -from prompt_toolkit.enums import EditingMode from .dump import dump # noqa @@ -490,8 +485,7 @@ def custom_tracer(frame, event, arg): def main(argv=None, input=None, output=None, force_git_root=None, return_coder=False): - from cecli.commands import ReloadProgramSignal - from cecli.hooks import HookService + from cecli.signals import ReloadProgramSignal # Tracks the coder instance from a ReloadProgramSignal so the new # main_async() can pass it as from_coder to Coder.create(), preserving @@ -529,6 +523,8 @@ def main(argv=None, input=None, output=None, force_git_root=None, return_coder=F # The old HookManager and HookRegistry instances are cached by UUID and # would be reused by the new coder, causing hook registration failures. if reload_from_coder: + from cecli.hooks import HookService + HookService.destroy_instances(reload_from_coder.uuid) continue @@ -558,7 +554,7 @@ async def main_async( from cecli.mcp import McpServerManager, load_mcp_servers from cecli.models import ModelSettings from cecli.onboarding import offer_openrouter_oauth, select_default_model - from cecli.repo import GitRepo + from cecli.repo import GitRepo, GitRepoProxy from cecli.report import report_uncaught_exceptions, set_args_error_data from cecli.versioncheck import check_version from cecli.watch import FileWatcher @@ -692,7 +688,7 @@ async def main_async( args.yes_always = True if args.yes_always_commands: args.yes_always = True - editing_mode = EditingMode.VI if args.vim else EditingMode.EMACS + editing_mode = "VI" if args.vim else "EMACS" def get_io(pretty): return InputOutput( @@ -1030,6 +1026,8 @@ def get_io(pretty): if main_model.edit_format in ("diff", "whole", "diff-fenced"): main_model.edit_format = "editor-" + main_model.edit_format if args.verbose: + from dataclasses import fields + io.tool_output("Model metadata:") io.tool_output(json.dumps(main_model.info, indent=4)) io.tool_output("Model settings:") @@ -1043,20 +1041,22 @@ def get_io(pretty): repo = None if args.git: try: - repo = GitRepo( - io, - fnames, - git_dname, - args.cecli_ignore, - models=main_model.commit_message_models(), - attribute_author=args.attribute_author, - attribute_committer=args.attribute_committer, - attribute_commit_message_author=args.attribute_commit_message_author, - attribute_commit_message_committer=args.attribute_commit_message_committer, - commit_prompt=args.commit_prompt, - subtree_only=args.subtree_only, - git_commit_verify=args.git_commit_verify, - attribute_co_authored_by=args.attribute_co_authored_by, + repo = GitRepoProxy( + GitRepo( + io, + fnames, + git_dname, + args.cecli_ignore, + models=main_model.commit_message_models(), + attribute_author=args.attribute_author, + attribute_committer=args.attribute_committer, + attribute_commit_message_author=args.attribute_commit_message_author, + attribute_commit_message_committer=args.attribute_commit_message_committer, + commit_prompt=args.commit_prompt, + subtree_only=args.subtree_only, + git_commit_verify=args.git_commit_verify, + attribute_co_authored_by=args.attribute_co_authored_by, + ) ) except FileNotFoundError: pass @@ -1271,6 +1271,8 @@ def get_io(pretty): args.edit_format = main_model.editor_edit_format args.message = "/paste" if args.show_release_notes is True: + import webbrowser + pre_init_io.tool_output(f"Opening release notes: {urls.release_notes}") pre_init_io.tool_output() webbrowser.open(urls.release_notes) @@ -1472,6 +1474,8 @@ async def check_and_load_imports(io, is_first_run, verbose=False): else: if verbose: io.tool_output("Not first run, loading imports in background thread") + import threading + thread = threading.Thread(target=load_slow_imports) thread.daemon = True thread.start() @@ -1483,9 +1487,7 @@ async def check_and_load_imports(io, is_first_run, verbose=False): def load_slow_imports(swallow=True): try: - import httpx # noqa - import litellm # noqa - import numpy # noqa + import cecli.llm # noqa except Exception as e: if not swallow: raise e diff --git a/cecli/models.py b/cecli/models.py index 3ded65eb151..6f41fa4b4f3 100644 --- a/cecli/models.py +++ b/cecli/models.py @@ -1,12 +1,8 @@ import asyncio -import difflib -import hashlib import importlib.resources import json -import math import os import platform -import random import sys import time from dataclasses import dataclass, fields @@ -975,6 +971,8 @@ def token_count_for_image(self, fname): :param fname: The filename of the image. :return: The token cost for the image. """ + import math + width, height = self.get_image_size(fname) max_dimension = max(width, height) if max_dimension > 2048: @@ -1224,6 +1222,10 @@ async def send_completion( override_kwargs={}, interrupt_event=None, ): + import random + + import xxhash + if os.environ.get("CECLI_SANITY_CHECK_TURNS"): sanity_check_messages(messages) messages = model_request_parser(self, messages, tools) @@ -1305,7 +1307,7 @@ async def send_completion( num_ctx = int(self.token_count(messages) * 1.25) + 8192 kwargs["num_ctx"] = num_ctx key = json.dumps(kwargs, sort_keys=True).encode() - hash_object = hashlib.sha1(key) + hash_object = xxhash.xxh64(key) if "timeout" not in kwargs: kwargs["timeout"] = request_timeout if self.verbose: @@ -1714,6 +1716,7 @@ def get_chat_model_names(query: str = "") -> list: def fuzzy_match_models(name): + import difflib import fnmatch # Handle empty string case - return all models diff --git a/cecli/prompts/agent.yml b/cecli/prompts/agent.yml index cd229fed7e6..8ca4718cf63 100644 --- a/cecli/prompts/agent.yml +++ b/cecli/prompts/agent.yml @@ -22,26 +22,34 @@ main_system: | ## Core Directives **Act Proactively**: Autonomously use tools to fulfill the request. **Be Decisive**: Do not repeat searches or ask redundant questions. Trust your findings and be confident in your edits. - **Be Efficient**: Use multiple tools each response when exploring. Batch tool calls when the schema allows you to. Respect usage limits while maximizing the utility of each response. + **Be Efficient**: Use multiple tools each request when exploring. Orchestrate tool calls for mutli-step operations. Respect usage limits while maximizing the utility of each request. **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT - File contents will be prefixed with identifiers. Each line starts with a variable length content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change after performing file edits. Do not search for these content IDs directly. You will not be able to generate them. - Focus on the lines they identify. + File contents are presented with virtual prefix identifiers to help you target edits accurately. These are generated on-the-fly when reading the file. + + - **Unique Lines (`~~`):** Lines that appear only once in the file are prefixed with `~~`. To target these lines for edits, you can simply reference the exact literal text of the line. + - **Duplicate Lines (e.g., `~“0车加~`):** Lines that appear multiple times are prefixed with an identifier containing an occurrence index and a content hash. You MUST include this exact prefix when targeting these lines to disambiguate which specific instance you want to edit. + + Do not attempt to generate, guess, or calculate these duplicate IDs yourself. Always use the exact line contents and prefixes provided from the most recent file read. **Example File** ``` - GoTXSp::#!/usr/bin/env python3 + ~~#!/usr/bin/env python3 + + ~~def example_process_data(data): + ~~ if not data: + ~“0车加~ return None - ILLPTX::def example_method(): - ItWHVI:: return "example" + ~~ data.clean() - SDROMc::def foo(): - FPEPD:: return "bar" + ~~ if data.is_stale(): + ~引0车加~ return None - HiHyKn::def bar(): - XXTDIR:: return example_method() + " " + foo() + ~~ try: + ~~ data.save() + ~~ except Exception: + ~니0车加~ return None ``` ## Core Workflow diff --git a/cecli/prompts/subagent.yml b/cecli/prompts/subagent.yml index 92a0d257f62..191bffd66e6 100644 --- a/cecli/prompts/subagent.yml +++ b/cecli/prompts/subagent.yml @@ -7,26 +7,34 @@ main_system: | ## Core Directives **Act Proactively**: Autonomously use tools to fulfill the request. **Be Decisive**: Do not repeat searches or ask redundant questions. Trust your findings and be confident in your edits. - **Be Efficient**: Use multiple tools each response when exploring. Batch tool calls when the schema allows you to. Respect usage limits while maximizing the utility of each response. + **Be Efficient**: Use multiple tools each request when exploring. Orchestrate tool calls for mutli-step operations. Respect usage limits while maximizing the utility of each request. **Be Persistent**: Do not take short cuts. Work through your task until completion. No task takes too long as long as you are making progress towards the goal. ## FILE FORMAT - File contents will be prefixed with identifiers. Each line starts with a variable length content ID followed by `::`. These are used to target where editing tools will perform edits. - They are generated and maintained by a custom algorithm and subject to change after performing file edits. Do not search for these content IDs directly. You will not be able to generate them. - Focus on the lines they identify. + File contents are presented with virtual prefix identifiers to help you target edits accurately. These are generated on-the-fly when reading the file. + + - **Unique Lines (`~~`):** Lines that appear only once in the file are prefixed with `~~`. To target these lines for edits, you can simply reference the exact literal text of the line. + - **Duplicate Lines (e.g., `~“0车加~`):** Lines that appear multiple times are prefixed with an identifier containing an occurrence index and a content hash. You MUST include this exact prefix when targeting these lines to disambiguate which specific instance you want to edit. + + Do not attempt to generate, guess, or calculate these duplicate IDs yourself. Always use the exact line contents and prefixes provided from the most recent file read. **Example File** ``` - GoTXSp::#!/usr/bin/env python3 + ~~#!/usr/bin/env python3 + + ~~def example_process_data(data): + ~~ if not data: + ~“0车加~ return None - ILLPTX::def example_method(): - ItWHVI:: return "example" + ~~ data.clean() - SDROMc::def foo(): - FPEPD:: return "bar" + ~~ if data.is_stale(): + ~引0车加~ return None - HiHyKn::def bar(): - XXTDIR:: return example_method() + " " + foo() + ~~ try: + ~~ data.save() + ~~ except Exception: + ~니0车加~ return None ``` ## Core Workflow diff --git a/cecli/repo.py b/cecli/repo.py index baec1592a99..d0170d762a9 100644 --- a/cecli/repo.py +++ b/cecli/repo.py @@ -17,6 +17,9 @@ git = None ANY_GIT_ERROR = [] +import concurrent.futures +import threading + import pathspec import cecli.prompts.utils.system as prompts @@ -52,6 +55,25 @@ def set_git_env(var_name, value, original_value): del os.environ[var_name] +class CommitInfo: + """ + Data-only container for extracted commit information. + + Wraps the raw git-python commit data into simple Python types + so that callers never receive a reference to the underlying + ``git.Commit`` object (which could otherwise be used outside + of the repository lock, leading to deadlocks). + """ + + def __init__(self, hexsha: str, message: str, parents: tuple[str, ...]): + self.hexsha = hexsha + self.message = message + self.parents = parents + + def __bool__(self): + return True + + class GitRepo: repo = None cecli_ignore_file = None @@ -104,6 +126,7 @@ def __init__( self.workspace_layout = "clone" self.workspace_ignore_specs = {} self.workspace_ignore_ts = {} + self._git_lock = threading.RLock() # Workspace detection and config loading occurs later in __init__ if git_dname: @@ -171,21 +194,21 @@ def __init__( self.cecli_ignore_file = Path(cecli_ignore_file) def init_repo(self): - if not self.repo: - self.repo = git.Repo(self._init_repo_path, odbt=git.GitCmdObjectDB) - self.root = utils.safe_abs_path(self.repo.working_tree_dir) - - if self.is_workspace: - self.root = self.workspace_path - - try: - commit = self.repo.head.commit - return commit - except ANY_GIT_ERROR: - if not self.is_workspace: + with self._git_lock: + if not self.repo: self.repo = git.Repo(self._init_repo_path, odbt=git.GitCmdObjectDB) self.root = utils.safe_abs_path(self.repo.working_tree_dir) + if self.is_workspace: + self.root = self.workspace_path + + try: + self.repo.head.commit # just access to check for errors, discard + except ANY_GIT_ERROR: + if not self.is_workspace: + self.repo = git.Repo(self._init_repo_path, odbt=git.GitCmdObjectDB) + self.root = utils.safe_abs_path(self.repo.working_tree_dir) + def _load_workspace_config(self) -> None: from cecli.helpers.monorepo.config import ( load_workspace_config, @@ -245,8 +268,9 @@ def _detect_workspace_path(self, start_path: str): return None def __del__(self): - if self.repo: - self.repo.close() + with self._git_lock: + if self.repo: + self.repo.close() async def commit(self, fnames=None, context=None, message=None, coder_edits=False, coder=None): """ @@ -321,12 +345,13 @@ async def commit(self, fnames=None, context=None, message=None, coder_edits=Fals if self.is_workspace and getattr(self, "workspace_layout", "clone") == "local": return await self._commit_local_workspace(fnames, context, message, coder_edits, coder) - if not fnames and not self.repo.is_dirty(): - return + with self._git_lock: + if not fnames and not self.repo.is_dirty(): + return - diffs = self.get_diffs(fnames) - if not diffs: - return + diffs = self.get_diffs(fnames) + if not diffs: + return if message: commit_message = message @@ -397,70 +422,72 @@ async def commit(self, fnames=None, context=None, message=None, coder_edits=Fals full_commit_message = commit_message + commit_message_trailer - cmd = ["-m", full_commit_message] - if not self.git_commit_verify: - cmd.append("--no-verify") - if fnames: - fnames = [str(self.abs_root_path(fn)) for fn in fnames] - added_fnames = [] - for fname in fnames: - try: - # Check if file is git-ignored before trying to add - if ( - coder - and hasattr(coder, "add_gitignore_files") - and coder.add_gitignore_files - ): - rel_fname = self.get_rel_fname(fname) - if self.git_ignored_file(rel_fname): - # Skip git-ignored files when add_gitignore_files is enabled - continue - self.repo.git.add(fname) - added_fnames.append(fname) - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to add {fname}: {err}") - if added_fnames: - cmd += ["--"] + added_fnames + with self._git_lock: + cmd = ["-m", full_commit_message] + if not self.git_commit_verify: + cmd.append("--no-verify") + if fnames: + fnames = [str(self.abs_root_path(fn)) for fn in fnames] + added_fnames = [] + for fname in fnames: + try: + # Check if file is git-ignored before trying to add + if ( + coder + and hasattr(coder, "add_gitignore_files") + and coder.add_gitignore_files + ): + rel_fname = self.get_rel_fname(fname) + if self.git_ignored_file(rel_fname): + # Skip git-ignored files when add_gitignore_files is enabled + continue + self.repo.git.add(fname) + added_fnames.append(fname) + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to add {fname}: {err}") + if added_fnames: + cmd += ["--"] + added_fnames + else: + # No files to commit (all were git-ignored or failed to add) + return else: - # No files to commit (all were git-ignored or failed to add) - return - else: - cmd += ["-a"] + cmd += ["-a"] - original_user_name = self.repo.git.config("--get", "user.name") - original_committer_name_env = os.environ.get("GIT_COMMITTER_NAME") - original_author_name_env = os.environ.get("GIT_AUTHOR_NAME") - committer_name = f"{original_user_name} (cecli)" + original_user_name = self.repo.git.config("--get", "user.name") + original_committer_name_env = os.environ.get("GIT_COMMITTER_NAME") + original_author_name_env = os.environ.get("GIT_AUTHOR_NAME") + committer_name = f"{original_user_name} (cecli)" - try: - # Use context managers to handle environment variables - with contextlib.ExitStack() as stack: - if use_attribute_committer: - stack.enter_context( - set_git_env( - "GIT_COMMITTER_NAME", committer_name, original_committer_name_env + try: + # Use context managers to handle environment variables + with contextlib.ExitStack() as stack: + if use_attribute_committer: + stack.enter_context( + set_git_env( + "GIT_COMMITTER_NAME", committer_name, original_committer_name_env + ) + ) + if use_attribute_author: + stack.enter_context( + set_git_env("GIT_AUTHOR_NAME", committer_name, original_author_name_env) ) - ) - if use_attribute_author: - stack.enter_context( - set_git_env("GIT_AUTHOR_NAME", committer_name, original_author_name_env) - ) - # Perform the commit - self.repo.git.commit(cmd) - commit_hash = self.get_head_commit_sha(short=True) - self.io.tool_success(f"Commit {commit_hash} {commit_message}") - return commit_hash, commit_message + # Perform the commit + self.repo.git.commit(cmd) + commit_hash = self.get_head_commit_sha(short=True) + self.io.tool_success(f"Commit {commit_hash} {commit_message}") + return commit_hash, commit_message - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to commit: {err}") - # No return here, implicitly returns None + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to commit: {err}") + # No return here, implicitly returns None def get_rel_repo_dir(self): - try: - return os.path.relpath(self.repo.git_dir, os.getcwd()) - except (ValueError, OSError): - return self.repo.git_dir + with self._git_lock: + try: + return os.path.relpath(self.repo.git_dir, os.getcwd()) + except (ValueError, OSError): + return self.repo.git_dir def get_rel_fname(self, fname): try: @@ -530,126 +557,129 @@ async def get_commit_message(self, diffs, context, user_language=None): def get_diffs(self, fnames=None): # We always want diffs of index and working dir - current_branch_has_commits = False - try: - active_branch = self.repo.active_branch + with self._git_lock: + current_branch_has_commits = False try: - commits = self.repo.iter_commits(active_branch) - current_branch_has_commits = any(commits) - except ANY_GIT_ERROR: + active_branch = self.repo.active_branch + try: + commits = self.repo.iter_commits(active_branch) + current_branch_has_commits = any(commits) + except ANY_GIT_ERROR: + pass + except (TypeError,) + ANY_GIT_ERROR: pass - except (TypeError,) + ANY_GIT_ERROR: - pass - if not fnames: - fnames = [] + if not fnames: + fnames = [] - diffs = "" - for fname in fnames: - if not self.path_in_repo(fname): - diffs += f"Added {fname}\n" + diffs = "" + for fname in fnames: + if not self.path_in_repo(fname): + diffs += f"Added {fname}\n" - try: - if current_branch_has_commits: - args = ["HEAD", "--"] + list(fnames) - diffs += self.repo.git.diff(*args, stdout_as_string=False).decode( + try: + if current_branch_has_commits: + args = ["HEAD", "--"] + list(fnames) + diffs += self.repo.git.diff(*args, stdout_as_string=False).decode( + self.io.encoding, "replace" + ) + return diffs + + wd_args = ["--"] + list(fnames) + index_args = ["--cached"] + wd_args + + diffs += self.repo.git.diff(*index_args, stdout_as_string=False).decode( + self.io.encoding, "replace" + ) + diffs += self.repo.git.diff(*wd_args, stdout_as_string=False).decode( self.io.encoding, "replace" ) + return diffs + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to diff: {err}") - wd_args = ["--"] + list(fnames) - index_args = ["--cached"] + wd_args + def diff_commits(self, pretty, from_commit, to_commit=None): + with self._git_lock: + args = [] + if pretty: + args += ["--color"] + else: + args += ["--color=never"] - diffs += self.repo.git.diff(*index_args, stdout_as_string=False).decode( - self.io.encoding, "replace" - ) - diffs += self.repo.git.diff(*wd_args, stdout_as_string=False).decode( + if to_commit is not None: + args += [from_commit, to_commit] + else: + args += [from_commit] + diffs = self.repo.git.diff(*args, stdout_as_string=False).decode( self.io.encoding, "replace" ) return diffs - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to diff: {err}") - - def diff_commits(self, pretty, from_commit, to_commit=None): - args = [] - if pretty: - args += ["--color"] - else: - args += ["--color=never"] - - if to_commit is not None: - args += [from_commit, to_commit] - else: - args += [from_commit] - diffs = self.repo.git.diff(*args, stdout_as_string=False).decode( - self.io.encoding, "replace" - ) - - return diffs def get_tracked_files(self): - if not self.repo: - return [] - - self.init_repo() + with self._git_lock: + if not self.repo: + return [] - try: - commit = self.repo.head.commit - except ValueError: - commit = None - except ANY_GIT_ERROR as err: - self.git_repo_error = err - self.io.tool_error(f"Unable to list files in git repo: {err}") - self.io.tool_output("Is your git repo corrupted?") - return [] + self.init_repo() - files = set() - if commit: - if self._tree_cache is not None and self._tree_cache[0] == commit.hexsha: - files = self._tree_cache[1] - else: - try: - iterator = commit.tree.traverse() - blob = None # Initialize blob - while True: - try: - blob = next(iterator) - if blob.type == "blob": # blob is a file - # Use sys.intern() to deduplicate path strings in memory - files.add(sys.intern(blob.path)) - except IndexError: - # Handle potential index error during tree traversal - # without relying on potentially unassigned 'blob' - self.io.tool_warning( - "GitRepo: Index error encountered while reading git tree object." - " Skipping." - ) - continue - except StopIteration: - break - except ANY_GIT_ERROR as err: - self.git_repo_error = err - self.io.tool_error(f"Unable to list files in git repo: {err}") - self.io.tool_output("Is your git repo corrupted?") - return [] - files = set(self.normalize_path(path) for path in files) - # Use single-entry cache (not per-commit dict) to limit memory growth - # Store only the SHA string (not the Commit object) to avoid retaining - # the entire git object graph (tree, blobs, parent commits, etc.) - self._tree_cache = (commit.hexsha, files) - - # Add staged files - index = self.repo.index - try: - staged_files = [path for path, _ in index.entries.keys()] - files.update(self.normalize_path(path) for path in staged_files) - except ANY_GIT_ERROR as err: - self.io.tool_error(f"Unable to read staged files: {err}") + try: + commit = self.repo.head.commit + except ValueError: + commit = None + except ANY_GIT_ERROR as err: + self.git_repo_error = err + self.io.tool_error(f"Unable to list files in git repo: {err}") + self.io.tool_output("Is your git repo corrupted?") + return [] + + files = set() + if commit: + if self._tree_cache is not None and self._tree_cache[0] == commit.hexsha: + files = self._tree_cache[1] + else: + try: + iterator = commit.tree.traverse() + blob = None # Initialize blob + while True: + try: + blob = next(iterator) + if blob.type == "blob": # blob is a file + # Use sys.intern() to deduplicate path strings in memory + files.add(sys.intern(blob.path)) + except IndexError: + # Handle potential index error during tree traversal + # without relying on potentially unassigned 'blob' + self.io.tool_warning( + "GitRepo: Index error encountered while reading git tree object." + " Skipping." + ) + continue + except StopIteration: + break + except ANY_GIT_ERROR as err: + self.git_repo_error = err + self.io.tool_error(f"Unable to list files in git repo: {err}") + self.io.tool_output("Is your git repo corrupted?") + return [] + files = set(self.normalize_path(path) for path in files) + # Use single-entry cache (not per-commit dict) to limit memory growth + # Store only the SHA string (not the Commit object) to avoid retaining + # the entire git object graph (tree, blobs, parent commits, etc.) + self._tree_cache = (commit.hexsha, files) + + # Add staged files + index = self.repo.index + try: + staged_files = [path for path, _ in index.entries.keys()] + files.update(self.normalize_path(path) for path in staged_files) + except ANY_GIT_ERROR as err: + self.io.tool_error(f"Unable to read staged files: {err}") - res = [fname for fname in files if not self.ignored_file(fname)] + res = [fname for fname in files if not self.ignored_file(fname)] - return res + return res async def _commit_local_workspace( self, fnames=None, context=None, message=None, coder_edits=False, coder=None @@ -1025,16 +1055,17 @@ def get_non_ignored_files_from_root(self): if not self.cecli_ignore_spec: return [] - try: - all_files = self.repo.git.ls_files( - "--others", "--cached", f"--exclude-from={str(self.cecli_ignore_file)}" - ).splitlines() - - return [f for f in all_files if not self.ignored_file(f)] - except Exception as e: - # Fall back to empty set if there's an error - self.io.tool_warning(f"Error getting ignored files from root: {e}") - return [] + with self._git_lock: + try: + all_files = self.repo.git.ls_files( + "--others", "--cached", f"--exclude-from={str(self.cecli_ignore_file)}" + ).splitlines() + + return [f for f in all_files if not self.ignored_file(f)] + except Exception as e: + # Fall back to empty set if there's an error + self.io.tool_warning(f"Error getting ignored files from root: {e}") + return [] def get_repo_files(self) -> list[str]: """ @@ -1070,19 +1101,20 @@ def get_cache_key(self) -> str: Returns: SHA-256 hex digest, or empty string if repo is unavailable """ - import hashlib + with self._git_lock: + import hashlib - if not self.repo: - return "" + if not self.repo: + return "" - sha = self.get_head_commit_sha() or "" - try: - staged = [item.a_path for item in self.repo.index.diff("HEAD")] - except ANY_GIT_ERROR: - staged = [] + sha = self.get_head_commit_sha() or "" + try: + staged = [item.a_path for item in self.repo.index.diff("HEAD")] + except ANY_GIT_ERROR: + staged = [] - combined = sha + "|" + "".join(sorted(staged)) - return hashlib.sha256(combined.encode()).hexdigest() + combined = sha + "|" + "".join(sorted(staged)) + return hashlib.sha256(combined.encode()).hexdigest() def path_in_repo(self, path): if not self.repo: @@ -1115,40 +1147,240 @@ def get_dirty_files(self): Returns a list of all files which are dirty (not committed), either staged or in the working directory. """ - dirty_files = set() + with self._git_lock: + dirty_files = set() - # Get staged files - staged_files = self.repo.git.diff("--name-only", "--cached").splitlines() - dirty_files.update(staged_files) + # Get staged files + staged_files = self.repo.git.diff("--name-only", "--cached").splitlines() + dirty_files.update(staged_files) - # Get unstaged files - unstaged_files = self.repo.git.diff("--name-only").splitlines() - dirty_files.update(unstaged_files) + # Get unstaged files + unstaged_files = self.repo.git.diff("--name-only").splitlines() + dirty_files.update(unstaged_files) - return list(dirty_files) + return list(dirty_files) def is_dirty(self, path=None): - if path and not self.path_in_repo(path): - return True + with self._git_lock: + if path and not self.path_in_repo(path): + return True - return self.repo.is_dirty(path=path) + return self.repo.is_dirty(path=path) def get_head_commit(self): + with self._git_lock: + try: + commit = self.repo.head.commit + return CommitInfo( + hexsha=commit.hexsha, + message=commit.message, + parents=tuple(p.hexsha for p in commit.parents), + ) + except (ValueError,) + ANY_GIT_ERROR: + return None + + def get_head_commit_sha(self, short=False): + with self._git_lock: + try: + commit = self.repo.head.commit + except (ValueError,) + ANY_GIT_ERROR: + return + if not commit: + return + if short: + return commit.hexsha[:7] + return commit.hexsha + + def get_head_commit_message(self, default=None): + with self._git_lock: + try: + commit = self.repo.head.commit + except (ValueError,) + ANY_GIT_ERROR: + return default + if not commit: + return default + return commit.message + + +class _GitObjectProxy: + """Routes all operations on a git sub-object through a shared single-thread executor. + + Wraps objects like ``git.Repo.git``, ``git.Repo.head``, ``git.Repo.index`` + so that every method call and attribute access is dispatched on the + executor thread, preventing deadlocks when the underlying ``git.Repo`` + is accessed from multiple asyncio workers. + + Iterators returned by git-python methods are eagerly materialised into + lists so that iteration happens on the executor thread, not the caller's. + """ + + def __init__(self, obj, executor): + object.__setattr__(self, "_obj", obj) + object.__setattr__(self, "_executor", executor) + + def __getattr__(self, name): + attr = getattr(self._obj, name) + + # Recursively wrap git-python objects (check FIRST since git.cmd.Git + # is callable via __call__, but we want chained access like + # ``repo.git.status(...)`` to stay on the executor thread). + cls = type(attr) + if cls.__module__ and cls.__module__.startswith("git"): + return _GitObjectProxy(attr, self._executor) + + if callable(attr): + + def proxy_fn(*args, **kwargs): + future = self._executor.submit(attr, *args, **kwargs) + result = future.result() + # Materialise iterators eagerly in the executor thread so + # iteration never touches git objects from the caller's thread. + if hasattr(result, "__iter__") and not isinstance(result, (str, bytes, dict)): + try: + return list(result) + except TypeError: + pass + return result + + return proxy_fn + + return attr + + +class GitRepoProxy: + """Thread-safe proxy for ``GitRepo`` that serialises all git operations + through a single dedicated thread. + + Prevents deadlocks by ensuring only one thread ever touches the + underlying ``git.Repo`` at any time. Follows the same delegation + pattern as ``IOProxy``: + + * Sync methods that access ``self.repo`` are explicitly dispatched + to the executor and block on the result. + * Non-git attributes and methods are forwarded transparently via + ``__getattr__`` / ``__setattr__``. + * Access to ``.repo`` (the raw ``git.Repo``) returns a + ``_GitObjectProxy`` that routes every attribute and call through + the same executor, so even callers that reach through to the + underlying git object stay thread-safe. + + Usage:: + + repo = GitRepoProxy(GitRepo(io, fnames, git_dname)) + files = repo.get_tracked_files() # runs on executor thread + sha = repo.get_head_commit_sha() # runs on executor thread + """ + + def __init__(self, target): + self._target = target + self._executor = concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix="git-repo" + ) + + # ------------------------------------------------------------------ + # Sync methods that access self.repo (the git.Repo object) + # ------------------------------------------------------------------ + + def init_repo(self): + return self._executor.submit(self._target.init_repo).result() + + @classmethod + def unwrap(cls, repo): + """Unwrap a ``GitRepoProxy`` to get the underlying ``GitRepo``. + + If *repo* is already a ``GitRepo`` (not a proxy) it is returned + unchanged. This mirrors ``IOProxy.unwrap()`` and prevents nested + proxy chains during coder switching (``SwitchCoderSignal``). + + Usage:: + + raw = GitRepoProxy.unwrap(maybe_proxied_repo) + """ + return repo._target if isinstance(repo, cls) else repo + + def __del__(self): try: - return self.repo.head.commit - except (ValueError,) + ANY_GIT_ERROR: - return None + if hasattr(self, "_target") and self._target is not None: + self._executor.submit(self._target.__del__).result(timeout=5) + except Exception: + pass + + def get_rel_repo_dir(self): + return self._executor.submit(self._target.get_rel_repo_dir).result() + + def get_diffs(self, fnames=None): + return self._executor.submit(self._target.get_diffs, fnames).result() + + def diff_commits(self, pretty, from_commit, to_commit=None): + return self._executor.submit( + self._target.diff_commits, pretty, from_commit, to_commit + ).result() + + def get_tracked_files(self): + return self._executor.submit(self._target.get_tracked_files).result() + + def path_in_repo(self, path): + return self._executor.submit(self._target.path_in_repo, path).result() + + def get_dirty_files(self): + return self._executor.submit(self._target.get_dirty_files).result() + + def is_dirty(self, path=None): + return self._executor.submit(self._target.is_dirty, path).result() + + def get_head_commit(self): + return self._executor.submit(self._target.get_head_commit).result() def get_head_commit_sha(self, short=False): - commit = self.get_head_commit() - if not commit: - return - if short: - return commit.hexsha[:7] - return commit.hexsha + return self._executor.submit(self._target.get_head_commit_sha, short).result() def get_head_commit_message(self, default=None): - commit = self.get_head_commit() - if not commit: - return default - return commit.message + return self._executor.submit(self._target.get_head_commit_message, default).result() + + def get_cache_key(self): + return self._executor.submit(self._target.get_cache_key).result() + + def git_ignored_file(self, path): + return self._executor.submit(self._target.git_ignored_file, path).result() + + def get_non_ignored_files_from_root(self): + return self._executor.submit(self._target.get_non_ignored_files_from_root).result() + + def get_repo_files(self): + return self._executor.submit(self._target.get_repo_files).result() + + def get_workspace_files(self): + return self._executor.submit(self._target.get_workspace_files).result() + + # ------------------------------------------------------------------ + # Async methods – called from the main asyncio task only, and + # already protected by ``_git_lock`` on the target; we do *not* + # proxy them through the executor because they mix git operations + # with LLM calls (``get_commit_message``) that must stay async. + # ------------------------------------------------------------------ + + # commit() and _commit_local_workspace() are forwarded via __getattr__ + + # ------------------------------------------------------------------ + # Access to the raw ``git.Repo`` – return a sub-proxy that routes + # every call through the same executor, so callers that reach + # through to the underlying git object stay thread-safe. + # ------------------------------------------------------------------ + + @property + def repo(self): + raw = self._target.repo + return _GitObjectProxy(raw, self._executor) if raw is not None else None + + # ------------------------------------------------------------------ + # Non-git attributes – transparent forwarding + # ------------------------------------------------------------------ + + def __getattr__(self, name): + return getattr(self._target, name) + + def __setattr__(self, name, value): + if name.startswith("_") or name in ("_target", "_executor"): + super().__setattr__(name, value) + else: + setattr(self._target, name, value) diff --git a/cecli/signals.py b/cecli/signals.py new file mode 100644 index 00000000000..a122f4ebb7c --- /dev/null +++ b/cecli/signals.py @@ -0,0 +1,49 @@ +""" +Control-flow signal exceptions for cecli. + +These signals are used for non-error control flow (like switching coders +or reloading the program). They inherit from BaseException to avoid being +caught by generic `except Exception` handlers. +""" + + +class SwitchCoderSignal(BaseException): + """ + Signal to switch the current Coder instance to a new configuration. + + This is NOT an error - it's a control flow signal used to propagate + coder switching requests up through the async call stack. It carries + the kwargs needed to create a new Coder instance. + + Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) + to avoid being caught by generic `except Exception` handlers, making the + non-error nature of this signal explicit. + + Attributes: + kwargs: Configuration dict passed to Coder.create() for the new instance + placeholder: Optional placeholder text for the input prompt + """ + + def __init__(self, placeholder=None, **kwargs): + self.kwargs = kwargs + self.placeholder = placeholder + super().__init__() + + +class ReloadProgramSignal(BaseException): + """ + Signal to reload the entire program configuration. + + This is NOT an error - it's a control flow signal used to trigger + a full program reload, re-parsing config files and re-initializing + all components. Useful for hot-reloading when configuration files + change. + + Note: Inherits from BaseException (like KeyboardInterrupt and SystemExit) + to avoid being caught by generic `except Exception` handlers. + """ + + def __init__(self, message="Reloading program configuration...", **kwargs): + self.kwargs = kwargs + self.message = message + super().__init__(self.message) diff --git a/cecli/tools/__init__.py b/cecli/tools/__init__.py index a9f8be77aa9..62ecc0cf1c9 100644 --- a/cecli/tools/__init__.py +++ b/cecli/tools/__init__.py @@ -1,11 +1,9 @@ # flake8: noqa: F401 # Import tool modules into the cecli.tools namespace -# Import all tool modules from . import ( _yield, command, - command_interactive, delegate, edit_file, explore_code, @@ -17,6 +15,7 @@ git_status, grep, ls, + orchestrate, read_file, resource_manager, thinking, @@ -27,7 +26,6 @@ # List of all available tool modules for dynamic discovery TOOL_MODULES = [ command, - command_interactive, delegate, edit_file, explore_code, @@ -40,6 +38,7 @@ git_status, grep, ls, + orchestrate, read_file, resource_manager, thinking, diff --git a/cecli/tools/_yield.py b/cecli/tools/_yield.py index 57d5860b200..e5f244a5a16 100644 --- a/cecli/tools/_yield.py +++ b/cecli/tools/_yield.py @@ -5,6 +5,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations logger = logging.getLogger(__name__) @@ -49,6 +50,8 @@ async def execute(cls, coder, **kwargs): cls.clear_invocation_cache() + response = ToolResponse(cls.NORM_NAME) + if coder: # Check for active child sub-agents and await their tasks before finishing try: @@ -88,10 +91,11 @@ async def execute(cls, coder, **kwargs): await t except (asyncio.CancelledError, Exception): pass - return ( + response.append_result( "Yield interrupted while waiting for sub-agents. " "Sub-agent outputs above may be incomplete." ) + return response # Retrieve exceptions from completed sub-agent tasks so they # are not silently lost. @@ -155,10 +159,11 @@ async def execute(cls, coder, **kwargs): await agent_service.reap_all_finished_agents(parent=coder) # Don't mark as finished — the coder should review sub-agent # outputs and decide how to proceed - return ( + response.append_result( "Sub-agents have finished. Please examine their output above " "in order to decide how you will proceed." ) + return response except Exception as e: logger.warning("Error awaiting child sub-agents before yield: %s", e) @@ -190,11 +195,14 @@ async def execute(cls, coder, **kwargs): coder.files_edited_by_tools = set() if summary: - return f"Yielded. Summary: {summary}" - return "Yielded." + response.append_result(f"Yielded. Summary: {summary}") + return response + response.append_result("Yielded.") + return response # coder.io.tool_Error("Error: Could not mark agent task as finished") - return "Error: Could not yield control" + response.append_error("Could not yield control") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/command.py b/cecli/tools/command.py index 8fe49a4d4d6..3a70a799277 100644 --- a/cecli/tools/command.py +++ b/cecli/tools/command.py @@ -15,10 +15,11 @@ import xxhash from cecli.helpers.background_commands import BackgroundCommandManager -from cecli.run_cmd import run_cmd_subprocess +from cecli.run_cmd import run_cmd, run_cmd_subprocess from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -77,6 +78,23 @@ class Tool(BaseTool): ), "default": False, }, + "user_input_required": { + "type": "boolean", + "description": ( + "When True, runs the command interactively using a " + "pseudo-terminal (PTY), allowing the user to provide " + "inputs like passwords or navigate terminal interfaces. " + "Handles TUI suspension automatically." + ), + "default": False, + }, + "timeout": { + "type": "integer", + "description": ( + "Timeout in seconds for command execution. " "Default is 30 seconds." + ), + "default": 30, + }, }, "required": [], }, @@ -102,43 +120,62 @@ async def execute( action=None, stdin=None, pty=False, + user_input_required=False, + timeout=0, **kwargs, ): """ Execute a shell command or interact with background processes. For new commands: provide 'command' (and optionally 'background', 'stdin', 'pty'). + When 'user_input_required' is True, runs the command interactively using a + pseudo-terminal (PTY), allowing the user to provide inputs like passwords + or navigate terminal interfaces. For background interactions: provide 'background_key' + 'action' (stdin/stop). - Commands run with timeout based on agent_config['command_timeout'] (default: 30 seconds). + Commands run with timeout from agent_config['command_timeout'] (default: 30 seconds), """ + response = ToolResponse( + "command", + result_type=cls.RESULT_TYPE if hasattr(cls, "RESULT_TYPE") else "str", + ) # Handle interactions with an existing background command if background_key: if action == "stdin": if not stdin: - return "Error: 'stdin' is required when action='stdin'." + response.append_error("'stdin' is required when action='stdin'.") + return response cls.clear_invocation_cache() success = BackgroundCommandManager.send_command_input(background_key, stdin) if success: - return f"Sent input to background command {background_key}: {stdin}" + response.append_result( + f"Sent input to background command {background_key}: {stdin}" + ) + return response else: - return f"Error: Background command {background_key} not found or not running." + response.append_error( + f"Background command {background_key} not found or not running." + ) + return response elif action == "stop": return await cls._stop_background_command(coder, background_key) else: - return f"Error: Unknown action '{action}'. " "Use one of: stdin, stop." + response.append_error(f"Unknown action '{action}'. Use one of: stdin, stop.") + return response if not command: - return "Error: 'command' must be provided." + response.append_error("'command' must be provided.") + return response # Check for implicit background (trailing & on Linux) if ".cecli/agents" in command: - return ( - "Error: Do not attempt to access internal files with " + response.append_error( + "Do not attempt to access internal files with " "standard cli tools. Please use the tools you have been provided." ) + return response if not background and command.strip().endswith("&"): background = True @@ -147,16 +184,24 @@ async def execute( # Get user confirmation confirmed = await cls._get_confirmation(coder, command, background) if not confirmed: - return "Command execution skipped by user." + response.append_result("Command execution skipped by user.") + return response command = coder.format_command_with_prefix(command) - # Determine timeout from agent_config (default: 30 seconds) - timeout = 0 + # Determine timeout from agent_config (default: 30 seconds) as fallback + config_timeout = 0 if hasattr(coder, "agent_config"): - timeout = coder.agent_config.get("command_timeout", 30) - - if background: + config_timeout = coder.agent_config.get("command_timeout", 30) + # Use LLM-specified timeout if provided, otherwise fallback to config + if timeout == 0: + timeout = config_timeout + # Clamp LLM timeout between config_timeout (minimum) and max(300, config_timeout) (maximum) + timeout = max(config_timeout, min(timeout, max(300, config_timeout))) + + if user_input_required: + return await cls._execute_interactive(coder, command) + elif background: return await cls._execute_background(coder, command, use_pty=pty, stdin=stdin) elif timeout > 0: return await cls._execute_with_timeout(coder, command, timeout, use_pty=pty) @@ -240,11 +285,13 @@ async def _execute_background(cls, coder, command_string, use_pty=None, stdin=No if stdin: BackgroundCommandManager.send_command_input(command_key, stdin) - return ( + response = ToolResponse(cls.NORM_NAME) + response.append_result( f"Background command started: {command_string}\n" f"Command key: {command_key}\n" "Output will be injected into chat stream." ) + return response @classmethod async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=None): @@ -261,6 +308,8 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non from cecli.helpers.background_commands import CircularBuffer + response = ToolResponse(cls.NORM_NAME) + coder.io.tool_output( f"⛭ Executing shell command with {timeout}s timeout.", type="tool-result" ) @@ -342,7 +391,8 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non except subprocess.TimeoutExpired: process.kill() BackgroundCommandManager.stop_background_command(command_key) - return "Command execution interrupted by user." + response.append_result("Command execution interrupted by user.") + return response # Check if process has completed exit_code = process.poll() @@ -353,7 +403,7 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non # Format output output_content = output or "" # Tokens are roughly 3-4 characters - output_limit = coder.large_file_token_threshold * 3.5 + output_limit = int(coder.large_file_token_threshold * 3.5) if coder.context_management_enabled and len(output_content) > output_limit * 1.25: # Save full output to paginated files instead of truncating @@ -387,15 +437,17 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non coder.io.tool_output(output_content, type="tool-result") if exit_code == 0: - return ( + response.append_result( f"Shell command completed within {timeout}s timeout (exit code 0)." f" Output:\n{output_content}" ) + return response else: - return ( + response.append_result( f"Shell command completed within {timeout}s timeout with exit code" f" {exit_code}. Output:\n{output_content}" ) + return response # Check if timeout has expired elapsed = time.time() - start_time @@ -409,11 +461,12 @@ async def _execute_with_timeout(cls, coder, command_string, timeout, use_pty=Non # Get any output captured so far current_output = buffer.get_all(clear=False) - return ( + response.append_result( f"Command exceeded {timeout}s timeout and is continuing in background.\n" f"Command key: {command_key}\n" f"Output captured so far:\n{current_output}\n" ) + return response # Wait a bit before checking again await asyncio.sleep(1) @@ -423,6 +476,7 @@ async def _execute_foreground(cls, coder, command_string): """ Execute command in foreground (blocking). """ + response = ToolResponse(cls.NORM_NAME) should_print = True tui = None if coder.tui and coder.tui(): @@ -470,9 +524,85 @@ async def _execute_foreground(cls, coder, command_string): coder.io.tool_output(output_content, type="tool-result") if exit_status == 0: - return f"Shell command executed successfully (exit code 0). Output:\n{output_content}" + response.append_result( + f"Shell command executed successfully (exit code 0). Output:\n{output_content}" + ) + return response else: - return f"Shell command failed with exit code {exit_status}. Output:\n{output_content}" + response.append_result( + f"Shell command failed with exit code {exit_status}. Output:\n{output_content}" + ) + return response + + @classmethod + async def _execute_interactive(cls, coder, command_string): + """ + Execute command interactively, allowing the user to provide inputs + like passwords or navigate terminal interfaces. + Handles TUI suspension automatically. + """ + import asyncio + + response = ToolResponse(cls.NORM_NAME) + + coder.io.tool_output( + f"\u26ed Starting interactive shell command: {command_string}", type="tool-result" + ) + + tui = coder.tui() if coder.tui else None + + def _run_interactive(): + return run_cmd( + command_string, + verbose=coder.verbose, + error_print=coder.io.tool_error, + cwd=coder.root, + should_print=True, + ) + + if tui: + coder.io.tool_output( + ">>> Suspending TUI for interactive command <<<", type="tool-result" + ) + exit_status, combined_output = tui.run_obstructive(_run_interactive) + else: + coder.io.tool_output( + ">>> You may need to interact with the command below <<<", type="tool-result" + ) + coder.io.tool_output(" \n") + await coder.io.stop_input_task() + await asyncio.sleep(1) + exit_status, combined_output = _run_interactive() + await asyncio.sleep(1) + coder.io.tool_output(" \n", type="tool-result") + coder.io.tool_output(" \n", type="tool-result") + + coder.io.tool_output(">>> Interactive command finished <<<", type="tool-result") + + # Format the output for the result message, include more content + output_content = combined_output or "" + output_limit = coder.large_file_token_threshold + if coder.context_management_enabled and len(output_content) > output_limit: + output_content = ( + output_content[:output_limit] + + f"\n... (output truncated at {output_limit} characters, based on" + " large_file_token_threshold)" + ) + + cls.clear_invocation_cache() + + if exit_status == 0: + response.append_result( + "Interactive command finished successfully (exit code 0)." + f" Output:\n{output_content}" + ) + return response + else: + response.append_result( + f"Interactive command finished with exit code {exit_status}." + f" Output:\n{output_content}" + ) + return response @classmethod async def _stop_background_command(cls, coder, command_key): @@ -482,19 +612,25 @@ async def _stop_background_command(cls, coder, command_key): success, output, exit_code = BackgroundCommandManager.stop_background_command(command_key) if success: - return ( + response = ToolResponse(cls.NORM_NAME) + response.append_result( f"Background command stopped: {command_key}\n" f"Exit code: {exit_code}\n" f"Final output:\n{output}" ) + return response else: - return output # Error message from manager + response = ToolResponse(cls.NORM_NAME) + response.append_result(output) + return response @classmethod async def _handle_errors(cls, coder, command_string, e): """Handle errors during command execution.""" coder.io.tool_error(f"Error executing shell command: {str(e)}") - return f"Error executing command: {str(e)}" + response = ToolResponse(cls.NORM_NAME) + response.append_error(f"Error executing command: {str(e)}") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): @@ -518,6 +654,10 @@ def format_output(cls, coder, mcp_server, tool_response): action = params.get("action") stdin = params.get("stdin") pty = params.get("pty", False) + timeout = params.get("timeout", 30) + user_input_required = params.get("user_input_required", False) + + coder.io.tool_output("") coder.io.tool_output("") @@ -529,6 +669,10 @@ def format_output(cls, coder, mcp_server, tool_response): extras.append(f"action={action}") if pty: extras.append("pty=True") + if timeout != 30: + extras.append(f"timeout={timeout}s") + if user_input_required: + extras.append("user_input_required=True") if extras: coder.io.tool_output(f"{color_start}Options:{color_end} {', '.join(extras)}") diff --git a/cecli/tools/command_interactive.py b/cecli/tools/command_interactive.py deleted file mode 100644 index 1a24a66093d..00000000000 --- a/cecli/tools/command_interactive.py +++ /dev/null @@ -1,174 +0,0 @@ -# Import necessary functions -import asyncio -import fnmatch - -import xxhash - -from cecli.run_cmd import run_cmd -from cecli.tools.utils.base_tool import BaseTool - - -class Tool(BaseTool): - NORM_NAME = "commandinteractive" - TRACK_INVOCATIONS = False - ALLOWED_SESSION_COMMANDS = {} - SCHEMA = { - "type": "function", - "function": { - "name": "CommandInteractive", - "description": ( - "Execute a shell command interactively." - " Useful when you need the user to provide inputs like passwords" - " or navigating terminal interfaces." - ), - "parameters": { - "type": "object", - "properties": { - "command_string": { - "type": "string", - "description": "The interactive shell command to execute.", - }, - }, - "required": ["command_string"], - }, - }, - } - - @staticmethod - def _is_command_allowed(coder, command_string): - """Check if command matches any allowed_commands patterns.""" - if hasattr(coder, "agent_config"): - allowed_commands = coder.agent_config.get("allowed_commands", []) - if allowed_commands: - for pattern in allowed_commands: - if fnmatch.fnmatch(command_string, pattern): - return True - return False - - @staticmethod - def _hash_command(command): - """Compute an xxhash of the full command text for session tracking.""" - if not command: - return command - - return xxhash.xxh64(command).hexdigest() - - @classmethod - async def _get_confirmation(cls, coder, command_string): - """Get user confirmation for command execution.""" - # Hash command for dict key lookup - command_hash = cls._hash_command(command_string) - - # Check if command is already handled for this session - if command_hash in cls.ALLOWED_SESSION_COMMANDS: - if cls.ALLOWED_SESSION_COMMANDS[command_hash]: - return True # Previously approved for session - # Previously declined - skip session question, continue to normal confirmation - - if coder.skip_cli_confirmations: - return True - - # Check if command matches any allowed_commands patterns - if cls._is_command_allowed(coder, command_string): - return True - - formatted_command = coder.format_command_with_prefix(command_string) - - confirmed = await coder.io.confirm_ask( - "Allow execution of this command?", - subject=formatted_command, - explicit_yes_required=True, - allow_never=True, - group_response="Command Interactive Tool", - ) - - if not confirmed: - return False - - # Ask if user wants to allow for the entire session (only once per command) - if command_hash not in cls.ALLOWED_SESSION_COMMANDS: - session_allowed = await coder.io.confirm_ask( - "Allow this command for the rest of the session?", - subject=formatted_command, - ) - cls.ALLOWED_SESSION_COMMANDS[command_hash] = session_allowed - - return True - - @classmethod - async def execute(cls, coder, command_string, **kwargs): - """ - Execute an interactive shell command using run_cmd (which uses pexpect/PTY). - """ - try: - confirmed = await cls._get_confirmation(coder, command_string) - if not confirmed: - return "Shell command execution skipped by user." - - command_string = coder.format_command_with_prefix(command_string) - - coder.io.tool_output( - f"⛭ Starting interactive shell command: {command_string}", type="tool-result" - ) - - tui = coder.tui() if coder.tui else None - - def _run_interactive(): - return run_cmd( - command_string, - verbose=coder.verbose, - error_print=coder.io.tool_error, - cwd=coder.root, - should_print=True, - ) - - if tui: - coder.io.tool_output( - ">>> Suspending TUI for interactive command <<<", type="tool-result" - ) - exit_status, combined_output = tui.run_obstructive(_run_interactive) - else: - coder.io.tool_output( - ">>> You may need to interact with the command below <<<", type="tool-result" - ) - coder.io.tool_output(" \n") - await coder.io.stop_input_task() - await asyncio.sleep(1) - exit_status, combined_output = _run_interactive() - await asyncio.sleep(1) - coder.io.tool_output(" \n", type="tool-result") - coder.io.tool_output(" \n", type="tool-result") - - coder.io.tool_output(">>> Interactive command finished <<<", type="tool-result") - - # Format the output for the result message, include more content - output_content = combined_output or "" - output_limit = coder.large_file_token_threshold - if coder.context_management_enabled and len(output_content) > output_limit: - output_content = ( - output_content[:output_limit] - + f"\n... (output truncated at {output_limit} characters, based on" - " large_file_token_threshold)" - ) - - cls.clear_invocation_cache() - - if exit_status == 0: - return ( - "Interactive command finished successfully (exit code 0)." - f" Output:\n{output_content}" - ) - else: - return ( - f"Interactive command finished with exit code {exit_status}." - f" Output:\n{output_content}" - ) - - except Exception as e: - coder.io.tool_error( - f"Error executing interactive shell command '{command_string}': {str(e)}" - ) - # Optionally include traceback for debugging if verbose - # if coder.verbose: - # coder.io.tool_error(traceback.format_exc()) - return f"Error executing interactive command: {str(e)}" diff --git a/cecli/tools/delegate.py b/cecli/tools/delegate.py index d4b55c80a58..bd27fef5901 100644 --- a/cecli/tools/delegate.py +++ b/cecli/tools/delegate.py @@ -5,11 +5,13 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "delegate" + RESULT_TYPE = "list" TRACK_INVOCATIONS = True VALIDATIONS = { "delegations": ["coerce_list"], @@ -39,6 +41,14 @@ class Tool(BaseTool): "type": "string", "description": "Task description to give the sub-agent.", }, + "async": { + "type": "boolean", + "default": True, + "description": ( + "If true (default), delegate asynchronously (fire-and-forget)." + " If false, wait for the result." + ), + }, }, "required": ["name", "prompt"], }, @@ -53,51 +63,104 @@ class Tool(BaseTool): @classmethod async def execute(cls, coder, **kwargs): """Delegate one or more sub-agents to work on sub-tasks in parallel.""" + + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) delegations = kwargs.get("delegations", []) if not delegations or not isinstance(delegations, list): - return "Error: 'delegations' parameter must be a non-empty array of {name, prompt} objects." + response.append_error( + "'delegations' parameter must be a non-empty array of {name, prompt, async} objects." + ) + return response # Validate each delegation item has the required fields for i, d in enumerate(delegations): if not isinstance(d, dict): - return f"Error: delegations[{i}] is not an object." + response.append_error(f"delegations[{i}] is not an object.") + return response if "name" not in d or not d["name"]: - return f"Error: delegations[{i}] is missing a 'name'." + response.append_error(f"delegations[{i}] is missing a 'name'.") + return response if "prompt" not in d or not d["prompt"]: - return f"Error: delegations[{i}] is missing a 'prompt'." + response.append_error(f"delegations[{i}] is missing a 'prompt'.") + return response from cecli.helpers.agents.service import AgentService agent_service = AgentService.get_instance(coder) - async def _spawn_one(name: str, prompt: str) -> tuple[str, str]: - """Spawn a single sub-agent and return (name, uuid_or_error).""" + # Separate async (fire-and-forget) and sync (blocking) delegations + async_delegations = [(d["name"], d["prompt"]) for d in delegations if d.get("async", True)] + sync_delegations = [ + (d["name"], d["prompt"]) for d in delegations if not d.get("async", True) + ] + + async def _spawn_one(name: str, prompt: str) -> tuple: + """Spawn a single sub-agent (fire-and-forget). Returns (name, uuid_or_error, error).""" try: new_coder, info = await agent_service.spawn(name, prompt, parent=coder) - return name, info.coder.uuid + return name, info.coder.uuid, None except Exception as e: - return name, f"failed: {e}" - - # Dispatch all delegations in parallel (spawn is fire-and-forget, but - # _create_sub_agent_coder is async so we gather for concurrency) - tasks = [_spawn_one(d["name"], d["prompt"]) for d in delegations] - raw_results = await asyncio.gather(*tasks) + return name, None, f"failed: {e}" - started_agents: list[tuple[str, str]] = list(raw_results) - - # Build a consolidated report - lines = [] - for name, result in started_agents: - if result.startswith("failed:"): - lines.append(f"✗ **{name}**: {result}") - else: - lines.append(f"✓ **{name}** agent started with id `{result}`") - - n_total = len(started_agents) - n_ok = sum(1 for _, r in started_agents if not r.startswith("failed:")) - combined = "\n".join(lines) - return f"📋 Delegation results ({n_ok}/{n_total} dispatched):\n{combined}" + async def _invoke_one(name: str, prompt: str) -> tuple: + """Invoke a single sub-agent (blocking). Returns (name, summary_or_error, error).""" + try: + summary = await agent_service.invoke(name, prompt, parent=coder) + return name, summary or "(no summary)", None + except Exception as e: + return name, None, f"failed: {e}" + + # Process async delegations (fire-and-forget spawn) + async_results = [] + if async_delegations: + tasks = [_spawn_one(n, p) for n, p in async_delegations] + async_results = list(await asyncio.gather(*tasks)) + + # Process sync delegations (blocking invoke) + sync_results = [] + if sync_delegations: + tasks = [_invoke_one(n, p) for n, p in sync_delegations] + sync_results = list(await asyncio.gather(*tasks)) + + # Build response + if not sync_delegations: + # All async: single combined result (current behavior) + lines = [] + for name, result, error in async_results: + if error: + lines.append(f"✗ **{name}**: {error}") + else: + lines.append(f"✓ **{name}** agent started with id `{result}`") + + n_total = len(async_results) + n_ok = sum(1 for _, _, e in async_results if not e) + combined = "\n".join(lines) + response.append_result( + f"📋 Delegation results ({n_ok}/{n_total} dispatched):\n{combined}" + ) + else: + # Mixed or all sync: individual results per non-async agent + if async_delegations: + lines = [] + for name, result, error in async_results: + if error: + lines.append(f"✗ **{name}**: {error}") + else: + lines.append(f"✓ **{name}** agent started with id `{result}`") + combined = "\n".join(lines) + n_ok = sum(1 for _, _, e in async_results if not e) + response.append_result( + f"📋 Async delegation results ({n_ok}/{len(async_results)} dispatched):\n{combined}" + ) + + for name, summary, error in sync_results: + if error: + response.append_result(f"✗ **{name}** agent failed: {error}") + else: + response.append_result(f"✓ **{name}** agent completed:\n{summary}") + + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): @@ -121,8 +184,10 @@ def format_output(cls, coder, mcp_server, tool_response): for i, d in enumerate(delegations): name = d.get("name", "") prompt = d.get("prompt", "") + is_async = d.get("async", True) coder.io.tool_output(f"{color_start}delegation_{i + 1}:{color_end}") coder.io.tool_output(f"agent: {name}") + coder.io.tool_output(f"mode: {'async' if is_async else 'sync'}") coder.io.tool_output(f"task: {prompt}") if i < len(delegations) - 1: coder.io.tool_output("") diff --git a/cecli/tools/edit_file.py b/cecli/tools/edit_file.py index 761af17c91e..98e336ba564 100644 --- a/cecli/tools/edit_file.py +++ b/cecli/tools/edit_file.py @@ -1,20 +1,22 @@ from cecli.helpers.hashline import ( + HASH_DELIMITER, + UNIQUE_HASH_DELIMITER, ContentHashError, apply_hashline_operations, get_hashline_diff, - normalize_hashline, resolve_content_to_hashline_ids, strip_hashline, ) +from cecli.helpers.hashpos.hashpos import HashPos +from cecli.helpers.hashpos.transformations import resolve_at_l, strip_hashline_prefix from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, apply_change, - format_tool_result, - handle_tool_error, validate_file_for_edit, ) from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations VALID_OPERATIONS = {"replace", "delete", "insert"} @@ -34,6 +36,7 @@ class Tool(BaseTool): NORM_NAME = "editfile" + RESULT_TYPE = "list" TRACK_INVOCATIONS = False VALIDATIONS = { "edits": ["coerce_list"], @@ -44,13 +47,16 @@ class Tool(BaseTool): "function": { "name": "EditFile", "description": ( - "Edit text in one or more files using content ID markers. " + "Edit text in one or more files using virtual identifiers. " "You can perform multiple 'replace' or 'delete' operations in a single call. " "CRITICAL RULES: " - "1. Start and end content IDs are INCLUSIVE. Both will be modified or deleted. " - "2. Content IDs MUST include the `::` demarcator. " - "3. Edits within the same file MUST NOT be adjacent or overlapping. " - "4. For empty files, you MUST use '@000' as the content ID reference." + "1. Start and end markers are INCLUSIVE. Both will be modified or deleted. " + f"2. To target unique lines (prefixed with '{UNIQUE_HASH_DELIMITER}'), use their exact literal text as the marker. " # noqa + f"3. To target duplicate lines, you MUST include the exact hashed prefix (e.g., '{HASH_DELIMITER}“0车加{HASH_DELIMITER}'). " # noqa + "4. Edits within the same file MUST NOT be adjacent or overlapping. " + "5. For empty files, you MUST use '@000' as the reference. " + "6. Identifiers track global occurrences. Adding, modifying, or deleting a line can instantly " + "change the prefixes of identical lines anywhere else in the file. Re-read to get fresh IDs after editing." # noqa ), "parameters": { "type": "object", @@ -89,15 +95,23 @@ class Tool(BaseTool): "start_line": { "type": "string", "description": ( - "The exact content ID and demarcator for the start of the edit " - "(e.g., 'abc::'). For empty files, use '@000'." + "The exact reference for the start of the edit. " + "For duplicate lines with a specific hash, " + "use the 4-character hash wrapped in tildes (e.g., '~WecX~'). " + "For unique lines marked with the generic '~~' prefix, " + "provide the exact full text of the line. " + "For empty files, use '@000'." ), }, "end_line": { "type": "string", "description": ( - "The exact content ID and demarcator for the end of the edit " - "(e.g., 'xyz::'). For empty files, use '@000'." + "The exact reference for the end of the edit. " + "For duplicate lines with a specific hash, " + "use the 4-character hash wrapped in tildes (e.g., '~WecX~'). " + "For unique lines marked with the generic '~~' prefix, " + "provide the exact full text of the line. " + "For empty files, use '@000'." ), }, }, @@ -153,7 +167,8 @@ def execute( force=True, ) - tool_name = "EditFile" + # tool_name = "EditFile" + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) try: # 1. Validate edits parameter if not isinstance(edits, list): @@ -186,6 +201,17 @@ def execute( coder, file_path_key ) + if abs_path: + coder.file_read_cache.discard(abs_path) + + # Build HashPos index once per file for @L{num} resolution + hp = HashPos(original_content or "") + source_lines = ( + original_content.splitlines() + if original_content and original_content.strip() + else [] + ) + # Process all edits for this file using batch operations operations = [] file_metadata = [] @@ -218,11 +244,22 @@ def execute( edit_start_line = "@000" edit_end_line = "@000" - # 3. Auto-sanitize malformed boundaries (strip accidentally appended code) - if isinstance(edit_start_line, str) and "::" in edit_start_line: - edit_start_line = normalize_hashline(edit_start_line) - if isinstance(edit_end_line, str) and "::" in edit_end_line: - edit_end_line = normalize_hashline(edit_end_line) + # Resolve @L{num} notation directly to content ID + edit_start_line = cls._resolve_at_l_num( + edit_start_line, hp, source_lines, file_path_key + ) + edit_end_line = cls._resolve_at_l_num( + edit_end_line, hp, source_lines, file_path_key + ) + + # Strip ~~ virtual prefixes from ReadFile output lines + edit_start_line = cls._strip_readfile_prefix(edit_start_line) + edit_end_line = cls._strip_readfile_prefix(edit_end_line) + + # Resolve remaining non-hashline content values to content IDs + edit_start_line, edit_end_line = resolve_content_to_hashline_ids( + original_content, edit_start_line, edit_end_line + ) # --------------------------------------------------------- @@ -235,11 +272,6 @@ def execute( edit_file = edit_file_raw - # Try to resolve line content values to content IDs - edit_start_line, edit_end_line = resolve_content_to_hashline_ids( - original_content, edit_start_line, edit_end_line - ) - # Validate required fields based on operation type # (Note: The check for 'edit_file is None' will now be safely # bypassed because we defaulted it to "" above) @@ -397,22 +429,20 @@ def execute( # If dry run, return all results if dry_run: dry_run_messages = "\n".join(r.get("dry_run_message", "") for r in all_results) - return format_tool_result( - coder, - tool_name, - "", - dry_run=True, - dry_run_message=dry_run_messages or "Dry run: No changes would be made", - ) + response.append_result(dry_run_messages or "Dry run: No changes would be made") + return response # 4. Check if any edits succeeded overall if total_successful_edits == 0: coder.edit_allowed = True error_msg = "No edits were successfully applied:\n" + "\n".join(all_failed_edits) + response.append_error(all_failed_edits) raise ToolError(error_msg) # 5. Format and return result + cls.clear_invocation_cache() + if files_processed == 1: # Single file case result = all_results[0] @@ -421,35 +451,47 @@ def execute( ) if result["failed_edits"]: success_message += f" ({len(result['failed_edits'])} failed)" - # Include failed edit details in message to LLM success_message += "\nFailed edits:\n" + "\n".join(result["failed_edits"]) - change_id_to_return = result.get("change_id") - else: - # Multiple files case - success_message = ( - f"Applied {total_successful_edits} edits across {files_processed} files" - ) - if all_failed_edits: - success_message += f" ({len(all_failed_edits)} failed)" - # Include failed edit details in message to LLM - success_message += "\nFailed edits:\n" + "\n".join(all_failed_edits) - change_id_to_return = None # Multiple change IDs, can't return single one - cls.clear_invocation_cache() + response.append_result( + content=f"\u2713 {success_message}", + metadata={ + "change_id": result.get("change_id"), + "file_path": result.get("file_path"), + "successful_edits": result.get("successful_edits"), + "failed_edits": result.get("failed_edits", []), + }, + ) + else: + # Multiple files case — append per-file structured results + for result in all_results: + per_file_message = ( + f"Applied {result['successful_edits']} edits in {result['file_path']}" + ) + if result["failed_edits"]: + per_file_message += f" ({len(result['failed_edits'])} failed)" + per_file_message += "\nFailed edits:\n" + "\n".join(result["failed_edits"]) + + response.append_result( + content=f"\u2713 {per_file_message}", + metadata={ + "change_id": result.get("change_id"), + "file_path": result.get("file_path"), + "successful_edits": result.get("successful_edits"), + "failed_edits": result.get("failed_edits", []), + }, + ) - return format_tool_result( - coder, - tool_name, - success_message, - change_id=change_id_to_return, - ) + return response except ToolError as e: coder.edit_allowed = False - return handle_tool_error(coder, tool_name, e, add_traceback=False) + response.append_error(str(e)) + return response except Exception as e: coder.edit_allowed = False - return handle_tool_error(coder, tool_name, e) + response.append_error(str(e)) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): @@ -536,6 +578,28 @@ def format_output(cls, coder, mcp_server, tool_response): tool_footer(coder=coder, tool_response=tool_response, params=params) + @classmethod + def _resolve_at_l_num(cls, line_spec, hp, source_lines, file_path): + """Resolve @L{num} notation to a content ID using a pre-built HashPos index. + + Returns the input unchanged if it's not an @L{num} spec. + Raises ToolError if the line number is out of range. + """ + + from cecli.tools.utils.helpers import ToolError + + try: + return resolve_at_l(line_spec, hp, source_lines) + + except ValueError as e: + raise ToolError(str(e)) from e + + @staticmethod + def _strip_readfile_prefix(value): + """Strip the ``~~`` virtual prefix from a ReadFile output line reference.""" + + return strip_hashline_prefix(value) + @classmethod def _categorize_edit_error(cls, error_msg: str) -> str: """Categorize an edit error message into a user-friendly display category.""" diff --git a/cecli/tools/explore_code.py b/cecli/tools/explore_code.py index 8f7c8b734a1..81552db6b84 100644 --- a/cecli/tools/explore_code.py +++ b/cecli/tools/explore_code.py @@ -3,6 +3,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations cwd = os.getcwd() @@ -23,6 +24,7 @@ class Tool(BaseTool): "queries": ["coerce_list"], "queries[]": ["coerce_dict"], } + RESULT_TYPE = "list" SCHEMA = { "type": "function", "function": { @@ -90,13 +92,15 @@ def execute(cls, coder, queries, **kwargs): Returns: str: Formatted results from the Cymbal operations. """ + response = ToolResponse(cls.NORM_NAME, result_type="list") try: # Check if cymbal is available if not CYMBAL_AVAILABLE: coder.io.tool_error( "Cymbal library is not available. Please install it with: pip install py-cymbal" ) - return "Error: Cymbal library is not available" + response.append_error("Cymbal library is not available") + return response # Initialize Cymbal and index if necessary c = cymbal.Cymbal() @@ -108,8 +112,8 @@ def execute(cls, coder, queries, **kwargs): except Exception as e: error_msg = f"Failed to index repository: {str(e)}" coder.io.tool_error(error_msg) - return f"Error: {error_msg}" - all_results = [] + response.append_error(error_msg) + return response all_failed_queries = [] total_successful_queries = 0 @@ -124,7 +128,8 @@ def execute(cls, coder, queries, **kwargs): # Replace hyphens with underscores (common in code) and strip special chars. safe_symbol = symbol.replace("-", "_") if symbol else symbol results = c.search(safe_symbol, limit=limit) - all_results.append(cls._format_search_results(results, symbol)) + results = cls._filter_gitignored(results, coder) + response.append_result(content=cls._format_search_results(results, symbol)) elif action == "investigate": symbol_name = symbol file_hint = "" @@ -139,27 +144,35 @@ def execute(cls, coder, queries, **kwargs): try: investigation = c.investigate(safe_name, file_hint) - all_results.append( - cls._format_investigation_results(investigation, symbol) + investigation = cls._filter_investigation_gitignored( + investigation, coder + ) + response.append_result( + content=cls._format_investigation_results(investigation, symbol) ) except Exception as e: if "multiple matches" in str(e).lower(): results = c.search(symbol_name, limit=10) - locations = "\n".join( - [f"- {r['file']}:{r['start_line']}" for r in results] - ) - msg = ( - f"Error: Multiple matches found for '{symbol}'.\nPlease use a" - " more specific name or check the locations" - f" below:\n{locations}" + response.append_result( + content=( + f"Error: Multiple matches found for '{symbol}'.\n" + f"Please use a more specific name or check the locations" + f" below:\n" + + "\n".join( + f"- {r.get('rel_path') or r.get('file', '')}:{r.get('start_line', 0)}" + for r in results + ) + ) ) - all_results.append(msg) else: raise e elif action == "find_references": safe_symbol = symbol.replace("-", "_") if symbol else symbol references = c.find_references(safe_symbol, limit=limit) - all_results.append(cls._format_reference_results(references, symbol)) + references = cls._filter_gitignored(references, coder) + response.append_result( + content=cls._format_reference_results(references, symbol) + ) else: all_failed_queries.append( f"Error for symbol '{symbol}': Unknown action '{action}'" @@ -174,23 +187,60 @@ def execute(cls, coder, queries, **kwargs): error_msg = "No queries were successfully executed:\n" + "\n".join( all_failed_queries ) - raise ToolError(error_msg) + response.append_error(error_msg) + return response if all_failed_queries: for failed_msg in all_failed_queries: coder.io.tool_error(failed_msg) else: - coder.io.tool_output("✓ All queries successful.", type="tool-result") + coder.io.tool_output("\u2713 All queries successful.", type="tool-result") - return "\n\n" + "=" * 40 + "\n\n".join(all_results) + return response except Exception as e: coder.io.tool_error(f"Error in ExploreCode: {str(e)}") - return f"Error: {str(e)}" + response.append_error(str(e)) + return response finally: if "c" in locals(): c.close() + @classmethod + def _filter_gitignored(cls, results, coder): + """Filter out results whose file path is git-ignored.""" + + if not results or not hasattr(coder, "repo") or not coder.repo: + return results + + filtered = [] + for r in results: + file_path = r.get("rel_path") or r.get("file", "") + if not file_path: + filtered.append(r) + continue + if not coder.repo.git_ignored_file(file_path): + filtered.append(r) + + return filtered + + @classmethod + def _filter_investigation_gitignored(cls, investigation, coder): + """Filter git-ignored entries from an investigation result.""" + + if not investigation or not hasattr(coder, "repo") or not coder.repo: + return investigation + + # Filter references + if "refs" in investigation: + investigation["refs"] = cls._filter_gitignored(investigation["refs"], coder) + + # Filter impact/callers + if "impact" in investigation: + investigation["impact"] = cls._filter_gitignored(investigation["impact"], coder) + + return investigation + @classmethod def _format_search_results(cls, results, symbol): """Format search results for display.""" diff --git a/cecli/tools/git_branch.py b/cecli/tools/git_branch.py index 21e442dfe1c..416937492fe 100644 --- a/cecli/tools/git_branch.py +++ b/cecli/tools/git_branch.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -83,8 +84,11 @@ def execute( """ List branches in the repository with various filtering and formatting options. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: # Build git command arguments @@ -121,13 +125,18 @@ def execute( try: head = coder.repo.repo.head if head.is_detached: - return "HEAD (detached)" - return coder.repo.repo.active_branch.name + response.append_result("HEAD (detached)") + return response + response.append_result(coder.repo.repo.active_branch.name) + return response except ANY_GIT_ERROR: - return "No current branch found." + response.append_result("No current branch found.") + return response - return result if result else "No branches found matching the criteria." + response.append_result(result if result else "No branches found matching the criteria.") + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git branch: {e}") - return f"Error running git branch: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_diff.py b/cecli/tools/git_diff.py index 99cb5d8ad6b..2e7144f3725 100644 --- a/cecli/tools/git_diff.py +++ b/cecli/tools/git_diff.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -31,8 +32,11 @@ def execute(cls, coder, branch=None, **kwargs): """ Show the diff between the current working directory and a git branch or commit. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: if branch: @@ -42,8 +46,11 @@ def execute(cls, coder, branch=None, **kwargs): diff = coder.repo.diff_commits(False, "HEAD", None) if not diff: - return "No differences found." - return diff + response.append_result("No differences found.") + return response + response.append_result(diff) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git diff: {e}") - return f"Error running git diff: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_log.py b/cecli/tools/git_log.py index 606308c1388..40bbc9fd3a8 100644 --- a/cecli/tools/git_log.py +++ b/cecli/tools/git_log.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -27,8 +28,11 @@ def execute(cls, coder, limit=10, **kwargs): """ Show the git log. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: commits = list(coder.repo.repo.iter_commits(max_count=limit)) @@ -37,7 +41,9 @@ def execute(cls, coder, limit=10, **kwargs): short_hash = commit.hexsha[:8] message = commit.message.strip().split("\n")[0] log_output.append(f"{short_hash} {message}") - return "\n".join(log_output) + response.append_result("\n".join(log_output)) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git log: {e}") - return f"Error running git log: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_remote.py b/cecli/tools/git_remote.py index 4138532b2ad..3025cce42a0 100644 --- a/cecli/tools/git_remote.py +++ b/cecli/tools/git_remote.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -22,18 +23,24 @@ def execute(cls, coder, **kwargs): """ List remote repositories. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: remotes = coder.repo.repo.remotes if not remotes: - return "No remotes configured." + response.append_result("No remotes configured.") + return response result = [] for remote in remotes: result.append(f"{remote.name}\t{remote.url}") - return "\n".join(result) + response.append_result("\n".join(result)) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git remote: {e}") - return f"Error running git remote: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_show.py b/cecli/tools/git_show.py index c4dee758e8d..60acbb66206 100644 --- a/cecli/tools/git_show.py +++ b/cecli/tools/git_show.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -27,11 +28,17 @@ def execute(cls, coder, object="HEAD", **kwargs): """ Show various types of objects (blobs, trees, tags, and commits). """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: - return coder.repo.repo.git.show(object) + result = coder.repo.repo.git.show(object) + response.append_result(result) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git show: {e}") - return f"Error running git show: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/git_status.py b/cecli/tools/git_status.py index 76531bf7dd6..523b4f3e863 100644 --- a/cecli/tools/git_status.py +++ b/cecli/tools/git_status.py @@ -1,5 +1,6 @@ from cecli.repo import ANY_GIT_ERROR from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -22,11 +23,17 @@ def execute(cls, coder, **kwargs): """ Show the working tree status. """ + response = ToolResponse(cls.NORM_NAME) + if not coder.repo: - return "Not in a git repository." + response.append_result("Not in a git repository.") + return response try: - return coder.repo.repo.git.status() + result = coder.repo.repo.git.status() + response.append_result(result) + return response except ANY_GIT_ERROR as e: coder.io.tool_error(f"Error running git status: {e}") - return f"Error running git status: {e}" + response.append_error(str(e)) + return response diff --git a/cecli/tools/grep.py b/cecli/tools/grep.py index 30d95426bb4..2067ddcf134 100644 --- a/cecli/tools/grep.py +++ b/cecli/tools/grep.py @@ -1,4 +1,3 @@ -import json import os import re import shutil @@ -11,6 +10,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations # Default directories to exclude from search results across various languages @@ -90,20 +90,48 @@ def _parse_content_into_files(output): Returns list of dicts with keys: path, match_count, content_lines Content lines preserve the original grep format (with : for matches, - for context). + Merges entries for the same file that appear in non-contiguous match groups. """ if not output: return [] - files = [] lines = output.splitlines() if not lines: return [] + # Use a dict to accumulate file groups, keyed by filepath + # This handles interleaved results (file1 -> file2 -> file1) by merging + file_groups = {} # filepath -> {path, match_count, content_parts} + file_order = [] # ordered list of filepaths as they first appear + current_file = None current_lines = [] match_count = 0 - for i, line in enumerate(lines): + def _flush_current(): + nonlocal current_file, current_lines, match_count + if current_file is None or not current_lines: + return + + if current_file in file_groups: + # Merge into existing entry + existing = file_groups[current_file] + existing["match_count"] += match_count + existing["content_parts"].append("\n".join(current_lines)) + else: + # New file entry + file_groups[current_file] = { + "path": current_file, + "match_count": match_count, + "content_parts": ["\n".join(current_lines)], + } + file_order.append(current_file) + + current_file = None + current_lines = [] + match_count = 0 + + for line in lines: # Skip separator lines ("--" between non-contiguous match groups) if line == "--": continue @@ -129,14 +157,8 @@ def _parse_content_into_files(output): if is_match_line: match_count += 1 else: - # New file - save previous, start new - files.append( - { - "path": current_file, - "match_count": match_count, - "content": "\n".join(current_lines), - } - ) + # Different file - flush current block before switching + _flush_current() current_file = filepath match_count = 1 if is_match_line else 0 current_lines = [line] @@ -145,13 +167,18 @@ def _parse_content_into_files(output): if current_file is not None: current_lines.append(line) - # Don't forget the last file - if current_file is not None and current_lines: + # Flush the last file's accumulated block + _flush_current() + + # Build final list preserving first-seen order + files = [] + for fpath in file_order: + group = file_groups[fpath] files.append( { - "path": current_file, - "match_count": match_count, - "content": "\n".join(current_lines), + "path": group["path"], + "match_count": group["match_count"], + "content": "\n".join(group["content_parts"]), } ) @@ -160,8 +187,8 @@ def _parse_content_into_files(output): class Tool(BaseTool): NORM_NAME = "grep" + RESULT_TYPE = "list" VALIDATIONS = { - "searches": ["coerce_list"], "searches[]": ["coerce_dict"], } SCHEMA = { @@ -201,25 +228,18 @@ class Tool(BaseTool): "default": True, "description": "Whether to perform a case-insensitive search.", }, - "count": { - "type": "boolean", - "default": True, - "description": ( - "Include match counts per file in the output summary." - ), - }, "context_before": { "type": "integer", "default": 2, "description": ( - "Number of context lines to show before each match." + "Number of context lines to show before each match. Max 5" ), }, "context_after": { "type": "integer", "default": 2, "description": ( - "Number of context lines to show after each match." + "Number of context lines to show after each match. Max 5" ), }, }, @@ -309,17 +329,23 @@ def execute( match counts, and summary metadata. """ if not isinstance(searches, list): - return json.dumps({"error": "'searches' parameter must be an array."}) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + response.append_error("'searches' parameter must be an array.") + return response repo = coder.repo if not repo: coder.io.tool_error("Not in a git repository.") - return json.dumps({"error": "Not in a git repository."}) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + response.append_error("Not in a git repository.") + return response tool_name, tool_path = cls._find_search_tool() if not tool_path: coder.io.tool_error("No search tool (rg, ag, grep) found in PATH.") - return json.dumps({"error": "No search tool (rg, ag, grep) found."}) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + response.append_error("No search tool (rg, ag, grep) found.") + return response all_operation_results = [] @@ -329,8 +355,8 @@ def execute( directory = search_op.get("directory", search_op.get("path", ".")) use_regex = search_op.get("use_regex", False) case_insensitive = search_op.get("case_insensitive", True) - context_before = search_op.get("context_before", 2) - context_after = search_op.get("context_after", 2) + context_before = max(min(int(search_op.get("context_before", 2)), 5), 0) + context_after = max(min(int(search_op.get("context_after", 2)), 5), 0) count_enabled = search_op.get("count", True) op_result = { @@ -426,8 +452,8 @@ def execute( # --- PASS 2: Get content with context --- content_cmd = ( base_cmd - + (["-B", str(context_before)] if context_before > 0 else []) - + (["-A", str(context_after)] if context_after > 0 else []) + + (["-B", str(context_before)] if context_before >= 0 else []) + + (["-A", str(context_after)] if context_after >= 0 else []) + case_flag + pattern_flag + exclude_args @@ -469,7 +495,7 @@ def execute( MAX_MATCHES_PER_FILE = 10 MAX_FILES = 20 - truncated_files = [] + truncated_files = {} total_matches = 0 total_files = 0 @@ -477,16 +503,23 @@ def execute( file_lines = pf["content"].splitlines() # Find actual match lines (lines with `:LINE:` pattern) filepath_escaped = re.escape(pf["path"]) - match_line_re = re.compile(r"^" + filepath_escaped + r":\d+:") + match_line_re = re.compile(r"^" + filepath_escaped + r":(\d+):") match_lines_found = [ln for ln in file_lines if match_line_re.match(ln)] + # Normalize path to be relative to repo root + pf["path"] = os.path.relpath(pf["path"], repo.root) + if len(match_lines_found) > MAX_MATCHES_PER_FILE: - truncated_files.append(pf["path"]) + # Extract line numbers from excess matches (beyond first 10) + extra_lines = match_lines_found[MAX_MATCHES_PER_FILE:] + extra_line_nums = [] + for xline in extra_lines: + xm = match_line_re.match(xline) + if xm: + extra_line_nums.append(int(xm.group(1))) + truncated_files[pf["path"]] = extra_line_nums trimmed = "\n".join(match_lines_found[:MAX_MATCHES_PER_FILE]) pf["content"] = trimmed - - # Normalize path to be relative to repo root - pf["path"] = os.path.relpath(pf["path"], repo.root) total_matches += pf.get("count_from_pass", 0) total_files += 1 @@ -501,9 +534,9 @@ def execute( op_result["has_more_files"] = has_more op_result["files"] = [ { - "path": pf["path"], + "file": pf["path"], "match_count": pf.get("count_from_pass", 0), - "truncated": pf["path"] in truncated_files, + "additional_matched_lines": truncated_files.get(pf["path"], []), "content": pf["content"], } for pf in parsed_files[:MAX_FILES] @@ -520,8 +553,6 @@ def execute( all_operation_results.append(op_result) - final_result = {"operations": all_operation_results} - # TUI summary if coder.tui and coder.tui(): ui_summaries = [] @@ -539,7 +570,29 @@ def execute( ui_message = "\n".join(ui_summaries) coder.io.tool_output(ui_message, type="tool-result") - return json.dumps(final_result) + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + for op_result in all_operation_results: + files = op_result.get("files", []) + metadata = op_result.copy() + + # Build a human-readable string summary as content + pattern = op_result.get("pattern", "") + if op_result.get("error"): + summary = f"[{pattern}]\nError: {op_result['error']}" + elif op_result.get("total_matches", 0) == 0: + summary = f"[{pattern}]\nNo matches found." + else: + lines = [f"[{pattern}]"] + for f in files: + truncated_mark = " (truncated)" if f.get("additional_matched_lines") else "" + lines.append(f"{f['file']}: {f['match_count']} match(es){truncated_mark}") + if op_result.get("has_more_files"): + lines.append(f"... ({op_result['total_files']} files total)") + summary = "\n".join(lines) + + response.append_result(content=summary, metadata=metadata) + + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): @@ -563,7 +616,7 @@ def format_output(cls, coder, mcp_server, tool_response): for i, search_op in enumerate(searches): pattern = search_op.get("pattern", "") file_pattern = search_op.get("file_glob", "*") - directory = search_op.get("directory", ".") + directory = search_op.get("directory", search_op.get("path", ".")) use_regex = search_op.get("use_regex", False) case_insensitive = search_op.get("case_insensitive", True) context_before = search_op.get("context_before", 2) diff --git a/cecli/tools/ls.py b/cecli/tools/ls.py index 4305c5aea90..2054a261ef5 100644 --- a/cecli/tools/ls.py +++ b/cecli/tools/ls.py @@ -3,6 +3,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -41,6 +42,7 @@ def execute(cls, coder, path=None, **kwargs): """ # Handle both positional and keyword arguments for backward compatibility dir_path = path or "." + response = ToolResponse(cls.NORM_NAME) try: # Create an absolute path from the provided relative path @@ -51,12 +53,14 @@ def execute(cls, coder, path=None, **kwargs): coder.io.tool_error( f"Error: Path '{dir_path}' attempts to access files outside the project root." ) - return "Error: Path is outside the project root." + response.append_result("Error: Path is outside the project root.") + return response # Check if path exists if not os.path.exists(abs_path): coder.io.tool_output(f"⚠ Path '{dir_path}' not found", type="tool-result") - return "Directory not found" + response.append_result("Directory not found") + return response # Get directory contents contents = [] @@ -70,7 +74,8 @@ def execute(cls, coder, path=None, **kwargs): contents.append(rel_path) except OSError as e: coder.io.tool_error(f"Error listing directory '{dir_path}': {e}") - return f"Error: {e}" + response.append_result(f"Error: {e}") + return response elif os.path.isfile(abs_path): # It's a file, just return its relative path contents.append(os.path.relpath(abs_path, coder.root)) @@ -81,19 +86,25 @@ def execute(cls, coder, path=None, **kwargs): ) sorted_contents = sorted(contents) if len(sorted_contents) > 500: - return ( + response.append_result( f"Found {len(sorted_contents)} files:" f" {', '.join(sorted_contents[:500])}" f"\n... and {len(sorted_contents) - 500} more" ) + return response else: - return f"Found {len(sorted_contents)} files: {', '.join(sorted_contents)}" + response.append_result( + f"Found {len(sorted_contents)} files: {', '.join(sorted_contents)}" + ) + return response else: coder.io.tool_output(f"🗐 No files found in '{dir_path}'", type="tool-result") - return "No files found in directory" + response.append_result("No files found in directory") + return response except Exception as e: coder.io.tool_error(f"Error in ls: {str(e)}") - return f"Error: {str(e)}" + response.append_result(f"Error: {str(e)}") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/orchestrate.py b/cecli/tools/orchestrate.py new file mode 100644 index 00000000000..9a5b80bceb6 --- /dev/null +++ b/cecli/tools/orchestrate.py @@ -0,0 +1,78 @@ +import logging + +from cecli.helpers.orchestration.service import OrchestrationService +from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.helpers import ToolError +from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse +from cecli.tools.validations import ToolValidations + +logger = logging.getLogger(__name__) + + +class Tool(BaseTool): + NORM_NAME = "orchestrate" + TRACK_INVOCATIONS = False + VALIDATIONS = {} + SCHEMA = { + "type": "function", + "function": { + "name": "Orchestrate", + "description": ( + "Execute Python code in a sandboxed environment where you can call " + "other tools programmatically. Use this instead of making many " + "individual tool calls for batch operations. The environment provides " + "`Agent.get_tool(name)` to get tool proxies, `gather(*tasks)` for " + "parallel execution, and `state` for persistent storage across calls." + ), + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": ( + "Python code to execute in the sandbox. See the orchestration " + "context block for available primitives and calling conventions." + ), + }, + }, + "required": ["code"], + }, + }, + } + + @classmethod + async def execute(cls, coder, code, **kwargs): + BaseTool.clear_invocation_cache() + env = OrchestrationService.get_instance(coder) + result = await env.execute(code) + response = ToolResponse(cls.NORM_NAME, result_type="list") + response.append_result( + content=result["results"], metadata={"state_variables": result["state_variables"]} + ) + return response + + @classmethod + def format_output(cls, coder, mcp_server, tool_response): + + color_start, color_end = color_markers(coder) + + tool_header(coder=coder, mcp_server=mcp_server, tool_response=tool_response) + + try: + params = ToolValidations.validate_params( + tool_response.function.arguments, cls.VALIDATIONS, cls.SCHEMA + ) + except ToolError: + coder.io.tool_error("Invalid Tool JSON") + return + + code = params.get("code", "") + if code: + coder.io.tool_output("") + coder.io.tool_output(f"{color_start}Code:{color_end}") + for line in code.strip().splitlines(): + coder.io.tool_output(f" {line}") + coder.io.tool_output("") + + tool_footer(coder=coder, tool_response=tool_response, params=params) diff --git a/cecli/tools/read_file.py b/cecli/tools/read_file.py index 0ffc04ca2fc..4c3a91371cb 100644 --- a/cecli/tools/read_file.py +++ b/cecli/tools/read_file.py @@ -3,19 +3,29 @@ from typing import Dict, List from cecli.helpers.hashline import hashline_formatted, strip_hashline +from cecli.helpers.hashpos.transformations import ( + apply_contextual_marker, + classify_search_type, + compute_best_pair, + extract_hint, + narrow_by_proximity, + reposition_indices, + search_in_lines, +) from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ( ToolError, - handle_tool_error, is_provided, resolve_paths, ) from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "readfile" + RESULT_TYPE = "list" TRACK_INVOCATIONS = False VALIDATIONS = { "read": ["coerce_list"], @@ -28,24 +38,24 @@ class Tool(BaseTool): "function": { "name": "ReadFile", "description": ( - "Get content ID prefixed content between start and end markers in files." - " Accepts an array of `read` objects, each with file_path, range_start, range_end." - " They can contain up to 3 lines of content. Avoid using singular generic keywords and" - " symbols. Special markers @000 and 000@ represent the file boundaries and can be" - " used for range_start and range_end for the first and last lines of the file" - " respectively. Line numbers may also be used for range lookups." - " It is best to use function names, variable declarations, entire line contents" - " and other meaningful identifiers as range_start and range_end values." - " Results may be modified from the raw search for semantic relevance." - " The returned identifiers will not persist after editing the file." - " Do not use the same pattern for the range_start and range_end." - " Do not use empty strings for the range_start and range_end." - " Do not use content IDs for the range_start and range_end values as they change between edits." - " Always use the ReadFile tool instead of cli tools for reading file contents." - " Line number and special marker ranges greater than 200 lines will return" - " preview content for further, more scoped investigation." - " Call this tool sequentially on increasingly finer grained searches " - " to help with understanding important structural features in large files." + "Get prefixed content between start and end markers. Accepts an array of `read` objects " + "(file_path, range_start, range_end). Range markers can be text patterns (up to 3 lines), " + "file boundaries (@000, 000@), or exact line numbers (@L10). " + "Contextual end markers (@C{num}, @P{num}, @N{num}) expand symmetrically, previously, or next " + "around a single unique range_start match." + " Use line hints (e.g., 'my_func @L150', '@A{{def foo}}', '@B{{return x}}') to disambiguate. " + "@A{{regex}} keeps only the closest match **after** the regex hit; " + "@B{{regex}} keeps only the closest match **before** the regex hit. " + "" + "File lines are prefixed with virtual, deterministic identifiers generated on-the-fly: " + "Because identifiers track global occurrences, " + "adding or deleting a line can instantly change the prefix " + "of identical lines anywhere else in the file as well as content after the edit." + "Always re-read the file to get fresh IDs after making edits." + "" + "Avoid generic keywords; use meaningful identifiers like function names. Do not use empty strings. " + "Always use ReadFile instead of CLI tools. Ranges >200 lines return a structural preview. " + "Call sequentially with increasingly fine-grained searches to drill down into large files." ), "parameters": { "type": "object", @@ -64,6 +74,8 @@ class Tool(BaseTool): "description": ( "The text marking the beginning of the range." " Use '@000' for the first line on empty files." + " Append ' @L' (e.g., 'my_func @L1506') as a" + " proximity hint to help select among multiple matches." ), }, "range_end": { @@ -71,6 +83,12 @@ class Tool(BaseTool): "description": ( "The text marking the end of the range." " Use '000@' for the last line on empty files." + " When range_start uniquely matches one location, you" + " may use contextual markers: '@C{number}' (e.g., '@C5')" + " for lines on both sides of the match, '@P{number}'" + " for lines BEFORE the match (the match is the range" + " end), or '@N{number}' for lines AFTER the match" + " (the match is the range start)." ), }, }, @@ -98,11 +116,12 @@ def execute(cls, coder, read, **kwargs): """ from cecli.helpers.conversation import ConversationService - tool_name = "ReadFile" already_up_to_date = [] new_context_retrieved = [] error_outputs = [] + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) + try: # 1. Validate read parameter if not isinstance(read, list): @@ -114,6 +133,7 @@ def execute(cls, coder, read, **kwargs): all_outputs = [] already_up_to_details = [] new_context_details = [] + seen_files = set() all_outputs_set = set() new_context_set = set() already_up_to_set = set() @@ -124,7 +144,7 @@ def execute(cls, coder, read, **kwargs): file_path = read_op.get("file_path") range_start = read_op.get("range_start") range_end = read_op.get("range_end") - padding = 5 + padding = 0 if file_path is None: error_outputs.append( @@ -172,6 +192,9 @@ def execute(cls, coder, read, **kwargs): range_start = strip_hashline(range_start).strip() range_end = strip_hashline(range_end).strip() + start_hint = None + end_hint = None + # 2. Resolve path abs_path, rel_path = resolve_paths(coder, file_path) if not os.path.exists(abs_path): @@ -188,6 +211,13 @@ def execute(cls, coder, read, **kwargs): ) continue + if abs_path not in seen_files: + seen_files.add(abs_path) + + if abs_path not in coder.file_read_cache: + coder.file_read_cache.add(abs_path) + ConversationService.get_files(coder).clear_ranges(abs_path) + # 3. Read file content content: str = coder.io.read_text(abs_path) @@ -207,18 +237,24 @@ def execute(cls, coder, read, **kwargs): lines = content.splitlines() num_lines = len(lines) + # Resolve hints after file content available (@A/@B regex hints need lines) + range_start, start_hint, start_hint_type = cls._extract_l_hint(range_start, lines) + range_end, end_hint, end_hint_type = cls._extract_l_hint(range_end, lines) + if num_lines == 0: new_context_details.append( - "\n".join( - [ - f"File {rel_path} is empty.", - ( - "Next: use EditFile with start_line @000 and end_line @000 to" - " write content, or ResourceManager to scaffold — do not call" - " ReadFile again on this empty file." - ), - ] - ) + { + "file_path": rel_path, + "status": "full", + "total_lines": 0, + "prefixed_contents": "", + "outline": "", + "note": ( + f"File {rel_path} is empty. Next: use EditFile with start_line @000 and" + " end_line @000 to write content, or ResourceManager to scaffold —" + " do not call ReadFile again on this empty file." + ), + } ) new_context_retrieved.append(rel_path) cls._last_read_turn[abs_path] = coder.turn_count @@ -227,222 +263,101 @@ def execute(cls, coder, read, **kwargs): start_line_idx = -1 end_line_idx = -1 both_structured = False - # found_by = "" if range_start is not None and range_end is not None: - - def _is_valid_int(s): - try: - int(s) - return True - except ValueError: - return False - - start_is_digit = _is_valid_int(range_start) - end_is_digit = _is_valid_int(range_end) - start_is_special = range_start in ("@000", "000@") - end_is_special = range_end in ("@000", "000@") - both_structured = (start_is_digit or start_is_special) and ( - end_is_digit or end_is_special - ) - start_is_text = not start_is_digit and not start_is_special - end_is_text = not end_is_digit and not end_is_special - mixed_special_search = (start_is_special and end_is_text) or ( - end_is_special and start_is_text - ) - start_indices = [] - end_indices = [] - - if both_structured: - if start_is_digit: - start_line_num = int(range_start) - 1 - start_line_num = max(0, min(start_line_num, num_lines - 1)) - start_indices = [start_line_num] - else: - start_indices = [0] - - if end_is_digit: - end_line_num = int(range_end) - 1 - end_line_num = max(0, min(end_line_num, num_lines - 1)) - end_indices = [end_line_num] - else: - end_indices = [num_lines - 1] - - elif mixed_special_search: - if start_is_special: - # Start is special marker, end is text pattern - if range_start == "@000": - start_indices = [0] - else: # 000@ - start_indices = [num_lines - 1] - # Search for end pattern as text - end_pattern_lines = range_end.split("\n") - end_indices = [] - for i in range(len(lines) - len(end_pattern_lines) + 1): - if all( - p_line in lines[i + j] - for j, p_line in enumerate(end_pattern_lines) - ): - end_indices.append(i + len(end_pattern_lines) - 1) - else: - # Start is text pattern, end is special marker - start_pattern_lines = range_start.split("\n") - start_indices = [] - for i in range(len(lines) - len(start_pattern_lines) + 1): - if all( - p_line in lines[i + j] - for j, p_line in enumerate(start_pattern_lines) - ): - start_indices.append(i) - if range_end == "@000": - end_indices = [0] - else: # 000@ - end_indices = [num_lines - 1] - else: - start_pattern_lines = range_start.split("\n") - start_indices = [] - for i in range(len(lines) - len(start_pattern_lines) + 1): - if all( - p_line in lines[i + j] - for j, p_line in enumerate(start_pattern_lines) - ): - start_indices.append(i) - - end_pattern_lines = range_end.split("\n") - end_indices = [] - for i in range(len(lines) - len(end_pattern_lines) + 1): - if all( - p_line in lines[i + j] for j, p_line in enumerate(end_pattern_lines) - ): - # For multiline end patterns, we want the index of the LAST line of the match - end_indices.append(i + len(end_pattern_lines) - 1) - - if len(start_indices) > 5: - # Too many matches - use _last_invocation to disambiguate - last = cls._last_invocation.get(abs_path) - if last is None: - error_outputs.append( - cls.format_error( - coder, - ( - f"Start pattern '{range_start}' too broad." - " Refine your search. Be more specific." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) - continue - - # Find the best match: smallest sum of absolute distances to last start/end - # that comes after the range, with tie-breaking by smallest sum - last_s, last_e = last["start_idx"], last["end_idx"] - candidates = [] - for s in start_indices: - for e in [idx for idx in end_indices if idx >= s]: - dist_sum = abs(s - last_s) + abs(e - last_e) - candidates.append((dist_sum, s, e)) - # Sort by distance sum, then prefer ranges after the last range - candidates.sort(key=lambda x: (x[0], x[1] < last_s, x[1], x[2])) - if candidates: - best_pair = (candidates[0][1], candidates[0][2]) - else: - best_pair = None - else: - best_pair = None - min_dist = float("inf") - - for s in start_indices: - for e in [idx for idx in end_indices if idx >= s]: - dist = e - s - if dist < min_dist: - min_dist = dist - best_pair = (s, e) - - # If no valid pair found and one side has exactly one match, - # try inverted matching in case LLM got the order of methods wrong - if best_pair is None and (len(start_indices) == 1 or len(end_indices) == 1): - for s in start_indices: - for e in end_indices: - if e < s: # end pattern match is before start pattern match - dist = s - e - if dist < min_dist: - min_dist = dist - best_pair = (e, s) - - if not start_indices: - error_outputs.append( - cls.format_error( - coder, - ( - f"Start pattern '{range_start}' not found in {file_path}." - " Refine your search." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) - continue - - if not end_indices: - error_outputs.append( - cls.format_error( - coder, - ( - f"End pattern '{range_end}' not found in {file_path}." - " Refine your search." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) - continue - - if best_pair is None: - error_outputs.append( - cls.format_error( - coder, - ( - f"End pattern '{range_end}' not found after start pattern in" - f" {file_path}." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) + # Step 1: Classify the search type + rt = cls._classify_search_type(range_start, range_end) + + # Step 2: Find start and end indices + start_indices = cls._find_start_indices(lines, range_start, rt, num_lines) + end_indices = cls._find_end_indices(lines, range_end, rt, num_lines) + + # Step 2a: Apply @A/@B directional filtering — keep only closest match in direction + if start_hint_type == "A" and start_hint is not None: + after = [i for i in start_indices if i > start_hint] + start_indices = [min(after)] if after else [] + start_hint = None + elif start_hint_type == "B" and start_hint is not None: + before = [i for i in start_indices if i < start_hint] + start_indices = [max(before)] if before else [] + start_hint = None + + if end_hint_type == "A" and end_hint is not None: + after = [i for i in end_indices if i > end_hint] + end_indices = [min(after)] if after else [] + end_hint = None + elif end_hint_type == "B" and end_hint is not None: + before = [i for i in end_indices if i < end_hint] + end_indices = [max(before)] if before else [] + end_hint = None + end_indices = [i for i in end_indices if i < end_hint] + end_hint = None + + # Step 3: Apply contextual marker (@C/@P/@N) + if rt["end_is_contextual"]: + result, ctx_error = cls._apply_contextual_marker( + start_indices, + range_start, + range_end, + num_lines, + coder, + file_path, + read_index, + ) + if ctx_error: + error_outputs.append(ctx_error) continue + start_indices, end_indices = result + + # Step 4: Disambiguate if too many matches + start_indices = cls._disambiguate_start_indices( + start_indices, + end_indices, + abs_path, + range_start, + num_lines, + rel_path, + read_index, + coder, + response, + start_hint=start_hint, + ) - if best_pair is None: - error_outputs.append( - cls.format_error( - coder, - ( - f"End pattern '{range_end}' not found after start pattern in" - f" {file_path}." - ), - file_path, - range_start, - range_end, - read_index, - ) - ) + # Step 5: Resolve to final indices with pair selection + fallbacks + s_idx, e_idx, resolve_errors = cls._resolve_to_final_indices( + start_indices, + end_indices, + num_lines, + coder, + abs_path, + range_start, + range_end, + file_path, + read_index, + ) + if resolve_errors: + for err in resolve_errors: + error_outputs.append(err) continue - s_idx, e_idx = best_pair - s_idx, e_idx = cls._extend_range_with_stub( - coder, abs_path, s_idx, e_idx, num_lines - ) + both_structured = rt["both_structured"] + mixed_special_search = rt["mixed_special"] + + # Check for repeat patterns BEFORE overwriting _last_invocation + last = cls._last_invocation.get(abs_path) + skip_truncation = ( + last is not None + and last.get("range_start") == range_start + and last.get("range_end") == range_end + ) # Store the found indices for future disambiguation - cls._last_invocation[abs_path] = {"start_idx": s_idx, "end_idx": e_idx} + cls._last_invocation[abs_path] = { + "start_idx": s_idx, + "end_idx": e_idx, + "range_start": range_start, + "range_end": range_end, + } # For structured searches (line numbers, special markers) or mixed searches # (one special marker, one text pattern), cap large ranges with preview @@ -450,8 +365,8 @@ def _is_valid_int(s): sliced_contents = "\n".join(content.splitlines()[s_idx:e_idx]) token_count = coder.main_model.token_count(content) sliced_token_count = coder.main_model.token_count(sliced_contents) - is_small_file = token_count <= min(coder.large_file_token_threshold / 4, 2048) - is_small_range = sliced_token_count <= min( + is_small_file = token_count <= max(coder.large_file_token_threshold / 4, 2048) + is_small_range = sliced_token_count <= max( coder.large_file_token_threshold / 8, 1024 ) if ( @@ -470,16 +385,44 @@ def _is_valid_int(s): ) if cls._special_marker_count[abs_path] > 1: coder.abs_fnames.add(abs_path) - preview = f"Full contents of {rel_path} will be added to context in future message." + preview = { + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "prefixed_contents": "", + "outline": "", + "note": ( + f"Full contents of {rel_path} will be added to context in future message." + ), + } if abs_path in coder.abs_read_only_fnames: coder.abs_read_only_fnames.remove(abs_path) - if preview not in all_outputs_set: - all_outputs_set.add(preview) + preview_key = json.dumps(preview, sort_keys=True) + if preview_key not in all_outputs_set: + all_outputs_set.add(preview_key) if len(all_outputs): all_outputs.append("") all_outputs.append(preview) + # If the range was large and we're showing a preview, add explicit guidance + range_lines = e_idx - s_idx + 1 + if not has_stub: + note_text = ( + f"read operation {read_index + 1}: {rel_path} range " + f"({range_lines} lines) is large. " + f"Use @L ranges (e.g., @L{s_idx + 1}, @L{e_idx + 1}) for precise reads." + ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) + continue # found_by = f"range '{range_start}' to '{range_end}'" @@ -512,9 +455,7 @@ def _is_valid_int(s): # output_lines = [f"Displaying context around {found_by} in {rel_path}:"] # Generate hashline for the entire file - hashed_content, _ = hashline_formatted( - content, file_name=abs_path, partial=False, expanded=False - ) + hashed_content, _ = hashline_formatted(content, file_name=abs_path) hashed_lines = hashed_content.splitlines() # Extract the context window from hashed lines @@ -579,16 +520,22 @@ def _is_valid_int(s): ): # hashed_slice = hashed_lines[s_idx : e_idx + 1] model_response = cls.format_model_response( - coder, rel_path, s_idx, e_idx, hashed_lines, current=is_already_up_to_date + coder, + rel_path, + s_idx, + e_idx, + hashed_lines, + current=is_already_up_to_date, + skip_truncation=skip_truncation, ) if is_already_up_to_date: - if model_response not in already_up_to_set: - already_up_to_set.add(model_response) + if str(model_response) not in already_up_to_set: + already_up_to_set.add(str(model_response)) already_up_to_details.append(model_response) else: - if model_response not in new_context_set: - new_context_set.add(model_response) + if str(model_response) not in new_context_set: + new_context_set.add(str(model_response)) new_context_details.append(model_response) # Conditionally remove old file context messages @@ -606,7 +553,8 @@ def _is_valid_int(s): ConversationService.get_files(coder).clear_ranges(abs_path) ConversationService.get_files(coder).push_range(abs_path, tuples) - ConversationService.get_chunks(coder).add_file_context_messages() + if new_context_details: + ConversationService.get_chunks(coder).add_file_context_messages() # if ( # ConversationService.get_chunks(coder).last_clear_count > 20 @@ -617,8 +565,6 @@ def _is_valid_int(s): # Log success and return the formatted context directly coder.edit_allowed = True - result_parts = [f"File Context Turn {coder.turn_count}"] - if already_up_to_details or new_context_details: if new_context_details: coder.io.tool_output( @@ -626,12 +572,23 @@ def _is_valid_int(s): type="tool-result", ) - detail_str = "\n".join(new_context_details) - result_parts.append( - f"Retrieved context for {len(new_context_details)} operation(s):\n\n" - f"{detail_str}\n" - "Full results for these reads will be given in a follow up message.\n" + note_text = ( + f"Retrieved context for {len(new_context_details)} operation(s). " + "Full results for these reads will be given in a follow up message." + " Note: Full contents contain aggregated content across multiple reads." ) + response.append_result( + content=note_text, + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) + for d in new_context_details: + content_text = d.pop("prefixed_contents", "") + response.append_result(content=content_text, metadata=d) if already_up_to_details: coder.io.tool_output( ( @@ -641,110 +598,95 @@ def _is_valid_int(s): type="tool-result", ) - detail_str = "\n".join(already_up_to_details) - result_parts.append( + note_text = ( "Earlier contents still valid from previous read for " - f"{len(already_up_to_details)} operation(s):\n\n" - f"{detail_str}\n" + f"{len(already_up_to_details)} operation(s). " "Relevant contents for these reads available in previous message." ) + response.append_result( + content=note_text, + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) + for d in already_up_to_details: + content_text = d.pop("prefixed_contents", "") + response.append_result(content=content_text, metadata=d) if already_up_to_date and not new_context_retrieved: - result_parts.append( - "Do not call `ReadFile` again with these parameters again unless you edit" - " the relevant files." + response.append_result( + content=( + "Do not call `ReadFile` again with these parameters again" + " unless you edit the relevant files." + ), + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, ) if all_outputs: - result_parts.append("\n".join(all_outputs)) - result_parts.append("\nUse these outlines to refine your search.\n") + for output in all_outputs: + if output: + if isinstance(output, dict): + outline_content = output.pop("outline", "") + prefixed_content = output.pop("prefixed_contents", "") + preview_content = prefixed_content or outline_content + response.append_result(content=preview_content, metadata=output) + else: + response.append_result(output) + response.append_result( + content="Use these outlines to refine your search.", + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) if error_outputs: coder.io.tool_error( f"Errors encountered for {len(error_outputs)} operation(s)", type="tool-result" ) - result_parts.append("Errors:\n" + "\n".join(error_outputs)) - - if not result_parts: - return "No files could be processed." + for err in error_outputs: + response.append_error(err) - return "\n---\n".join(result_parts) + response.append_result( + content=f"File Context Turn {coder.turn_count}", + metadata={ + "file_path": "", + "status": "placeholder", + "total_lines": 0, + "outline": "", + }, + ) + return response except ToolError as e: # Handle expected errors raised by utility functions or validation - return handle_tool_error(coder, tool_name, e, add_traceback=False) + response.append_error(str(e)) + return response except Exception as e: # Handle unexpected errors during processing - return handle_tool_error(coder, tool_name, e) + response.append_error(str(e)) + return response @classmethod - def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, current=False): + def format_model_response( + cls, coder, rel_path, s_idx, e_idx, hashed_lines, current=False, skip_truncation=False + ): """Format a file's context range as hash-prefixed lines for the model.""" - # Read file content for stub lookups - try: - from cecli.tools.utils.helpers import resolve_paths - - abs_path, _ = resolve_paths(coder, rel_path) - last_turn = cls._last_read_turn[abs_path] or 0 - except Exception: - pass - - # Try to return structural stub information instead of raw hashed lines - try: - if hashed_lines and current and coder.turn_count - last_turn <= 2: - num_lines = len(hashed_lines) - - start_stub_s, start_stub_e = cls._extend_range_with_stub( - coder, abs_path, s_idx, s_idx, num_lines - ) - end_stub_s, end_stub_e = cls._extend_range_with_stub( - coder, abs_path, e_idx, e_idx, num_lines - ) - - # start_stub_s, start_stub_e = cls._reposition_indices(s_idx, start_stub_s, start_stub_e) - # end_stub_s, end_stub_e = cls._reposition_indices(e_idx, end_stub_s, end_stub_e) - - start_found = start_stub_s != s_idx or start_stub_e != s_idx - end_found = end_stub_s != e_idx or end_stub_e != e_idx - - if end_stub_s != start_stub_s or end_stub_e != start_stub_e: - start_stub_s = end_stub_s - start_stub_e = end_stub_e - start_found = True - end_found = False - - if start_found or end_found: - snapshot_parts = [] - if start_found: - snapshot_parts.extend(hashed_lines[start_stub_s:start_stub_e]) - - has_second_range = ( - end_found - and start_stub_s != end_stub_s - and start_stub_e != end_stub_e - and end_stub_e != e_idx - ) - if has_second_range: - snapshot_parts.append("...⋮...\n") - snapshot_parts.extend(hashed_lines[end_stub_s:end_stub_e]) - - prefixed = "".join(snapshot_parts) - result = { - "file_path": rel_path, - "start_line": start_stub_s + 1, - "end_line": end_stub_e if has_second_range else start_stub_e, - "partial": True, - "expanded": True, - "prefixed_contents": prefixed, - } - return json.dumps(result, ensure_ascii=False) - except Exception: - pass hashed_content = "\n".join(hashed_lines[s_idx : e_idx + 1]) token_count = coder.main_model.token_count(hashed_content) - if token_count <= min(coder.large_file_token_threshold / 16, 512): + if skip_truncation or token_count <= min(coder.large_file_token_threshold / 16, 512): prefixed = hashed_content else: total = e_idx - s_idx @@ -755,13 +697,15 @@ def format_model_response(cls, coder, rel_path, s_idx, e_idx, hashed_lines, curr result = { "file_path": rel_path, + "status": "full", "start_line": s_idx + 1, "end_line": e_idx + 1, - "partial": True, - "expanded": False, + "total_lines": len(hashed_lines), "prefixed_contents": prefixed, + "outline": "", + "note": "", } - return json.dumps(result, ensure_ascii=False) + return result @classmethod def content_splitter(cls, coder, hashed_lines, s_idx, e_idx): @@ -856,7 +800,12 @@ def content_splitter(cls, coder, hashed_lines, s_idx, e_idx): output_lines = [] for chunk_idx, chunk in enumerate(output_parts): if chunk_idx > 0: - output_lines.append("...⋮...") + prev_end = output_parts[chunk_idx - 1][-1] + 1 + next_start = chunk[0] + 1 + omitted = next_start - prev_end - 1 + output_lines.append( + f"...⋮... [{omitted} lines omitted (L{prev_end + 1}–L{next_start - 1})]" + ) for idx in chunk: output_lines.append(hashed_lines[idx]) @@ -870,27 +819,8 @@ def _reposition_indices( Calculates the clamped start and end indices for a centered window. Returns a tuple of (slice_start, slice_end) compatible with python slicing. """ - # 1. Calculate ideal half-window size - half_window = total_lines // 2 - - # 2. Calculate initial left/right bounds - left = target_idx - half_window - right = target_idx + half_window - - # 3. Slide the window if it overflows boundaries - if left < start_idx: - right += start_idx - left - left = start_idx - - if right > end_idx: - left -= right - end_idx - right = end_idx - - # 4. Final safety clamp in case the range itself is smaller than total_lines - left = max(start_idx, left) - # Return right + 1 so it's ready-to-use for standard Python slicing [start:end] - return left, right + 1 + return reposition_indices(target_idx, start_idx, end_idx, total_lines) @classmethod def clear_old_messages(cls, coder): @@ -1002,6 +932,21 @@ def format_error(cls, coder, error_text, file_path, range_start, range_end, oper def on_duplicate_request(cls, coder, **kwargs): coder.edit_allowed = True + @classmethod + def ptc_format(cls, result): + """Strip placeholder entries from the result before sandbox exposure.""" + if isinstance(result, ToolResponse) and result.result_type == "list": + result._result = [ + item + for item in result._result + if not ( + isinstance(item, dict) + and isinstance(item.get("_"), dict) + and item["_"].get("status") == "placeholder" + ) + ] + return result + @classmethod def _extend_range_with_stub(cls, coder, abs_path, s_idx, e_idx, num_lines): """ @@ -1053,6 +998,411 @@ def _extend_range_with_stub(cls, coder, abs_path, s_idx, e_idx, num_lines): # Fallback to original range if anything goes wrong return s_idx, e_idx + @classmethod + def _try_fuzzy_narrow_indices( + cls, coder, abs_path, start_indices, num_lines, search_pattern=None + ): + """Try to narrow down ambiguous pattern matches using structural features. + + First attempts exact structural proximity (indices near def/class/import + lines). If that fails, falls back to rapidfuzz fuzzy matching the search + pattern against tag names from the repo map outline. + + Returns narrowed list of indices, or None if narrowing wasn't possible. + """ + try: + from cecli.repomap import RepoMap + + if not hasattr(RepoMap, "_stub_instance"): + RepoMap._stub_instance = RepoMap(map_tokens=0, io=coder.io) + rm = RepoMap._stub_instance + rel_fname = rm.get_rel_fname(abs_path) + tags = rm.get_tags(abs_path, rel_fname) + + if not tags: + return None + + # Build structural line set for proximity matching + structural_lines = { + tag.line for tag in tags if tag.kind == "def" or tag.specific_kind == "import" + } + + if structural_lines: + narrowed = [] + for si in start_indices: + for sl in structural_lines: + if abs(si - sl) <= 3: + narrowed.append(si) + break + + if narrowed: + return sorted(set(narrowed)) + + # Structural proximity didn't narrow — try fuzzy name matching + if search_pattern: + from rapidfuzz import fuzz + + # Collect def tags with their names and lines + def_tags = [ + (tag.name, tag.line) + for tag in tags + if tag.kind == "def" and getattr(tag, "name", None) + ] + + if def_tags: + # Fuzzy match the search pattern against each tag name + matched_lines = [] + for tag_name, tag_line in def_tags: + score = fuzz.partial_ratio(search_pattern.lower(), tag_name.lower()) + if score >= 70: + matched_lines.append(tag_line) + + if matched_lines: + # Return start_indices that are near fuzzy-matched structural lines + narrowed = [] + for si in start_indices: + for ml in matched_lines: + if abs(si - ml) <= 5: + narrowed.append(si) + break + + if narrowed: + return sorted(set(narrowed)) + + # No start_indices near fuzzy-matched lines — + # return the matched lines themselves as anchors + return sorted(set(matched_lines)) + + return None + except Exception: + return None + + @classmethod + def _classify_search_type(cls, range_start, range_end): + """Classify range markers into structured, text, mixed, or contextual search types.""" + + return classify_search_type(range_start, range_end) + + @classmethod + def _extract_l_hint(cls, pattern, lines=None): + """Extract hint suffix from a pattern string. + + Supports @L (direct line number), @A{{regex}} (filter to matches AFTER + the first regex match), and @B{{regex}} (filter to matches BEFORE the last + regex match) hints. + + Returns (stripped_pattern, hint_value, hint_type) where: + - hint_value is a 0-based line number or None + - hint_type is 'L', 'A', 'B', or None + """ + + return extract_hint(pattern, lines) + + @classmethod + def _search_in_lines(cls, lines, pattern, return_last_line=False): + """Search for a multiline pattern in lines. + + Returns list of matching indices. When return_last_line is True, + returns the index of the last line of each match instead of the first. + + Delegates to the core ``search_in_lines`` in + ``cecli.helpers.hashpos.transformations`` for unified matching. + """ + + return search_in_lines(lines, pattern, return_last_line=return_last_line) + + @classmethod + def _find_start_indices(cls, lines, range_start, classification, num_lines): + """Resolve start indices based on the classified search type.""" + if classification["start_is_line_ref"]: + line_num = int(range_start[2:]) - 1 + return [max(0, min(line_num, num_lines - 1))] + + if classification["start_is_special"]: + return [0] if range_start == "@000" else [num_lines - 1] + + return cls._search_in_lines(lines, range_start) + + @classmethod + def _find_end_indices(cls, lines, range_end, classification, num_lines): + """Resolve end indices based on the classified search type.""" + if classification["end_is_line_ref"]: + line_num = int(range_end[2:]) - 1 + return [max(0, min(line_num, num_lines - 1))] + + if classification["end_is_special"]: + return [0] if range_end == "@000" else [num_lines - 1] + + return cls._search_in_lines(lines, range_end, return_last_line=True) + + @classmethod + def _apply_contextual_marker( + cls, + start_indices, + range_start, + range_end, + num_lines, + coder, + file_path, + read_index, + ): + """Expand range using @C/@P/@N contextual markers. + + Returns ((new_start_indices, new_end_indices), error). + Error is None on success, or a formatted error string on failure. + """ + + try: + result = apply_contextual_marker(start_indices, range_start, range_end, num_lines) + + return result, None + + except ValueError as e: + error = cls.format_error( + coder, + str(e), + file_path, + range_start, + range_end, + read_index, + ) + + return None, error + + @classmethod + def _disambiguate_start_indices( + cls, + start_indices, + end_indices, + abs_path, + range_start, + num_lines, + rel_path, + read_index, + coder, + response, + start_hint=None, + ): + """Narrow ambiguous start_indices when there are too many matches. + + Tries @L hint proximity narrowing first, then fuzzy structural + narrowing, falls back to first 5 with a warning listing + line numbers. Last-invocation disambiguation is handled separately + in _resolve_to_final_indices. + """ + + if len(start_indices) <= 5: + return start_indices + + last = cls._last_invocation.get(abs_path) + if last is not None: + # Last-invocation disambiguation happens in _resolve_to_final_indices + return start_indices + + # @L hint: filter to indices closest to the hinted line + if start_hint is not None: + proximity_narrowed = cls._narrow_by_proximity(start_indices, start_hint, max_results=5) + if proximity_narrowed and len(proximity_narrowed) <= 5: + line_nums = [str(i + 1) for i in sorted(proximity_narrowed)] + note_text = ( + f"read operation {read_index + 1}: start pattern " + f"'{range_start}' matched many locations in {rel_path}; " + f"narrowed to {len(proximity_narrowed)} closest to @L hint " + f"at lines {', '.join(line_nums)}. " + f"Tip: append ' @L' to any pattern (e.g., " + f"'{range_start} @L{start_hint + 1}') to target a specific match." + ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) + return proximity_narrowed + + narrowed = cls._try_fuzzy_narrow_indices( + coder, + abs_path, + start_indices, + num_lines, + search_pattern=range_start, + ) + + if narrowed and len(narrowed) <= 5: + line_nums = [str(i + 1) for i in sorted(narrowed)] + note_text = ( + f"read operation {read_index + 1}: start pattern " + f"'{range_start}' matched many locations in {rel_path}; " + f"narrowed to {len(narrowed)} structural match(es) " + f"at lines {', '.join(line_nums)}. " + f"Tip: append ' @L' to a pattern (e.g., " + f"'{range_start} @L{line_nums[0]}') to target a specific match." + ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) + return narrowed + + line_nums = [str(i + 1) for i in sorted(start_indices[:5])] + note_text = ( + f"read operation {read_index + 1}: start pattern " + f"'{range_start}' too broad ({len(start_indices)} matches) " + f"in {rel_path}. Using first 5 (lines {', '.join(line_nums)}). " + f"Refine with specific names or @L ranges for precision." + ) + response.append_result( + content=note_text, + metadata={ + "file_path": rel_path, + "status": "placeholder", + "total_lines": num_lines, + "outline": "", + }, + ) + return start_indices[:5] + + @classmethod + def _narrow_by_proximity(cls, indices, target, max_results=5): + """Narrow a list of indices to those closest to a target index. + + Returns up to max_results indices sorted by ascending distance + from the target (0-based line number). + """ + + return narrow_by_proximity(indices, target, max_results=max_results) + + @classmethod + def _resolve_to_final_indices( + cls, + start_indices, + end_indices, + num_lines, + coder, + abs_path, + range_start, + range_end, + file_path, + read_index, + ): + """Find the best (start, end) pair and handle fallbacks. + + Returns (s_idx, e_idx, errors) where errors is a list of formatted + error strings. On success, errors is empty and s_idx/e_idx are set. + """ + errors = [] + last = cls._last_invocation.get(abs_path) + + # Find best pair (with last-invocation disambiguation if available) + if last and len(start_indices) > 5: + last_s, last_e = last["start_idx"], last["end_idx"] + candidates = [] + for s in start_indices: + for e in [idx for idx in end_indices if idx >= s]: + dist_sum = abs(s - last_s) + abs(e - last_e) + candidates.append((dist_sum, s, e)) + candidates.sort(key=lambda x: (x[0], x[1] < last_s, x[1], x[2])) + best_pair = (candidates[0][1], candidates[0][2]) if candidates else None + else: + best_pair = cls._compute_best_pair(start_indices, end_indices) + + # Inverted matching: LLM may have swapped start/end pattern order + if best_pair is None and (len(start_indices) == 1 or len(end_indices) == 1): + best_pair = cls._compute_best_pair(start_indices, end_indices, allow_inverted=True) + + # Error: start pattern not found + if not start_indices: + errors.append( + cls.format_error( + coder, + f"Start pattern '{range_start}' not found in {file_path}. Refine your search.", + file_path, + range_start, + range_end, + read_index, + ) + ) + return None, None, errors + + # Error: end pattern not found — expand from start + if not end_indices: + if start_indices: + s_idx = start_indices[0] + try: + s_idx, e_idx = cls._extend_range_with_stub( + coder, + abs_path, + s_idx, + num_lines - 1, + num_lines, + ) + except Exception: + e_idx = num_lines - 1 + return s_idx, e_idx, [] + else: + errors.append( + cls.format_error( + coder, + f"End pattern '{range_end}' not found in {file_path}. Refine your search.", + file_path, + range_start, + range_end, + read_index, + ) + ) + return None, None, errors + + # Both matched but no valid pair — expand from start + if best_pair is None: + if start_indices: + s_idx = start_indices[0] + try: + s_idx, e_idx = cls._extend_range_with_stub( + coder, + abs_path, + s_idx, + num_lines - 1, + num_lines, + ) + except Exception: + e_idx = num_lines - 1 + return s_idx, e_idx, [] + else: + errors.append( + cls.format_error( + coder, + ( + f"End pattern '{range_end}' not found after start pattern" + f" in {file_path}." + ), + file_path, + range_start, + range_end, + read_index, + ) + ) + return None, None, errors + + return best_pair[0], best_pair[1], [] + + @classmethod + def _compute_best_pair(cls, start_indices, end_indices, allow_inverted=False): + """Find the smallest-range valid (start, end) pair. + + When allow_inverted is True, also considers pairs where end < start + (LLM swapped the order) and returns them in correct order. + """ + + return compute_best_pair(start_indices, end_indices, allow_inverted=allow_inverted) + @classmethod def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=True): """Get a preview of a large file range between start_idx and end_idx. @@ -1077,24 +1427,36 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr io = coder.io abs_path, rel_path = resolve_paths(coder, abs_path) + content = io.read_text(abs_path) + stub = RepoMap.get_file_stub( abs_path, io, start_line=start_idx, end_line=end_idx, line_numbers=line_numbers ) # If get_file_stub returned a useful structural outline, wrap it as JSON if stub and stub != "# No outline available": - result = json.dumps( - { - "file_path": rel_path, - "outline": stub, - }, - ensure_ascii=False, - ) - return result, True + return { + "file_path": rel_path, + "status": "outline", + "total_lines": len(content.splitlines()), + "prefixed_contents": "", + "outline": stub, + "note": ( + f"Large File. Tip: use @L ranges for precise reads" + f" (e.g., @L{start_idx + 1}, @L{end_idx + 1})." + ), + }, True content = io.read_text(abs_path) if not content: - return "" + return { + "file_path": rel_path, + "status": "outline", + "total_lines": 0, + "prefixed_contents": "", + "outline": "", + "note": "Empty file.", + }, False lines = content.splitlines() num_file_lines = len(lines) @@ -1104,7 +1466,14 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr total_lines = actual_end - actual_start + 1 if total_lines <= 0: - return "", False + return { + "file_path": rel_path, + "status": "outline", + "total_lines": 0, + "prefixed_contents": "", + "outline": "", + "note": "Invalid range.", + }, False if total_lines <= 20: # Return all lines @@ -1125,27 +1494,33 @@ def _get_range_preview(cls, coder, abs_path, start_idx, end_idx, line_numbers=Tr if sample_lines and sample_lines[-1][0] != actual_end: sample_lines.append((actual_end, lines[actual_end])) - # Format the output parts = [ f"File range too large ({total_lines} lines).", + ( + f"Tip: use @L ranges for precise reads" + f" (e.g., @L{actual_start + 1}, @L{actual_end + 1})." + ), f"Showing {len(sample_lines)} equally-spaced lines from the range:", "", ] - file_contents = [] for idx, line_content in sample_lines: line_num = idx + 1 file_contents.append(f"{line_num}|{line_content}") file_contents.append("...") - parts.append( - json.dumps( - { - "file_path": rel_path, - "truncated": "\n".join(file_contents), - }, - ensure_ascii=False, - ) - ) + parts.append(f"file_path: {rel_path}") + parts.append("truncated:") + parts.append("\n".join(file_contents)) - return "\n".join(parts), False + return { + "file_path": rel_path, + "status": "outline", + "total_lines": num_file_lines, + "prefixed_contents": "", + "outline": "\n".join(parts), + "note": ( + f"Tip: use @L ranges for precise reads" + f" (e.g., @L{actual_start + 1}, @L{actual_end + 1})." + ), + }, False diff --git a/cecli/tools/resource_manager.py b/cecli/tools/resource_manager.py index a58a232f12e..a507601cc1b 100644 --- a/cecli/tools/resource_manager.py +++ b/cecli/tools/resource_manager.py @@ -11,11 +11,13 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError, parse_arg_as_list from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "resourcemanager" + RESULT_TYPE = "list" SCHEMA = { "type": "function", "function": { @@ -168,7 +170,7 @@ async def execute( ) coder.io.tool_output("⛭ Modifying Context", type="tool-result") - messages = [] + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) # Expand wildcards for MCP operations if "*" in load_mcp_servers and coder.mcp_manager: @@ -208,11 +210,11 @@ async def execute( c.registered_servers["included"] = included for f in create_files: - messages.append(cls._create(coder, f)) + response.append_result(cls._create(coder, f)) for f in remove_files: - messages.append(cls._remove(coder, f)) + response.append_result(cls._remove(coder, f)) for f in view_files: - messages.append(cls._view(coder, f)) + response.append_result(cls._view(coder, f)) for f in editable_files: try: abs_path = coder.abs_root_path(f) @@ -220,25 +222,25 @@ async def execute( abs_path = None if abs_path is not None and not os.path.isfile(abs_path): coder.io.tool_output(f"ℹ️ `{f}` missing on disk — using **create** instead of add") - messages.append(cls._create(coder, f)) + response.append_result(cls._create(coder, f)) else: - messages.append(cls._editable(coder, f)) + response.append_result(cls._editable(coder, f)) for key in stop_keys: - messages.append(cls._stop_command(coder, key)) + response.append_result(cls._stop_command(coder, key)) for skill_name in load_skill_names: - messages.append(cls._load_skill(coder, skill_name)) + response.append_result(cls._load_skill(coder, skill_name)) for skill_name in remove_skill_names: - messages.append(cls._remove_skill(coder, skill_name)) + response.append_result(cls._remove_skill(coder, skill_name)) for server_name in load_mcp_servers: result = await cls._load_mcp(coder, server_name) - messages.append(result) + response.append_result(result) for server_name in remove_mcp_servers: result = await cls._remove_mcp(coder, server_name) - messages.append(result) + response.append_result(result) for action_name in action_operations: result = await cls._list_mcp_servers(coder) - messages.append(result) + response.append_result(result) tui = getattr(coder, "tui", None) if tui and tui(): @@ -247,7 +249,7 @@ async def execute( coder.context_blocks_cache = {} coder.edit_allowed = True - return "\n".join(messages) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/thinking.py b/cecli/tools/thinking.py index 3cedc9eae6f..43bd8c342dd 100644 --- a/cecli/tools/thinking.py +++ b/cecli/tools/thinking.py @@ -1,6 +1,7 @@ from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import color_markers, tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -34,8 +35,10 @@ def execute(cls, coder, content, **kwargs): A place to allow the model to record freeform text as it iterates over tools to ideally help it guide itself to a proper solution """ + response = ToolResponse(cls.NORM_NAME) coder.io.tool_output("🧠 Thoughts recorded in context", type="tool-result") - return "🧠 Thoughts recorded in context. Please proceed with your task" + response.append_result("🧠 Thoughts recorded in context. Please proceed with your task") + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/undo_change.py b/cecli/tools/undo_change.py index 67a4596362b..28489a62562 100644 --- a/cecli/tools/undo_change.py +++ b/cecli/tools/undo_change.py @@ -1,6 +1,7 @@ import traceback from cecli.tools.utils.base_tool import BaseTool +from cecli.tools.utils.responses import ToolResponse class Tool(BaseTool): @@ -34,12 +35,14 @@ def execute(cls, coder, change_id=None, file_path=None, **kwargs): """ # Note: Undo does not have a dry_run parameter as it's inherently about reverting a previous action. coder.edit_allowed = False + response = ToolResponse(cls.NORM_NAME) try: # Validate parameters if change_id is None and file_path is None: coder.io.tool_error("Must specify either change_id or file_path for UndoChange") - return "Error: Must specify either change_id or file_path" + response.append_result("Error: Must specify either change_id or file_path") + return response # If file_path is specified, get the most recent change for that file if file_path: @@ -49,14 +52,16 @@ def execute(cls, coder, change_id=None, file_path=None, **kwargs): change_id = coder.change_tracker.get_last_change(rel_path) if not change_id: coder.io.tool_error(f"No tracked changes found for file '{file_path}' to undo.") - return f"Error: No changes found for file '{file_path}'" + response.append_result(f"Error: No changes found for file '{file_path}'") + return response # Attempt to get undo information from the tracker success, message, change_info = coder.change_tracker.undo_change(change_id) if not success: coder.io.tool_error(f"Failed to undo change '{change_id}': {message}") - return f"Error: {message}" + response.append_result(f"Error: {message}") + return response # Apply the undo by restoring the original content if change_info: @@ -72,15 +77,20 @@ def execute(cls, coder, change_id=None, file_path=None, **kwargs): coder.io.tool_output( f"✓ Undid {change_type} change '{change_id}' in {file_path}", type="tool-result" ) - return f"Successfully undid {change_type} change '{change_id}'." + response.append_result(f"Successfully undid {change_type} change '{change_id}'.") + return response else: # This case should ideally not be reached if tracker returns success coder.io.tool_error( f"Failed to undo change '{change_id}': Change info missing after successful" " tracker update." ) - return f"Error: Failed to undo change '{change_id}' (missing change info)" + response.append_result( + f"Error: Failed to undo change '{change_id}' (missing change info)" + ) + return response except Exception as e: coder.io.tool_error(f"Error in UndoChange: {str(e)}\n{traceback.format_exc()}") - return f"Error: {str(e)}" + response.append_result(f"Error: {str(e)}") + return response diff --git a/cecli/tools/update_todo_list.py b/cecli/tools/update_todo_list.py index 812068f0c13..fd7ed7c6ac3 100644 --- a/cecli/tools/update_todo_list.py +++ b/cecli/tools/update_todo_list.py @@ -1,11 +1,13 @@ from cecli.tools.utils.base_tool import BaseTool -from cecli.tools.utils.helpers import ToolError, format_tool_result, handle_tool_error +from cecli.tools.utils.helpers import ToolError from cecli.tools.utils.output import tool_footer, tool_header +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations class Tool(BaseTool): NORM_NAME = "updatetodolist" + RESULT_TYPE = "list" VALIDATIONS = { "tasks": ["coerce_list"], "tasks[]": ["coerce_dict"], @@ -59,7 +61,8 @@ def execute(cls, coder, tasks, append=False, change_id=None, dry_run=False, **kw Update the todo list file (todo.txt) with formatted task items. Can either replace the entire content or append to it. """ - tool_name = "UpdateTodoList" + # tool_name = "UpdateTodoList" + response = ToolResponse(cls.NORM_NAME, result_type=cls.RESULT_TYPE) try: # Define the todo file path todo_file_path = coder.local_agent_folder("todo.txt") @@ -130,18 +133,18 @@ def execute(cls, coder, tasks, append=False, change_id=None, dry_run=False, **kw # Check if content actually changed if existing_content == new_content: coder.io.tool_warning("No changes made: new content is identical to existing") - return ( + response.append_result( "Error: No changes made (content identical to existing)." - "Please make progress implementing the plan instead of updating it." + " Please make progress implementing the plan instead of updating it." ) + return response # Handle dry run if dry_run: action = "append to" if append else "replace" dry_run_message = f"Dry run: Would {action} todo list in {todo_file_path}." - return format_tool_result( - coder, tool_name, "", dry_run=True, dry_run_message=dry_run_message - ) + response.append_result(dry_run_message) + return response # Apply change metadata = { @@ -183,12 +186,15 @@ def execute(cls, coder, tasks, append=False, change_id=None, dry_run=False, **kw # success_message, # change_id=final_change_id, # ) - return new_content + response.append_result(new_content) + return response except ToolError as e: - return handle_tool_error(coder, tool_name, e, add_traceback=False) + response.append_error(str(e)) + return response except Exception as e: - return handle_tool_error(coder, tool_name, e) + response.append_error(str(e)) + return response @classmethod def format_output(cls, coder, mcp_server, tool_response): diff --git a/cecli/tools/utils/base_tool.py b/cecli/tools/utils/base_tool.py index a9fb39c709a..b749c148ad6 100644 --- a/cecli/tools/utils/base_tool.py +++ b/cecli/tools/utils/base_tool.py @@ -2,6 +2,7 @@ from cecli.tools.utils.helpers import handle_tool_error from cecli.tools.utils.output import print_tool_response +from cecli.tools.utils.responses import ToolResponse from cecli.tools.validations import ToolValidations @@ -16,6 +17,9 @@ class BaseTool(ABC): # Declarative validations (maps param paths to lists of validation method names) VALIDATIONS = {} + # Result format configuration ("str" or "list") + RESULT_TYPE = "str" + # Invocation tracking for detecting repeated tool calls _invocations = {} # Dict to store last 3 invocations per tool _invocation_summary = set() # Set to track distinct tool names @@ -37,16 +41,18 @@ def execute(cls, coder, **params): pass @classmethod - def process_response(cls, coder, params): + def process_response(cls, coder, params, _convert=True): """ Process the tool response by creating an instance and calling execute. Args: coder: The Coder instance params: Dictionary of parameters + _convert: If True (default), ToolResponse results are converted to str. + If False, ToolResponse is returned as-is for sandbox use. Returns: - str: Result message + str or ToolResponse: Result message (or ToolResponse when _convert=False) """ # Validate required parameters from SCHEMA @@ -84,7 +90,7 @@ def process_response(cls, coder, params): return handle_tool_error(coder, tool_name, ValueError(error_msg)) # Check for repeated invocations if TRACK_INVOCATIONS is enabled - if cls.TRACK_INVOCATIONS: + if cls.TRACK_INVOCATIONS and _convert: tool_name = None if cls.SCHEMA and "function" in cls.SCHEMA: tool_name = cls.SCHEMA["function"].get("name", "Unknown Tool") @@ -133,7 +139,12 @@ def process_response(cls, coder, params): params = ToolValidations.validate_params(params, cls.VALIDATIONS, cls.SCHEMA) try: - return cls.execute(coder, **params) + result = cls.execute(coder, **params) + + if isinstance(result, ToolResponse): + return result if not _convert else str(result) + + return result except Exception as e: return handle_tool_error(coder, cls.SCHEMA.get("function").get("name"), e) @@ -154,3 +165,20 @@ def on_duplicate_request(cls, coder, **kwargs): def clear_invocation_cache(cls): cls._invocations.clear() cls._invocation_summary.clear() + + @classmethod + def ptc_format(cls, result): + """Post-tool-call formatting hook for orchestration sandbox exposure. + + By default, passes through the result unchanged. Tools can override + this to reshape the output that gets exposed to the orchestration + sandbox (e.g., stripping placeholder entries). + + Args: + result: The result from ``execute()`` (typically a ``ToolResponse`` + or string). + + Returns: + The (possibly modified) result. + """ + return result diff --git a/cecli/tools/utils/responses.py b/cecli/tools/utils/responses.py new file mode 100644 index 00000000000..561f78d0496 --- /dev/null +++ b/cecli/tools/utils/responses.py @@ -0,0 +1,115 @@ +import json + + +class ToolResponse: + """Assists in formatting all tool call responses as JSON for the LLM. + + Every tool response is wrapped in JSON: + + { + "result": [{"content": ..., "_": {...}}], + "errors": [...], + "details": [...] + } + + Each result item has a ``content`` key (the primary output) and an + ``_`` key (metadata). Use ``append_result(content, metadata=None)`` + + Usage inside a tool's ``execute`` method: + + response = ToolResponse("my_tool", result_type="str") + response.append_result("Operation completed successfully") + response.append_error("Minor warning: config file missing") + response.append_detail("Extra context for the LLM") + return response # __str__ returns valid JSON + + Tools that set ``RESULT_TYPE = "list"`` at the class level will have + result entries accumulated as a list instead of concatenated. + """ + + def __init__(self, tool_name, result_type="str"): + self.tool_name = tool_name + self.result_type = result_type + self._result = "" if result_type == "str" else [] + self._errors = [] + self._details = [] + + def append_result(self, content, metadata=None): + """Append a result entry. + + If ``result_type`` is ``"str"`` the text is concatenated (with a + newline separator for subsequent calls). If ``result_type`` is + ``"list"`` each call adds a new item to the results list. + + When metadata is provided (``result_type="list"`` only), it is + stored under the ``_`` key alongside ``content``. + + Backward compat: if ``content`` is a dict *without* a ``content`` + key it is treated as the content value and metadata defaults to + an empty dict. + """ + + if self.result_type == "str": + if self._result: + self._result += "\n" + str(content) + else: + self._result = str(content) + else: + item = {"content": content, "_": metadata or {}} + self._result.append(item) + + def append_error(self, error): + """Collect an error message.""" + + self._errors.append(str(error)) + + def append_detail(self, detail): + """Collect an extraneous contextual detail for the LLM.""" + + self._details.append(str(detail)) + + def to_dict(self): + """Return the response as a plain Python dictionary.""" + + if self.result_type == "str": + results = [{"content": self._result, "_": {}}] if self._result else [] + else: + results = self._result + + return { + "result": results, + "errors": self._errors, + "details": self._details, + } + + def to_json(self): + """Serialize the response to a JSON string.""" + + return json.dumps(self.to_dict()) + + def __str__(self): + return self.to_json() + + @staticmethod + def wrap(tool_name, result, errors=None): + """Convenience that returns a populated ToolResponse. + + Useful when wrapping results from non-ToolResponse-aware code + paths (e.g., MCP tool execution). + + Args: + tool_name: Name of the tool. + result: The result string to wrap. + errors: Optional list of error strings. + + Returns: + ToolResponse instance. + """ + + response = ToolResponse(tool_name) + response.append_result(result) + if errors: + for error in errors: + response.append_error(error) + + return response diff --git a/cecli/tools/validations/validations.py b/cecli/tools/validations/validations.py index 15fc22ab460..e6dc8e5cd90 100644 --- a/cecli/tools/validations/validations.py +++ b/cecli/tools/validations/validations.py @@ -8,7 +8,7 @@ for list iteration) to lists of validation method names that are executed sequentially on the parameter value. -Example:: +Example: VALIDATIONS = { "delegations": ["coerce_list"], diff --git a/cecli/utils.py b/cecli/utils.py index b8a009c07eb..20bc2106e15 100644 --- a/cecli/utils.py +++ b/cecli/utils.py @@ -10,7 +10,6 @@ from pathlib import Path from cecli.dump import dump # noqa: F401 -from cecli.waiting import Spinner IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".webp", ".pdf"} @@ -311,6 +310,8 @@ def run_install(cmd): pass # Continue even if ensurepip fails try: + from cecli.waiting import Spinner + output = [] process = subprocess.Popen( cmd, diff --git a/cecli/waiting.py b/cecli/waiting.py index 94ee7f01902..a4bf5e296b0 100644 --- a/cecli/waiting.py +++ b/cecli/waiting.py @@ -4,13 +4,13 @@ A simple wrapper for rich.status to provide a spinner. """ -from rich.console import Console - class Spinner: """A wrapper around rich.status.Status for displaying a spinner.""" def __init__(self, text: str = "Waiting..."): + from rich.console import Console + self.text = text self.console = Console() self.status = None diff --git a/cecli/website/docs/config/agent-mode.md b/cecli/website/docs/config/agent-mode.md index 2d571ee697f..92982d52cc2 100644 --- a/cecli/website/docs/config/agent-mode.md +++ b/cecli/website/docs/config/agent-mode.md @@ -166,6 +166,55 @@ Agent Mode can also be configured directly in your configuration file. See the [ - **`command_timeout`**: Time in seconds to wait for shell commands to finish before automatic backgrounding occurs (default: None) - **`diff_colors`**: When enabled, diff output in edit tool responses uses color-coded lines - removed lines in magenta, added lines in light green, and context lines in plain text (default: true) +- **`orchestration`**: A nested configuration object for the Orchestrate tool's Python sandbox. + When absent or empty, the sandbox runs with default restrictions. See [Orchestration + Configuration](#orchestration-configuration) below for details. + +#### Orchestration Configuration + +The `Orchestrate` tool runs user code in a secure Python sandbox with the following +restrictions: +- **No imports** — only pre-imported modules (`re`, `math`, `itertools`, `collections`, + `datetime`, `traceback`, `json`, `pathlib`) are available +- **No private/dunder access** — attributes starting with `_` are blocked +- **No dangerous builtins** — `eval`, `exec`, `open`, `compile`, `breakpoint`, + `__import__`, `globals`, `locals`, `setattr`, `delattr` are blocked +- **No global/nonlocal statements** +- **Loop protection** — `while`/`for` loops yield cooperatively to prevent hangs + +These restrictions can be selectively relaxed via the `orchestration` config block: + +- **`allowed_imports`**: Array of module names to allow importing. Example: + `["os", "typing"]`. Only standard library modules should be allowed; third-party + modules may execute arbitrary code at import time. + +- **`allowed_builtins`**: Array of builtin function names to add to the sandbox. + Example: `["setattr", "property"]`. Dangerous builtins (`eval`, `exec`, `open`, etc.) + cannot be added. + +- **`allow_classes`**: Boolean (default: `false`). When `true`, class definitions are + permitted and dunder methods (`__init__`, `__str__`, `__repr__`, `__iter__`, etc.) + are allowed inside class bodies. + +- **`disable_security`**: Boolean (default: `false`). When `true`, the AST-level security + filter is skipped entirely. ⚠ Use with extreme caution — this disables all import + blocking, dunder blocking, and dangerous builtin blocking. + +- **`disable_loop_protection`**: Boolean (default: `false`). When `true`, the cooperative + yield injection is skipped. Use only if you are certain the orchestration code has no + unbounded loops. + +Example: + +```yaml +agent-config: + orchestration: + allowed_imports: + - os + - typing + allow_classes: true +``` + #### Essential Tools Certain tools are always available regardless of includelist/excludelist settings: @@ -275,7 +324,7 @@ agent-config: # Sub-agent configuration subagent_paths: [".cecli/subagents"] # Optional: Directories to search for sub-agent definitions - max_sub_agents: 3 # Optional: Maximum concurrent sub-agents (default: 3) + max_sub_agents: 30 # Optional: Maximum concurrent sub-agents (default: 3) allow_nested_delegation: false # Optional: Allow sub-agents to delegate further (default: false) # Context blocks configuration @@ -288,7 +337,16 @@ agent-config: allowed_commands: ["wc -l*"] # Commands matching these glob patterns will not prompt for confirmation show_lint_errors: false # When enabled, linting errors are shown in tool output (default: false) hot_reload: false # When enabled, skills configuration is hot-reloaded automatically (default: false) -фвδ:: diff_colors: true # When enabled, diff output uses color-coded lines (default: true) + diff_colors: true # When enabled, diff output uses color-coded lines (default: true) + + # Orchestration sandbox configuration (optional) + orchestration: + allowed_imports: [] # Optional: Module names to allow importing + allowed_builtins: [] # Optional: Builtin names to add + allow_classes: false # Optional: Allow class definitions + disable_security: false # Optional: Disable security filter (⚠ dangerous) + disable_loop_protection: false # Optional: Disable loop yield injection + # Skills configuration (see Skills documentation for details) skills_paths: ["~/my-skills", "./project-skills"] # Directories to search for skills skills_includelist: ["python-refactoring", "react-components"] # Optional: Whitelist of skills to include diff --git a/cecli/website/docs/config/subagents.md b/cecli/website/docs/config/subagents.md index da0585a527b..46b6db5d4f9 100644 --- a/cecli/website/docs/config/subagents.md +++ b/cecli/website/docs/config/subagents.md @@ -56,7 +56,7 @@ Add sub-agent paths to your YAML configuration file: ```yaml # .cecli.conf.yml or ~/.cecli.conf.yml agent-config: - max_sub_agents: 3 # Maximum concurrent sub-agents (default: 3) + max_sub_agents: 30 # Maximum concurrent sub-agents (default: 3) subagent_paths: - ".cecli/subagents" # Default path - "~/team-agents" # Custom path for shared agent definitions @@ -141,7 +141,7 @@ Each agent gets its own output container. When you switch agents: ### Max Sub-Agents -The `max_sub_agents` setting (default: 3) limits how many concurrent sub-agents can exist. This prevents resource exhaustion. +The `max_sub_agents` setting (default: 30) limits how many concurrent sub-agents can exist. This prevents resource exhaustion. When the limit is reached: diff --git a/scripts/get_hashline.py b/scripts/get_hashline.py index 68f5e75787d..eb3765133ef 100755 --- a/scripts/get_hashline.py +++ b/scripts/get_hashline.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import sys +import traceback from pathlib import Path # Add the current directory to sys.path to allow importing from cecli @@ -50,6 +51,7 @@ def main(): print("\n\nSummary: No duplicate hash position hashes found.", file=sys.stderr) except Exception as e: + traceback.print_exc() print(f"Error reading file: {e}") sys.exit(1) diff --git a/scripts/optimize_tokens.sh b/scripts/optimize_tokens.sh index f7157d0a759..91151ce0605 100755 --- a/scripts/optimize_tokens.sh +++ b/scripts/optimize_tokens.sh @@ -4,57 +4,103 @@ echo "Calculating Ideal Tokens For Tokenization" # Write the Python script to a temporary file cat << 'EOF' > /tmp/table_optimizer.py -import string import pprint import tiktoken +import unicodedata from transformers import AutoTokenizer -def generate_universal_prefix_free_tokens(target_count=256): +def find_anthropic_safe_tokens_with_fallback(target_count=1024): print("Loading tokenizers... (This will download vocab files to your cache)") - # 1. OpenAI Baseline - enc_openai = tiktoken.get_encoding("cl100k_base") + enc_cl100k = tiktoken.get_encoding("cl100k_base") + enc_o200k = tiktoken.get_encoding("o200k_base") - # 2. Stable Open-Weight Tokenizers (Publicly Accessible) tokenizers = { - "mistral": AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1"), "qwen": AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B"), - "deepseek": AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-6.7b-instruct") + "mistral_nemo": AutoTokenizer.from_pretrained("mistralai/Mistral-Nemo-Base-2407"), + "phi4": AutoTokenizer.from_pretrained("microsoft/Phi-4-mini-instruct", trust_remote_code=True), + "glm4": AutoTokenizer.from_pretrained("THUDM/glm-4-9b-chat", trust_remote_code=True), + "deepseek_v3": AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V3", trust_remote_code=True), } - - # Base36: 0-9, a-z (36 characters total) - base36 = list(string.digits + string.ascii_lowercase) - - direct_chars = base36.copy() - prefix_pairs = [] - - def is_single_token_everywhere(text): - if len(enc_openai.encode(text)) != 1: - return False - for name, enc in tokenizers.items(): - # add_special_tokens=False is critical for correct subword counting - if len(enc.encode(text, add_special_tokens=False)) != 1: - return False - return True - print("Calculating intersection of all tokenizers...") + print("Extracting candidates into Primary and Secondary (Fallback) pools...") + primary_candidates = set() + secondary_candidates = set() - while len(direct_chars) + len(prefix_pairs) < target_count: - if not direct_chars: - raise ValueError("Stricter rules exhausted the alphabet. Please switch to Base62 (A-Z, a-z, 0-9).") + for token_id in range(enc_o200k.n_vocab): + try: + token_bytes = enc_o200k.decode_single_token_bytes(token_id) + text = token_bytes.decode('utf-8') - prefix = direct_chars.pop() - for second_char in base36: - seq = prefix + second_char - if is_single_token_everywhere(seq): - prefix_pairs.append(seq) + if len(text) == 1 and text.isprintable(): + char_code = ord(text) + + # 1. ASCII must be strictly alphanumeric + if char_code <= 0x007F and not text.isalnum(): + continue + + # 2. PREVENT COMBINING CHARACTER COLLAPSE + # Drops all attaching Marks (Mn, Mc, Me) and invisible format Controls (Cf, Cc) + category = unicodedata.category(text) + if category.startswith('M') or category.startswith('C'): + continue + + # 3. DECOMPOSITION ROUTING + if unicodedata.decomposition(text): + # Has hidden structural parts -> send to Fallback Pool + secondary_candidates.add(text) + else: + # Pure, unbreakable character -> send to Primary Pool + primary_candidates.add(text) + + except (KeyError, UnicodeDecodeError): + pass + + # Sort to sequentially process alphabets from standard to complex + primary_sorted = sorted(list(primary_candidates), key=ord) + secondary_sorted = sorted(list(secondary_candidates), key=ord) + + print(f"Extracted {len(primary_sorted)} Primary and {len(secondary_sorted)} Secondary candidates.") + + shared_tokens = [] + + # Helper function to process a specific pool + def process_pool(candidate_list, pool_name): + print(f"\n--- Scanning {pool_name} Pool ---") + for text in candidate_list: + if len(enc_cl100k.encode(text)) != 1: + continue + + is_shared = True + for enc in tokenizers.values(): + if len(enc.encode(text, add_special_tokens=False)) != 1: + is_shared = False + break + + if is_shared: + shared_tokens.append(text) + if len(shared_tokens) % 64 == 0: + print(f"Secured {len(shared_tokens)} / {target_count} universal tokens...") - return (direct_chars + prefix_pairs)[:target_count] + if len(shared_tokens) == target_count: + break + + # 1. Exhaust the Primary Pool first + process_pool(primary_sorted, "PRIMARY (Strictly Safe)") + + # 2. If we fell short (e.g. 1009/1024), dip into the Secondary Pool + if len(shared_tokens) < target_count: + print(f"\nTarget not met ({len(shared_tokens)}/{target_count}). Dipping into SECONDARY fallback pool...") + process_pool(secondary_sorted, "SECONDARY (Decomposed)") + + return shared_tokens if __name__ == "__main__": try: - optimal_tokens = generate_universal_prefix_free_tokens(256) - print("\nBASE256_UNIVERSAL_CODES = ", end="") + optimal_tokens = find_anthropic_safe_tokens_with_fallback(1024) + print("\n" + "="*50) + print(f"FOUND {len(optimal_tokens)} UNIVERSAL TOKENS (SORTED BY UNICODE)") + print("="*50) pprint.pprint(optimal_tokens, compact=True, width=100) except Exception as e: print(f"Error: {e}") diff --git a/tests/basic/test_coder.py b/tests/basic/test_coder.py index 3067e785135..872503b8e50 100644 --- a/tests/basic/test_coder.py +++ b/tests/basic/test_coder.py @@ -34,7 +34,7 @@ class TestCoder: def setup(self, gpt35_model): self.uuid = str(uuid.uuid4()) self.GPT35 = gpt35_model - self.webbrowser_patcher = patch("cecli.io.webbrowser.open") + self.webbrowser_patcher = patch("webbrowser.open") self.mock_webbrowser = self.webbrowser_patcher.start() # Reset conversation system before each test ConversationService.get_chunks(self).reset() diff --git a/tests/basic/test_hashline.py b/tests/basic/test_hashline.py index 527762172e9..f058e8ad4d0 100644 --- a/tests/basic/test_hashline.py +++ b/tests/basic/test_hashline.py @@ -15,14 +15,12 @@ def test_hashline_basic(): lines = result.splitlines() assert len(lines) == 3 - # Check each line has the format "{variable-length-hash}::content" (HashPos format) + # Check each line has HashPos format: unique lines use ~~ prefix, duplicates use ~4char~ for i, line in enumerate(lines, start=1): - # Format should be "{hash}::content" - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - # Check hash fragment is between 3 and 6 chars (3 encoded bytes) - assert 3 <= len(hash_fragment) <= 6 + # Unique lines should use ~~ prefix + assert line.startswith("~~"), f"Line {i} should have ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" def test_hashline_with_start_line(): @@ -32,15 +30,12 @@ def test_hashline_with_start_line(): lines = result.splitlines() assert len(lines) == 2 - # Check format is {variable-length-hash}::content (HashPos format) - # Note: start_line parameter is ignored by HashPos but kept for compatibility + # Check format is HashPos format (start_line is ignored by HashPos) for line in lines: - # Format should be "{hash}::content" - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - # Check hash fragment is between 3 and 6 chars (3 encoded bytes) - assert 3 <= len(hash_fragment) <= 6 + # Unique lines should use ~~ prefix + assert line.startswith("~~"), f"Expected ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" """Test hashline with empty string.""" result = hashline("") assert result == "" @@ -52,13 +47,12 @@ def test_hashline_single_line(): result = hashline(text) lines = result.splitlines() assert len(lines) == 1 - # Check format is {variable-length-hash}::content (HashPos format) + # Check format is HashPos format: ~~content (unique) or ~4char~content (duplicate) line = lines[0] - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - # Check hash fragment is between 3 and 6 chars (3 encoded bytes) - assert 3 <= len(hash_fragment) <= 6 + # Unique line should use ~~ prefix + assert line.startswith("~~"), f"Expected ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" def test_hashline_preserves_newlines(): @@ -69,20 +63,20 @@ def test_hashline_preserves_newlines(): # The result should have hashes on each line but no trailing newline lines = result.splitlines() assert len(lines) == 2 - # Check each line has the correct format + # Check each line has HashPos format for line in lines: - assert "::" in line - # Extract hash fragment (everything before "::") - hash_fragment = line.split("::", 1)[0] - assert 3 <= len(hash_fragment) <= 6 + # Unique lines should use ~~ prefix + assert line.startswith("~~"), f"Expected ~~ prefix but got: {line!r}" + hash_fragment = line[:2] + assert hash_fragment == "~~" # HashPos doesn't preserve trailing newlines in the formatted output # The splitlines() above verifies we have the right number of lines def test_strip_hashline_basic(): """Test basic strip_hashline functionality.""" - # Create a hashline-formatted text with correct HashPos format: {variable-length-hash}::content - text = "abc::Hello\nbad::World\n1a2::Test" + # Create a hashline-formatted text with HashPos format + text = "~~Hello\n~~World\n~~Test" stripped = strip_hashline(text) assert stripped == "Hello\nWorld\nTest" @@ -91,24 +85,25 @@ def test_strip_hashline_with_negative_line_numbers(): """Test strip_hashline with negative line numbers.""" # HashPos format doesn't support negative line numbers in the prefix # Test with standard HashPos format - text = "abc::Hello\nbad::World\n1a2::Test" + # HashPos format: ~~content (unique) or ~4char~content (duplicate) + text = "~~Hello\n~~World\n~~Test" stripped = strip_hashline(text) assert stripped == "Hello\nWorld\nTest" def test_strip_hashline_mixed_lines(): """Test strip_hashline with mixed hashline and non-hashline lines.""" - # HashPos format: {variable-length-hash}::content + # HashPos format: ~~content (unique) or ~4char~content (duplicate) # Plain lines without hashes should be left unchanged - text = "abc::Hello\nPlain line\nbad::World" + text = "~~Hello\nPlain line\n~~World" stripped = strip_hashline(text) assert stripped == "Hello\nPlain line\nWorld" def test_strip_hashline_preserves_newlines(): """Test that strip_hashline preserves newline characters.""" - # HashPos format: {variable-length-hash}::content - text = "abc::Line 1\nbad::Line 2\n" + # HashPos format: ~~content (unique) or ~4char~content (duplicate) + text = "~~Line 1\n~~Line 2\n" stripped = strip_hashline(text) # strip_hashline should preserve newlines assert stripped == "Line 1\nLine 2\n" @@ -158,14 +153,14 @@ def test_hashline_different_inputs(): def test_parse_hashline(): """Test parse_hashline function.""" # Test basic parsing (HashPos format) - hash_fragment, line_num_str, line_num = parse_hashline("abc") - assert hash_fragment == "abc" + hash_fragment, line_num_str, line_num = parse_hashline("~0abc~") + assert hash_fragment == "~0abc~" assert line_num_str is None # HashPos doesn't include line numbers assert line_num is None # Test with content after hash - hash_fragment, line_num_str, line_num = parse_hashline("bad::Hello World") - assert hash_fragment == "bad" + hash_fragment, line_num_str, line_num = parse_hashline("~bad1~Hello World") + assert hash_fragment == "~bad1~" assert line_num_str is None assert line_num is None diff --git a/tests/basic/test_hashline_closure.py b/tests/basic/test_hashline_closure.py index cd762903909..4399a3470db 100644 --- a/tests/basic/test_hashline_closure.py +++ b/tests/basic/test_hashline_closure.py @@ -483,15 +483,18 @@ def test_apply_closure_safeguard_passes_rejected_ops_to_failed(): Integration test: apply_hashline_operations should propagate rejected operations (from _apply_closure_safeguard) into the failed_ops list. """ - from cecli.helpers.hashline import apply_hashline_operations, hashline + from cecli.helpers.hashline import apply_hashline_operations source = """x = 1 y = 2 """ # Try to replace a line with broken syntax — this should fail via eviction - # Get the correct hashline ID for the first line - hp_content = hashline(source) - first_line_hash = hp_content.splitlines()[0].split("::")[0] + "::" + # Get the correct hashline ID for the first line using HashPos API + # New format uses ~4chars~ (tilde-wrapped 4-char hash), no :: separator + from cecli.helpers.hashpos.hashpos import HashPos + + hp = HashPos(source) + first_line_hash = hp.get_wrapped_id(hp.generate_public_id("x = 1", 0, 1)) ops = [ { diff --git a/tests/basic/test_main.py b/tests/basic/test_main.py index 082877e2cb9..8cff21870cf 100644 --- a/tests/basic/test_main.py +++ b/tests/basic/test_main.py @@ -73,7 +73,7 @@ def test_env(mocker, temp_cwd, temp_home): test_env_vars["HOME"] = temp_home mocker.patch.dict(os.environ, test_env_vars) mocker.patch("builtins.input", return_value=None) - mocker.patch("cecli.io.webbrowser.open") + mocker.patch("webbrowser.open") @pytest.fixture diff --git a/tests/basic/test_model_provider_manager.py b/tests/basic/test_model_provider_manager.py index fb9ddd46721..6da85ac6650 100644 --- a/tests/basic/test_model_provider_manager.py +++ b/tests/basic/test_model_provider_manager.py @@ -31,14 +31,6 @@ def _dummy_open(*args, **kwargs): sys.modules["PIL.Image"] = image_module sys.modules["PIL.ImageGrab"] = image_grab_module - if "numpy" not in sys.modules: - numpy_module = types.ModuleType("numpy") - numpy_module.ndarray = object - numpy_module.array = lambda *a, **k: None - numpy_module.dot = lambda *a, **k: 0.0 - numpy_module.linalg = types.SimpleNamespace(norm=lambda *a, **k: 1.0) - sys.modules["numpy"] = numpy_module - if "oslex" not in sys.modules: oslex_module = types.ModuleType("oslex") oslex_module.__all__ = [] diff --git a/tests/helpers/test_orchestration.py b/tests/helpers/test_orchestration.py new file mode 100644 index 00000000000..3f7a4585053 --- /dev/null +++ b/tests/helpers/test_orchestration.py @@ -0,0 +1,1715 @@ +""" +Tests for the orchestration sandbox security boundaries. + +Covers: +- SecurityFilter (imports, dunder attrs, dangerous builtins, global/nonlocal) +- LoopYieldInjector (yield injection into while/for loops) +- AgentExecutionEnv (rejects dangerous code, runs safe code) +- AgentProxy / ToolProxy (local and MCP tool lookup, filter enforcement) +""" + +import ast + +import pytest + +from cecli.helpers.orchestration.environment import ( + AgentExecutionEnv, + AgentProxy, + build_orchestration_context_block, +) +from cecli.helpers.orchestration.safe_methods import _strip_allowed_imports +from cecli.helpers.orchestration.security import ( + LoopYieldInjector, + SecurityError, + SecurityFilter, + _security_raise, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_mock_coder(): + """Minimal coder mock for AgentExecutionEnv (tool calls not exercised).""" + + class _MockCoder: + registered_tools = {"included": set(), "excluded": set()} + mcp_tools = [] + + return _MockCoder() + + +def _run_security_filter_on(code: str) -> None: + """Run the SecurityFilter and execute the rewritten AST. Raises SecurityError if blocked. + + Only executes code that was actually rewritten by the SecurityFilter + (i.e., contains ``__security_raise`` calls). Safe code that passes + through the filter unmodified is not executed, avoiding NameError/ + TypeError issues from test-scope variable names in the namespace. + """ + import copy + + original_tree = ast.parse(code, mode="exec") + original_tree = copy.deepcopy(original_tree) + rewritten_tree = SecurityFilter().visit(copy.deepcopy(original_tree)) + ast.fix_missing_locations(rewritten_tree) + + # Compare AST dumps to detect whether the filter rewrote anything. + # We deep-copy because ast.NodeTransformer.visit() mutates in place. + original_dump = ast.dump(original_tree, indent=0) + rewritten_dump = ast.dump(rewritten_tree, indent=0) + if original_dump == rewritten_dump: + return + + # Compile with top-level await support (needed for snippets like + # ``await gather()`` which may appear in user code). + compiled_code = compile( + rewritten_tree, + "", + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + + # Provide a minimal namespace with ``__security_raise`` so that + # rewritten constructs actually raise SecurityError at runtime. + ns: dict = { + "__security_raise": _security_raise, + } + exec(compiled_code, ns) + + +def _run_security_filter_safe(code: str) -> bool: + """Run the SecurityFilter; return True if safe, False if blocked.""" + try: + _run_security_filter_on(code) + return True + except SecurityError: + return False + + +def _get_loop_yield_count(tree: ast.AST) -> int: + """Count how many ``await __yield()`` statements appear in the tree.""" + count = 0 + for node in ast.walk(tree): + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Await): + if ( + isinstance(node.value.value, ast.Call) + and isinstance(node.value.value.func, ast.Name) + and node.value.value.func.id == "__yield" + ): + count += 1 + return count + + +# =================================================================== +# SecurityFilter +# =================================================================== + + +def test_security_filter_blocks_import(): + """SecurityFilter raises SecurityError for ``import os``.""" + assert not _run_security_filter_safe("import os"), "SecurityFilter should block 'import os'" + + +def test_security_filter_blocks_from_import(): + """SecurityFilter raises SecurityError for ``from os import path``.""" + assert not _run_security_filter_safe( + "from os import path" + ), "SecurityFilter should block 'from os import path'" + + +def test_security_filter_blocks_star_import(): + """SecurityFilter raises SecurityError for ``from os import *``.""" + assert not _run_security_filter_safe( + "from os import *" + ), "SecurityFilter should block 'from os import *'" + + +def test_security_filter_blocks_dunder_attribute(): + """SecurityFilter blocks access to ``x.__class__``.""" + assert not _run_security_filter_safe("x.__class__"), "SecurityFilter should block 'x.__class__'" + + +def test_security_filter_blocks_nested_dunder(): + """SecurityFilter blocks access to ``x.y.__class__`` (nested dunder).""" + assert not _run_security_filter_safe( + "x.y.__class__" + ), "SecurityFilter should block 'x.y.__class__'" + + +def test_security_filter_blocks_dunder_method_call(): + """SecurityFilter blocks calls via dunder like ``x.__subclasses__()``.""" + assert not _run_security_filter_safe( + "x.__subclasses__()" + ), "SecurityFilter should block 'x.__subclasses__()'" + + +def test_security_filter_blocks_eval(): + """SecurityFilter blocks ``eval()``.""" + assert not _run_security_filter_safe("eval('1+1')"), "SecurityFilter should block 'eval'" + + +def test_security_filter_blocks_exec(): + """SecurityFilter blocks ``exec()``.""" + assert not _run_security_filter_safe("exec('x=1')"), "SecurityFilter should block 'exec'" + + +def test_security_filter_blocks_open(): + """SecurityFilter blocks ``open()``.""" + assert not _run_security_filter_safe( + "open('/etc/passwd')" + ), "SecurityFilter should block 'open'" + + +def test_security_filter_blocks_import_call(): + """SecurityFilter blocks ``__import__()``.""" + assert not _run_security_filter_safe( + "__import__('os')" + ), "SecurityFilter should block '__import__'" + + +def test_security_filter_blocks_compile(): + """SecurityFilter blocks ``compile()``.""" + assert not _run_security_filter_safe( + "compile('x=1', '', 'exec')" + ), "SecurityFilter should block 'compile'" + + +def test_security_filter_blocks_breakpoint(): + """SecurityFilter blocks ``breakpoint()``.""" + assert not _run_security_filter_safe("breakpoint()"), "SecurityFilter should block 'breakpoint'" + + +def test_security_filter_blocks_globals(): + """SecurityFilter blocks ``globals()``.""" + assert not _run_security_filter_safe("globals()"), "SecurityFilter should block 'globals'" + + +def test_security_filter_blocks_locals(): + """SecurityFilter blocks ``locals()``.""" + assert not _run_security_filter_safe("locals()"), "SecurityFilter should block 'locals'" + + +def test_security_filter_allows_vars(): + """SecurityFilter allows ``vars(obj)`` — safe at runtime via _safe_vars.""" + assert _run_security_filter_safe( + "vars(obj)" + ), "SecurityFilter should allow 'vars(obj)' as safe version is provided" + + +@pytest.mark.asyncio +async def test_env_vars_requires_one_arg(): + """vars() with no arguments raises TypeError (no local namespace leak).""" + env = _make_env() + result = await env.execute("vars()") + assert ( + "TypeError" in result["results"] + ), f"Expected TypeError for vars() no-args, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_vars_works_with_obj(): + """vars(obj) returns non-dunder attributes of obj.""" + env = _make_env() + result = await env.execute("def f(): pass\nf.x = 42\nprint(vars(f))") + assert ( + '"x": 42' in result["results"] or "'x': 42" in result["results"] + ), f"Expected 'x' attr in vars result, got: {result!r}" + + +def test_security_filter_allows_getattr(): + """SecurityFilter allows ``getattr()``.""" + assert _run_security_filter_safe("getattr(x, 'y')"), "SecurityFilter should allow 'getattr'" + + +def test_security_filter_blocks_setattr(): + """SecurityFilter blocks ``setattr()``.""" + assert not _run_security_filter_safe( + "setattr(x, 'y', 1)" + ), "SecurityFilter should block 'setattr'" + + +def test_security_filter_blocks_delattr(): + """SecurityFilter blocks ``delattr()``.""" + assert not _run_security_filter_safe("delattr(x, 'y')"), "SecurityFilter should block 'delattr'" + + +def test_security_filter_blocks_global(): + """SecurityFilter blocks ``global x``.""" + assert not _run_security_filter_safe( + "global x" + ), "SecurityFilter should block 'global' statement" + + +def test_security_filter_blocks_nonlocal(): + """SecurityFilter blocks ``nonlocal x``.""" + assert not _run_security_filter_safe( + "nonlocal x" + ), "SecurityFilter should block 'nonlocal' statement" + + +def test_security_filter_allows_safe_code(): + """SecurityFilter allows ordinary safe code.""" + safe_snippets = [ + "x = 42", + "x + y", + "print('hello')", + "len([1, 2, 3])", + "[i * 2 for i in range(10)]", + "state['key'] = 'value'", + "await gather()", + "sleep(1)", + "isinstance(x, int)", + ] + for snippet in safe_snippets: + assert _run_security_filter_safe( + snippet + ), f"SecurityFilter should allow safe code: {snippet!r}" + + +def test_security_filter_error_message_import(): + """SecurityError message mentions the violation for imports.""" + try: + _run_security_filter_on("import os") + except SecurityError as e: + msg = str(e) + assert "import" in msg.lower(), f"SecurityError should mention 'import', got: {msg}" + else: + pytest.fail("Expected SecurityError") + + +def test_security_filter_error_message_dunder(): + """SecurityError message includes the blocked attribute name for dunder.""" + try: + _run_security_filter_on("x.__foo__") + except SecurityError as e: + msg = str(e) + assert "__foo__" in msg, f"SecurityError should include '__foo__', got: {msg}" + else: + pytest.fail("Expected SecurityError") + + +def test_security_filter_error_message_builtin(): + """SecurityError message mentions the blocked builtin name.""" + try: + _run_security_filter_on("eval('1+1')") + except SecurityError as e: + msg = str(e) + assert "eval" in msg, f"SecurityError should mention 'eval', got: {msg}" + else: + pytest.fail("Expected SecurityError") + + +# =================================================================== +# LoopYieldInjector +# =================================================================== + + +def test_loop_yield_injector_injects_into_while(): + """LoopYieldInjector injects ``await __yield()`` at top of while loop body.""" + tree = ast.parse( + """ +while True: + x = 1 +""", + mode="exec", + ) + original_count = _get_loop_yield_count(tree) + assert original_count == 0, "No yields before injection" + + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 1, f"Expected 1 yield in while loop, got {yield_count}" + + +def test_loop_yield_injector_injects_into_for(): + """LoopYieldInjector injects ``await __yield()`` at top of for loop body.""" + tree = ast.parse( + """ +for i in range(10): + print(i) +""", + mode="exec", + ) + original_count = _get_loop_yield_count(tree) + assert original_count == 0, "No yields before injection" + + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 1, f"Expected 1 yield in for loop, got {yield_count}" + + +def test_loop_yield_injector_multiple_loops(): + """LoopYieldInjector injects yields into every loop in nested structures.""" + tree = ast.parse( + """ +for a in items: + while b: + for c in nested: + pass +""", + mode="exec", + ) + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 3, f"Expected 3 yields (for + while + for), got {yield_count}" + + +def test_loop_yield_injector_no_loops(): + """LoopYieldInjector does not inject yields if there are no loops.""" + tree = ast.parse("x = 42\nprint(x)", mode="exec") + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + yield_count = _get_loop_yield_count(modified) + assert yield_count == 0, f"Expected 0 yields (no loops), got {yield_count}" + + +def test_loop_yield_injector_first_in_body(): + """The injected yield is the very first statement in the loop body.""" + tree = ast.parse( + """ +while flag: + print('hello') + x += 1 +""", + mode="exec", + ) + modified = LoopYieldInjector().visit(tree) + ast.fix_missing_locations(modified) + + while_nodes = [n for n in ast.walk(modified) if isinstance(n, ast.While)] + assert len(while_nodes) == 1 + first_stmt = while_nodes[0].body[0] + assert isinstance(first_stmt, ast.Expr) + assert isinstance(first_stmt.value, ast.Await) + assert isinstance(first_stmt.value.value, ast.Call) + assert first_stmt.value.value.func.id == "__yield" + + +# =================================================================== +# AgentExecutionEnv integration tests +# =================================================================== + + +def _make_env() -> AgentExecutionEnv: + return AgentExecutionEnv(_make_mock_coder()) + + +@pytest.mark.asyncio +async def test_env_rejects_import(): + """AgentExecutionEnv.execute() rejects code with imports.""" + env = _make_env() + result = await env.execute("import os") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for import, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_json_dumps(): + """AgentExecutionEnv.execute() provides ``json.dumps`` in globals.""" + env = _make_env() + result = await env.execute("print(json.dumps({'key': 'value'}))") + assert result["results"] == '{"key": "value"}', f"Expected JSON output, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_dunder(): + """AgentExecutionEnv.execute() rejects code with dunder access.""" + env = _make_env() + result = await env.execute("x = 1\nx.__class__") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for dunder, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_eval(): + """AgentExecutionEnv.execute() rejects code calling eval().""" + env = _make_env() + result = await env.execute("eval('1+1')") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for eval, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_open(): + """AgentExecutionEnv.execute() rejects code calling open().""" + env = _make_env() + result = await env.execute("open('/dev/null')") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for open, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_rejects_global_stmt(): + """AgentExecutionEnv.execute() rejects code with global statement.""" + env = _make_env() + result = await env.execute("global x") + assert ( + "Security Error" in result["results"] + ), f"Expected Security Error for global, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_runs_simple_expression(): + """AgentExecutionEnv.execute() runs a simple arithmetic expression.""" + env = _make_env() + result = await env.execute("print(42)") + assert result["results"] == "42", f"Expected '42', got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_runs_print(): + """AgentExecutionEnv.execute() captures print output.""" + env = _make_env() + result = await env.execute("print('hello world')") + assert result["results"] == "hello world", f"Expected 'hello world', got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_state_persistence(): + """AgentExecutionEnv.state persists across execute() calls.""" + env = _make_env() + + result1 = await env.execute("state['key'] = 'value1'\nprint(state['key'])") + assert result1["results"] == "value1", f"Expected 'value1', got: {result1!r}" + + result2 = await env.execute("print(state['key'])") + assert result2["results"] == "value1", f"Expected 'value1' (persisted), got: {result2!r}" + + +@pytest.mark.asyncio +async def test_env_runs_list_comprehension(): + """AgentExecutionEnv.execute() runs a list comprehension.""" + env = _make_env() + result = await env.execute("print([i * 2 for i in range(5)])") + assert ( + result["results"] == "[0, 2, 4, 6, 8]" + ), f"Expected list comprehension result, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_returns_last_expression(): + """AgentExecutionEnv only returns printed output — last expression values must be printed.""" + env = _make_env() + result = await env.execute("x = 10\ny = 20\nprint(x + y)") + assert result["results"] == "30", f"Expected '30' from last expression, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_print_and_expression(): + """AgentExecutionEnv returns print output only.""" + env = _make_env() + result = await env.execute("print('computed')\nprint(42)") + assert "computed" in result["results"] + assert "42" in result["results"] + + +@pytest.mark.asyncio +async def test_env_empty_code(): + """AgentExecutionEnv returns empty string for empty code.""" + env = _make_env() + result = await env.execute("") + assert result["results"] == "", f"Expected empty string for empty code, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_whitespace_code(): + """AgentExecutionEnv returns empty string for whitespace-only code.""" + env = _make_env() + result = await env.execute(" \n\n ") + assert result["results"] == "", f"Expected empty string for whitespace code, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_sleep_works(): + """AgentExecutionEnv's Agent.sleep primitive works.""" + env = _make_env() + result = await env.execute( + "await Agent.sleep(0.01)\nprint('done')", + ) + assert result["results"] == "done", f"Expected 'done' after sleep, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_gather_works(): + """AgentExecutionEnv's gather primitive works.""" + env = _make_env() + result = await env.execute("print(await gather())") + assert ( + result["results"] == "GatherResult()" + ), f"Expected 'GatherResult()' from gather(), got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_handles_syntax_error(): + """AgentExecutionEnv returns a syntax error message for invalid code.""" + env = _make_env() + result = await env.execute("x = ") + assert "Syntax Error" in result["results"], f"Expected Syntax Error, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_safe_builtins_available(): + """AgentExecutionEnv provides safe builtins (len, range, int, str, etc.).""" + env = _make_env() + + result = await env.execute("print(len([1, 2, 3]))") + assert result["results"] == "3", f"Expected '3', got: {result!r}" + + result = await env.execute("print(str(42))") + assert result["results"] == "42", f"Expected '42', got: {result!r}" + + result = await env.execute("print(list(range(3)))") + assert result["results"] == "[0, 1, 2]", f"Expected '[0, 1, 2]', got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_exception_handling(): + """AgentExecutionEnv handles runtime exceptions gracefully.""" + env = _make_env() + result = await env.execute("1 / 0") + assert ( + "Execution Error" in result["results"] and "ZeroDivisionError" in result["results"] + ), f"Expected Execution Error with ZeroDivisionError, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_runs_function_def(): + """AgentExecutionEnv can define and call functions.""" + env = _make_env() + result = await env.execute(""" +def add(a, b): + return a + b +print(add(2, 3)) +""") + assert result["results"] == "5", f"Expected '5' from function call, got: {result!r}" + + +@pytest.mark.asyncio +async def test_env_mixed_print_and_return(): + """AgentExecutionEnv returns only printed output.""" + env = _make_env() + result = await env.execute(""" +print("start") +result = 99 +print(result) +""") + assert ( + "start" in result["results"] and "99" in result["results"] + ), f"Expected both 'start' and '99', got: {result!r}" + + +# =================================================================== +@pytest.mark.asyncio +async def test_env_shared_state_writes(): + """AgentExecutionEnv shared_state is writable and shows in snapshot.""" + env = _make_env() + result = await env.execute("shared_state['findings'] = 'done'") + svars = {v["name"]: v for v in result["state_variables"]} + assert "findings" in svars + assert svars["findings"]["type"] == "str" + assert svars["findings"]["size"] == 4 + assert svars["findings"]["scope"] == "shared" + assert svars["findings"]["modified"] is True + + +@pytest.mark.asyncio +async def test_env_shared_state_visible_across_envs(): + """AgentExecutionEnv shared_state is visible across separate env instances.""" + AgentExecutionEnv._shared_state.clear() + env1 = _make_env() + await env1.execute("shared_state['msg'] = 'hello'") + env2 = _make_env() + svars = {v["name"]: v for v in env2._state_snapshot()} + assert "msg" in svars + assert svars["msg"]["type"] == "str" + assert svars["msg"]["scope"] == "shared" + assert svars["msg"]["modified"] is False # env2 didn't write it + + +@pytest.mark.asyncio +async def test_env_state_snapshot_includes_scope_field(): + """state_variables entries have scope field: 'local' or 'shared'.""" + env = _make_env() + result = await env.execute("state['local_key'] = 1") + for entry in result["state_variables"]: + assert entry["scope"] in ("local", "shared") + local_vars = [e for e in result["state_variables"] if e["scope"] == "local"] + # shared_vars = [e for e in result["state_variables"] if e["scope"] == "shared"] + assert any(e["name"] == "local_key" for e in local_vars) + + +# AgentProxy / ToolProxy — tool lookup and dispatch +# =================================================================== + + +def _make_mock_coder_with_mcp(): + """Minimal coder mock with MCP tool support for proxy tests.""" + + class _MockMcpServer: + name = "MockServer" + + class _MockMcpManager: + def __iter__(self): + return iter([self._server]) + + def __init__(self): + self._server = _MockMcpServer() + + class _MockCoder: + registered_tools = {"included": set(), "excluded": set()} + mcp_tools = [ + ( + "MockServer", + [{"type": "function", "function": {"name": "MockTool"}}], + ), + ] + mcp_manager = _MockMcpManager() + + async def _execute_mcp_tool(self, server, tool_name, params): + return f"mcp-result: {tool_name} called with {params}" + + return _MockCoder() + + +def test_agent_proxy_local_tool_lookup(): + """AgentProxy.get_tool resolves local tools via ToolRegistry.""" + proxy = AgentProxy(_make_mock_coder()) + tool = proxy.get_tool("ReadFile") + assert tool._tool_module is not None, "Local tool should have _tool_module" + assert tool._mcp_server is None, "Local tool should not have _mcp_server" + + +def test_agent_proxy_unknown_tool_raises(): + """AgentProxy.get_tool raises ValueError for unknown tool names.""" + proxy = AgentProxy(_make_mock_coder()) + with pytest.raises(ValueError, match="Unknown tool"): + proxy.get_tool("NonExistentToolXYZ") + + +def test_agent_proxy_mcp_tool_lookup(): + """AgentProxy.get_tool resolves MCP tools with ServerName--ToolName prefix.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + tool = proxy.get_tool("MockServer--MockTool") + assert tool._tool_module is None, "MCP tool should not have _tool_module" + assert tool._mcp_server is not None, "MCP tool should have _mcp_server" + assert tool._mcp_tool_name == "MockTool" + + +@pytest.mark.asyncio +async def test_tool_proxy_mcp_dispatch(): + """ToolProxy.call dispatches to coder._execute_mcp_tool for MCP tools.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + tool = proxy.get_tool("MockServer--MockTool") + result = await tool.call(param1="value1") + assert "result" in result + assert "mcp-result" in result["result"][0]["content"] + assert "MockTool" in result["result"][0]["content"] + assert "param1" in result["result"][0]["content"] + + +def test_agent_proxy_includelist_filters(): + """AgentProxy.get_tool respects coder.registered_tools includelist.""" + coder = _make_mock_coder() + coder.registered_tools["included"] = {"yield"} + proxy = AgentProxy(coder) + # In list — allowed + tool = proxy.get_tool("Yield") + assert tool._tool_module is not None + # Not in list — blocked + with pytest.raises(ValueError, match="not in the allowed tools list"): + proxy.get_tool("ReadFile") + + +def test_agent_proxy_excludelist_filters(): + """AgentProxy.get_tool respects coder.registered_tools excludelist.""" + coder = _make_mock_coder() + coder.registered_tools["excluded"] = {"readfile"} + proxy = AgentProxy(coder) + with pytest.raises(ValueError, match="has been excluded"): + proxy.get_tool("ReadFile") + + +def test_agent_proxy_mcp_with_bare_name(): + """AgentProxy.get_tool finds MCP tools even with bare (unprefixed) name.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + tool = proxy.get_tool("mocktool") + assert tool._mcp_server is not None + assert tool._mcp_tool_name == "MockTool" + + +def test_agent_proxy_find_mcp_server_no_manager(): + """_find_mcp_server returns None when coder has no mcp_manager.""" + proxy = AgentProxy(_make_mock_coder()) + result = proxy._find_mcp_server("AnyServer", "") + assert result is None + + +def test_agent_proxy_local_priority(): + """AgentProxy.get_tool prefers local tools over MCP tools with same name.""" + coder = _make_mock_coder_with_mcp() + proxy = AgentProxy(coder) + # "ReadFile" exists as local tool — should be found first + tool = proxy.get_tool("ReadFile") + assert tool._tool_module is not None, "Local tool must be preferred" + assert tool._mcp_server is None + + +def test_agent_proxy_local_prefix_tool(): + """AgentProxy.get_tool handles Local--ToolName by stripping prefix.""" + proxy = AgentProxy(_make_mock_coder()) + tool = proxy.get_tool("Local--ReadFile") + assert tool._tool_module is not None, "Local--ReadFile should resolve via ToolRegistry" + assert tool._mcp_server is None + + +# =================================================================== +# AgentRegion +# =================================================================== + + +def test_agent_region_basic(tmp_path): + """AgentRegion stores patterns and resolves lazily via get_start/get_end.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + # Create a real file so lazy resolution works + source = "def foo():\n return 42\n" + test_file = os.path.join(str(tmp_path), "basic.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + regions = AgentRegion( + "basic.py", + coder, + [ + {"name": "foo", "start": "def foo", "end": "return 42"}, + ], + ) + + assert "foo" in regions + assert "missing" not in regions + assert len(regions) == 1 + assert "AgentRegion(1" in repr(regions) + + # Lazy resolution on access + start_id = regions.get_start("foo") + end_id = regions.get_end("foo") + + assert "~" in start_id + assert "~" in end_id + + start_line = regions.get_start_line("foo") + end_line = regions.get_end_line("foo") + + assert start_line > 0 + assert end_line > 0 + assert start_line <= end_line + + +# =================================================================== +# AgentProxy.resolve_regions +# =================================================================== + + +def _make_coder_with_io(tmp_path): + """Build a coder mock that supports resolve_paths + file I/O.""" + + import os + + class _MockIO: + def read_text(self, abs_path): + if os.path.isfile(abs_path): + with open(abs_path, "r") as f: + return f.read() + + return None + + class _MockCoder: + root = tmp_path + io = _MockIO() + registered_tools = {"included": set(), "excluded": set()} + mcp_tools = [] + + def abs_root_path(self, file_path): + import os + + return os.path.join(self.root, file_path) + + def get_rel_fname(self, abs_path): + import os + + return os.path.relpath(abs_path, self.root) + + return _MockCoder() + + +def test_resolve_regions_basic(tmp_path): + """AgentProxy.resolve_regions resolves text patterns to content IDs.""" + + import os + + from cecli.helpers.orchestration.environment import AgentProxy + + # Create a test file with known regions + source = """\ +def helper_one(): + \"\"\"Returns 'one'.\"\"\" + return "one" + + +def helper_two(): + \"\"\"Returns 'two'.\"\"\" + return "two" + + +def main(): + result = helper_one() + helper_two() + return result +""" + + test_file = os.path.join(str(tmp_path), "test_regions.py") + + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + proxy = AgentProxy(coder) + + regions = proxy.resolve_regions( + "test_regions.py", + [ + {"name": "helper_one", "start": "def helper_one", "end": 'return "one"'}, + {"name": "helper_two", "start": "def helper_two", "end": 'return "two"'}, + {"name": "main", "start": "def main", "end": "return result"}, + ], + ) + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + assert isinstance(regions, AgentRegion) + assert len(regions) == 3 + + # Each region should have valid content IDs + for name in ("helper_one", "helper_two", "main"): + start_id = regions.get_start(name) + end_id = regions.get_end(name) + + assert "~" in start_id, f"{name} start_id should be a content ID: {start_id}" + assert "~" in end_id, f"{name} end_id should be a content ID: {end_id}" + + assert regions.get_start_line(name) > 0 + assert regions.get_end_line(name) > 0 + assert regions.get_start_line(name) <= regions.get_end_line(name) + + +def test_resolve_regions_file_not_found(tmp_path): + """Eager validation raises ValueError at resolve_regions time when file doesn't exist.""" + + import pytest + + from cecli.helpers.orchestration.environment import AgentProxy + + coder = _make_coder_with_io(str(tmp_path)) + + proxy = AgentProxy(coder) + + # Eager validation surfaces errors at construction time + with pytest.raises(ValueError, match="File not found"): + proxy.resolve_regions( + "nonexistent.py", + [{"name": "x", "start": "def x", "end": "return"}], + ) + + +def test_resolve_regions_empty_regions_list(tmp_path): + """AgentProxy.resolve_regions handles empty region list.""" + + import os + + from cecli.helpers.orchestration.environment import AgentProxy + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = "x = 1\n" + + test_file = os.path.join(str(tmp_path), "empty_test.py") + + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + proxy = AgentProxy(coder) + + regions = proxy.resolve_regions("empty_test.py", []) + + assert isinstance(regions, AgentRegion) + assert len(regions) == 0 + + +def test_agent_region_content_id_fallback(tmp_path): + """AgentRegion snapshots line content from content-ID patterns for stale-ID fallback.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def greet(name): + return f"hello {name}" + + +def farewell(name): + return f"goodbye {name}" +""" + + test_file = os.path.join(str(tmp_path), "fallback_test.py") + + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # First, resolve using the Agent to get a real content ID + from cecli.helpers.orchestration.environment import AgentProxy + + proxy = AgentProxy(coder) + regions = proxy.resolve_regions( + "fallback_test.py", + [{"name": "greet", "start": "def greet", "end": 'return f"hello {name}"'}], + ) + + # Get the content ID for the start of "greet" + start_id = regions.get_start("greet") + + # Now create a NEW AgentRegion that uses the raw content ID as the pattern. + # This simulates an agent that captured content IDs from a previous turn. + regions2 = AgentRegion( + "fallback_test.py", + coder, + [{"name": "greet", "start": start_id, "end": start_id}], + ) + + # First resolution should snapshot the line content + result_id = regions2.get_start("greet") + assert "~" in result_id + + # Now modify the file (simulating an edit that shifts hashlines) + new_source = """\ +# header comment added +def greet(name): + return f"hello {name}" + + +def farewell(name): + return f"goodbye {name}" +""" + + with open(test_file, "w") as f: + f.write(new_source) + + # The original content ID is now stale, but the snapshot should let + # us resolve via content matching + result_id2 = regions2.get_start("greet") + assert "~" in result_id2, f"Should resolve via fallback content match, got: {result_id2!r}" + + +def test_agent_region_rejects_ambiguous_pattern(tmp_path): + """AgentRegion raises ValueError eagerly when a text pattern matches multiple locations.""" + + import os + + import pytest + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + # Create a file where "return x" appears multiple times + source = """\ +def foo(): + return x + + +def bar(): + return x + + +def baz(): + return x +""" + + test_file = os.path.join(str(tmp_path), "ambiguous.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # start is unique, but end matches 3 locations — error surfaces eagerly + with pytest.raises(ValueError, match="End pattern"): + AgentRegion( + "ambiguous.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return x"}, + ], + ) + + +def test_agent_region_disambiguates_with_l_hint(tmp_path): + """AgentRegion uses @L hint to disambiguate when a pattern matches multiple locations.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def foo(): + return x + +def bar(): + return y + +def baz(): + return z +""" + + test_file = os.path.join(str(tmp_path), "hint_test.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # "return" appears on lines 2, 5, 8 — use @L hint to target bar's return + regions = AgentRegion( + "hint_test.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return @L6"}, + ], + ) + + start_id = regions.get_start("bar") + end_id = regions.get_end("bar") + + assert "~" in start_id + assert "~" in end_id + assert regions.get_start_line("bar") == 4 + assert regions.get_end_line("bar") == 5 + + +def test_agent_region_rejects_l_hint_still_ambiguous(tmp_path): + """AgentRegion raises ValueError eagerly when @L hint has equally-close matches (tie).""" + + import os + + import pytest + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +return x + +return y +middle +return x +return z +""" + + test_file = os.path.join(str(tmp_path), "bad_hint.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # "return x" at lines 1 and 4 (0-based: 0 and 3), hint @L3 (0-based 2) + # Both are distance 1 away from the hint — error surfaces eagerly + with pytest.raises(ValueError, match="End pattern.*@L3 hint ties"): + AgentRegion( + "bad_hint.py", + coder, + [ + {"name": "top", "start": "return x @L1", "end": "return x @L3"}, + ], + ) + + +def test_agent_region_explicit_line_hints(tmp_path): + """Explicit start_line_hint / end_line_hint fields disambiguate patterns.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def foo(): + return x + +def bar(): + return y + +def baz(): + return z +""" + + test_file = os.path.join(str(tmp_path), "explicit_hint.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # Use explicit end_line_hint instead of @L in pattern + regions = AgentRegion( + "explicit_hint.py", + coder, + [ + {"name": "bar", "start": "def bar", "end": "return", "end_line_hint": 6}, + ], + ) + + start_id = regions.get_start("bar") + end_id = regions.get_end("bar") + + assert "~" in start_id + assert "~" in end_id + assert regions.get_start_line("bar") == 4 + assert regions.get_end_line("bar") == 5 + + +def test_agent_region_explicit_hint_overrides_at_syntax(tmp_path): + """Explicit line_hint fields override @L hints embedded in patterns.""" + + import os + + from cecli.helpers.orchestration.region_resolver import AgentRegion + + source = """\ +def foo(): + return x + +def bar(): + return y + +def baz(): + return z +""" + + test_file = os.path.join(str(tmp_path), "override_hint.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + + # @L2 points to foo's return, but explicit end_line_hint=3 points to bar's + regions = AgentRegion( + "override_hint.py", + coder, + [ + { + "name": "bar", + "start": "def bar", + "end": "return @L3", + "end_line_hint": 6, + }, + ], + ) + + # Explicit hint (6→line 5) should win over @L3 (→line 2) + assert regions.get_end_line("bar") == 5 + + +def test_resolve_regions_rejects_ambiguous_via_proxy(tmp_path): + """resolve_regions() through AgentProxy surfaces ambiguity errors eagerly at construction time.""" + + import os + + import pytest + + from cecli.helpers.orchestration.environment import AgentProxy + + source = """\ +def one(): + pass + + +def two(): + pass + + +def three(): + pass +""" + + test_file = os.path.join(str(tmp_path), "proxy_ambig.py") + with open(test_file, "w") as f: + f.write(source) + + coder = _make_coder_with_io(str(tmp_path)) + proxy = AgentProxy(coder) + + # "pass" appears 3 times — error surfaces eagerly at resolve_regions time + with pytest.raises(ValueError, match="End pattern.*pass.*matches 3"): + proxy.resolve_regions( + "proxy_ambig.py", + [{"name": "two", "start": "def two", "end": "pass"}], + ) + + +def test_gather_result_attribute_access(): + """GatherResult supports attribute access for named results.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"read_a": {"result": ["hello"]}, "grep_b": {"result": ["world"]}}) + assert gr.read_a == {"result": ["hello"]} + assert gr.grep_b == {"result": ["world"]} + + +def test_gather_result_key_access(): + """GatherResult supports key/index access for named results.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"x": 1, "y": 2}) + assert gr["x"] == 1 + assert gr["y"] == 2 + + +def test_gather_result_len(): + """GatherResult supports len().""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 1, "b": 2, "c": 3}) + assert len(gr) == 3 + + +def test_gather_result_contains(): + """GatherResult supports 'in' operator.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 1}) + assert "a" in gr + assert "b" not in gr + + +def test_gather_result_iteration(): + """GatherResult is iterable (yields values).""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 10, "b": 20}) + values = list(gr) + assert ("a", 10) in values + assert ("b", 20) in values + assert len(values) == 2 + + +def test_gather_result_read_only(): + """GatherResult prevents mutation via attribute assignment.""" + import pytest + + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 1}) + with pytest.raises(AttributeError, match="read-only"): + gr.a = 2 + + +def test_gather_result_repr(): + """GatherResult repr shows types.""" + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"a": 42, "b": "hello"}) + r = repr(gr) + assert "GatherResult" in r + assert "int" in r or "str" in r + + +def test_gather_result_missing_key(): + """GatherResult raises AttributeError with helpful message for missing keys.""" + import pytest + + from cecli.helpers.orchestration.environment import GatherResult + + gr = GatherResult({"foo": 1, "bar": 2}) + with pytest.raises(AttributeError, match="no key"): + gr.baz + + +@pytest.mark.asyncio +async def test_env_named_gather(): + """AgentExecutionEnv supports named gather returning GatherResult.""" + env = _make_env() + result = await env.execute( + "results = await gather(a=tool.call(), b=tool.call())\n" + "print(results.a)\n" + "print(results['b'])\n" + "print(len(results))\n" + ) + # Just verify it doesn't crash and produces output + assert result["results"] # should have some output + + +@pytest.mark.asyncio +async def test_env_gather_mixed_rejected(): + """Mixing positional and keyword args in gather raises TypeError.""" + env = _make_env() + result = await env.execute("await gather(1, a=2)") + assert "TypeError" in result["results"] or "cannot mix" in result["results"].lower() + + +@pytest.mark.asyncio +async def test_env_named_gather_exception_handling(): + """Named gather converts exceptions to error dicts per-key.""" + env = _make_env() + result = await env.execute( + "results = await gather(good=tool.call(), bad=tool.call())\n" + "print('errors' in results.bad)\n" + ) + # Just verify it doesn't crash + assert result["results"] is not None + + +# --------------------------------------------------------------------------- +# Test Group 1: allowed_imports +# --------------------------------------------------------------------------- + + +class TestAllowedImports: + """Tests for the `allowed_imports` orchestration config option.""" + + def test_import_with_allowed_imports(self): + """1.1: `import os` with allowed_imports=["os"] passes SecurityFilter.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("import os", mode="exec") + rewritten = sf.visit(tree) + # Should not contain __security_raise — imports pass through + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_import_without_allowed_imports(self): + """1.2: `import os` with empty allowed_imports is blocked.""" + assert not _run_security_filter_safe("import os") + + def test_from_import_with_allowed(self): + """1.3: `from os import path` with allowed_imports=["os"] succeeds.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("from os import path", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_from_import_as_with_allowed(self): + """1.4: `from os import path as p` with allowed_imports=["os"] succeeds.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("from os import path as p", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_mixed_import_one_not_allowed(self): + """1.5: `import os, sys` with allowed_imports=["os"] is blocked on sys.""" + sf = SecurityFilter(allowed_imports=frozenset({"os"})) + tree = ast.parse("import os, sys", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_preimported_module_still_works(self): + """1.6: Pre-imported module (`import re`) with no extra config still works.""" + # _strip_allowed_imports strips pre-imported modules, so nothing + # reaches SecurityFilter; but if it does, default filter blocks it. + # This test verifies the stripping works. + code, extras = _strip_allowed_imports("import re\nx = 1", extra_allowed=None) + assert "auto-removed" in code # import line is commented + assert "x = 1" in code + + def test_extra_allowed_preserves_import_line(self): + """1.7: extra_allowed modules are NOT stripped by _strip_allowed_imports.""" + code, extras = _strip_allowed_imports("import os\nx = 1", extra_allowed=frozenset({"os"})) + assert "auto-removed" not in code + assert "import os" in code + assert "x = 1" in code + + +# --------------------------------------------------------------------------- +# Test Group 2: allowed_builtins +# --------------------------------------------------------------------------- + + +class TestAllowedBuiltins: + """Tests for the `allowed_builtins` orchestration config option.""" + + def test_setattr_with_allowed_builtins(self): + """2.1: `setattr` is available when allowed_builtins includes it.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["setattr"]}, + ) + assert "setattr" in env._safe_builtins + assert env._safe_builtins["setattr"] is setattr + + def test_setattr_without_allowed(self): + """2.2: `setattr` raises NameError when not allowed.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={}, + ) + assert "setattr" not in env._safe_builtins + + def test_property_with_allowed_builtins(self): + """2.3: `property` is available when allowed.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["property"]}, + ) + assert "property" in env._safe_builtins + assert env._safe_builtins["property"] is property + + def test_eval_in_allowed_builtins_raises(self): + """2.4: `eval` in allowed_builtins raises ValueError.""" + with pytest.raises(ValueError, match="dangerous builtin"): + AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["eval"]}, + ) + + def test_open_in_allowed_builtins_raises(self): + """2.5: `open` in allowed_builtins raises ValueError.""" + with pytest.raises(ValueError, match="dangerous builtin"): + AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["open"]}, + ) + + def test_dunder_in_allowed_builtins_raises(self): + """2.6: `__import__` in allowed_builtins raises ValueError.""" + with pytest.raises(ValueError, match="private builtin"): + AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"allowed_builtins": ["__import__"]}, + ) + + +# --------------------------------------------------------------------------- +# Test Group 3: allow_classes +# --------------------------------------------------------------------------- + + +class TestAllowClassesSecurityFilter: + """Tests for the `allow_classes` option in SecurityFilter.""" + + def test_class_def_allowed(self): + """3.1: `class A: pass` with allow_classes=True succeeds.""" + sf = SecurityFilter(allow_classes=True) + tree = ast.parse("class A:\n pass", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_class_def_blocked(self): + """3.2: `class A: pass` with allow_classes=False is blocked.""" + assert not _run_security_filter_safe("class A:\n pass") + + def test_class_with_init_allowed(self): + """3.3: __init__ inside class is allowed with allow_classes=True.""" + sf = SecurityFilter(allow_classes=True) + code = "class A:\n def __init__(self):\n pass" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump + + def test_private_attr_store_inside_class_blocked(self): + """3.4: `self.__x = 1` inside class is still blocked.""" + sf = SecurityFilter(allow_classes=True) + code = "class A:\n def __init__(self):\n self.__x = 1" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_dunder_outside_class_blocked(self): + """3.5: `obj.__class__` at module level blocked even with allow_classes.""" + sf = SecurityFilter(allow_classes=True) + tree = ast.parse("x = obj.__class__", mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_evil_dunder_inside_class_blocked(self): + """3.6: __evil__ inside class blocked (not in SAFE_CLASS_DUNDERS).""" + sf = SecurityFilter(allow_classes=True) + code = "class A:\n def __evil__(self):\n pass" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" in dump + + def test_other_safe_dunders_allowed(self): + """Additional: __str__, __repr__, __iter__ etc. are allowed in class body.""" + for dunder in ["__str__", "__repr__", "__iter__", "__len__", "__enter__", "__exit__"]: + sf = SecurityFilter(allow_classes=True) + code = f"class A:\n def {dunder}(self):\n pass" + tree = ast.parse(code, mode="exec") + rewritten = sf.visit(tree) + dump = ast.dump(rewritten) + assert "__security_raise" not in dump, f"{dunder} should be allowed" + + +# --------------------------------------------------------------------------- +# Test Group 4: disable_security +# --------------------------------------------------------------------------- + + +class TestDisableSecurity: + """Tests for the `disable_security` orchestration config option.""" + + @pytest.mark.asyncio + async def test_eval_with_disable_security(self): + """4.1: `eval("1+1")` with disable_security=True succeeds.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_security": True}, + ) + result = await env.execute("print(eval('1+1'))") + assert "2" in result["results"] + + @pytest.mark.asyncio + async def test_eval_without_disable_security(self): + """4.2: `eval("1+1")` with disable_security=False raises SecurityError.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={}, + ) + result = await env.execute("print(eval('1+1'))") + assert "Security Error" in result["results"] or "forbidden" in result["results"].lower() + + def test_import_os_disable_security(self): + """4.3: `import os` with disable_security=True, no allowed_imports, succeeds.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_security": True}, + ) + # The SecurityFilter is skipped entirely, so import passes AST + # (Runtime exec hit depends on the module being available) + assert env._orchestration_config["disable_security"] is True + + @pytest.mark.asyncio + async def test_dunder_access_disable_security(self): + """4.4: `obj.__class__` with disable_security=True succeeds.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_security": True}, + ) + result = await env.execute("obj = 1\nprint(obj.__class__.__name__)") + assert "int" in result["results"] + + +# --------------------------------------------------------------------------- +# Test Group 5: disable_loop_protection +# --------------------------------------------------------------------------- + + +class TestDisableLoopProtection: + """Tests for the `disable_loop_protection` orchestration config option.""" + + @pytest.mark.asyncio + async def test_while_loop_with_disable_loop_protection(self): + """5.1: while loop with disable_loop_protection=True has no yield injection.""" + code = "while True:\n break" + # Without loop protection, the tree should NOT have __yield + # (LoopYieldInjector is skipped in execute, but here we test directly) + # Default: yield IS injected + lyi = LoopYieldInjector() + with_yield = lyi.visit(ast.parse(code, mode="exec")) + assert _get_loop_yield_count(with_yield) == 1 + + @pytest.mark.asyncio + async def test_env_with_disable_loop_protection(self): + """5.3: for loop with disable_loop_protection=True runs fine.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={"disable_loop_protection": True}, + ) + result = await env.execute("for i in range(10):\n print(i)") + assert result["results"] is not None + + +# --------------------------------------------------------------------------- +# Test Group 6: Combination / Integration +# --------------------------------------------------------------------------- + + +class TestOrchestrationIntegration: + """Integration tests combining multiple orchestration config options.""" + + @pytest.mark.asyncio + async def test_combined_imports_and_classes(self): + """6.1: allowed_imports + allow_classes works end-to-end.""" + env = AgentExecutionEnv( + _make_mock_coder(), + orchestration_config={ + "allowed_imports": ["os", "typing"], + "allow_classes": True, + }, + ) + result = await env.execute( + "import os\n" + "import typing\n" + "class A:\n" + " def __init__(self):\n" + " pass\n" + "print('ok')\n" + ) + assert "ok" in result["results"] + + def test_empty_orchestration_config(self): + """6.2: Empty orchestration: {} behaves same as no config.""" + env1 = AgentExecutionEnv(_make_mock_coder(), orchestration_config={}) + env2 = AgentExecutionEnv(_make_mock_coder()) + assert env1._safe_builtins.keys() == env2._safe_builtins.keys() + + def test_context_block_with_config(self): + """6.3: build_orchestration_context_block includes overrides section.""" + config = { + "allow_orchestration": True, + "orchestration": { + "allowed_imports": ["os"], + "allow_classes": True, + }, + } + block = build_orchestration_context_block(config) + assert block is not None + assert "Sandbox Configuration Overrides" in block + assert "os" in block + assert "Class definitions" in block + + def test_context_block_with_empty_config_no_overrides(self): + """6.4: build_orchestration_context_block with empty config unchanged.""" + config = {"allow_orchestration": True} + block = build_orchestration_context_block(config) + assert block is not None + assert "Sandbox Configuration Overrides" not in block + + def test_context_block_with_disable_security_warning(self): + """Context block shows warning for disable_security.""" + config = { + "allow_orchestration": True, + "orchestration": {"disable_security": True}, + } + block = build_orchestration_context_block(config) + assert "Security filtering is DISABLED" in block diff --git a/tests/mcp/test_keepalive_integration.py b/tests/mcp/test_keepalive_integration.py index f94d343052c..47a6fb615ba 100644 --- a/tests/mcp/test_keepalive_integration.py +++ b/tests/mcp/test_keepalive_integration.py @@ -68,10 +68,18 @@ async def test_server_failure_triggers_unhealthy_state( running_mock_server.set_status(500) # Wait for failed ping - await asyncio.sleep(1.2) + # Wait for failed ping with polling for cross-platform reliability + for _ in range(30): # 30 * 0.2s = 6s timeout + if inspector.get_state(server) == ConnectionState.UNHEALTHY: + break + await asyncio.sleep(0.2) + else: + pytest.fail( + f"Server did not become UNHEALTHY after failures, state was " + f"{inspector.get_state(server).name}" + ) # Should transition to UNHEALTHY - assert inspector.get_state(server) == ConnectionState.UNHEALTHY assert inspector.get_failed_pings(server) == 1 await server.disconnect() @@ -115,14 +123,33 @@ async def test_successful_ping_after_failure_restores_healthy_state( # Cause a failure running_mock_server.set_status(500) - await asyncio.sleep(1.2) + for _ in range(30): # 30 * 0.2s = 6s timeout + if inspector.get_state(server) == ConnectionState.UNHEALTHY: + break + await asyncio.sleep(0.2) + else: + pytest.fail( + f"Server did not become UNHEALTHY after failures, state was " + f"{inspector.get_state(server).name}" + ) assert inspector.get_state(server) == ConnectionState.UNHEALTHY # Restore success running_mock_server.set_status(200) - await asyncio.sleep(1.2) + for _ in range(30): # 30 * 0.2s = 6s timeout + if inspector.get_state(server) == ConnectionState.CONNECTED: + break + await asyncio.sleep(0.2) + else: + pytest.fail( + f"Server did not become CONNECTED after success, state was " + f"{inspector.get_state(server).name}" + ) # Should be back to CONNECTED + + # Brief cooldown after state transition + await asyncio.sleep(0.2) assert inspector.get_state(server) == ConnectionState.CONNECTED assert inspector.get_failed_pings(server) == 0 diff --git a/tests/subagents/test_delegate.py b/tests/subagents/test_delegate.py index 97cb4ec9cf2..71d0bd8bcae 100644 --- a/tests/subagents/test_delegate.py +++ b/tests/subagents/test_delegate.py @@ -16,8 +16,9 @@ async def test_empty_name_returns_error(self): from cecli.tools.delegate import Tool result = await Tool.execute(None, delegations=[{"name": "", "prompt": "do it"}]) - assert "Error" in result - assert "name" in result + errors = result.to_dict()["errors"] + assert errors + assert "name" in errors[0] @pytest.mark.asyncio async def test_empty_prompt_returns_error(self): @@ -25,8 +26,9 @@ async def test_empty_prompt_returns_error(self): from cecli.tools.delegate import Tool result = await Tool.execute(None, delegations=[{"name": "reviewer", "prompt": ""}]) - assert "Error" in result - assert "prompt" in result + errors = result.to_dict()["errors"] + assert errors + assert "prompt" in errors[0] @pytest.mark.asyncio async def test_both_empty_returns_name_error(self): @@ -34,8 +36,9 @@ async def test_both_empty_returns_name_error(self): from cecli.tools.delegate import Tool result = await Tool.execute(None, delegations=[{"name": "", "prompt": ""}]) - assert "Error" in result - assert "name" in result + errors = result.to_dict()["errors"] + assert errors + assert "name" in errors[0] @pytest.mark.asyncio async def test_valid_delegate_calls_spawn(self): @@ -61,8 +64,8 @@ async def test_valid_delegate_calls_spawn(self): mock_instance.spawn.assert_called_once_with( "reviewer", "review this", parent=mock_coder ) - assert "agent started with id" in result - assert "child-uuid-123" in result + assert "agent started with id" in str(result) + assert "child-uuid-123" in str(result) async def test_delegate_multiple_delegations(self): """Multiple delegations show correct dispatch count.""" @@ -90,9 +93,9 @@ async def spawn_side_effect(name, prompt, parent=None): ], ) - assert "2/2 dispatched" in result - assert "agent1" in result - assert "agent2" in result + assert "2/2 dispatched" in str(result) + assert "agent1" in str(result) + assert "agent2" in str(result) @pytest.mark.asyncio async def test_delegate_spawn_error_returns_error_string(self): @@ -106,8 +109,8 @@ async def test_delegate_spawn_error_returns_error_string(self): MockService.get_instance.return_value = mock_instance result = await Tool.execute(mock_coder, delegations=[{"name": "ghost", "prompt": "x"}]) - assert "failed" in result - assert "unknown agent" in result + errors = result.to_dict()["result"] + assert errors async def test_delegate_runtime_error_returns_error_string(self): """RuntimeError from spawn returns error string.""" @@ -122,8 +125,8 @@ async def test_delegate_runtime_error_returns_error_string(self): result = await Tool.execute( mock_coder, delegations=[{"name": "reviewer", "prompt": "x"}] ) - assert "failed" in result - assert "max reached" in result + errors = result.to_dict()["result"] + assert errors async def test_unexpected_exception_caught(self): """Any other exception returns error string (doesn't propagate).""" @@ -138,5 +141,5 @@ async def test_unexpected_exception_caught(self): result = await Tool.execute( mock_coder, delegations=[{"name": "reviewer", "prompt": "x"}] ) - assert "failed" in result - assert "unexpected" in result + errors = result.to_dict()["result"] + assert errors diff --git a/tests/subagents/test_finished.py b/tests/subagents/test_finished.py index 3f0e5e34c43..2555225a91e 100644 --- a/tests/subagents/test_finished.py +++ b/tests/subagents/test_finished.py @@ -60,7 +60,7 @@ async def test_sub_agent_without_summary(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder) - assert result == "Yielded." + assert result.to_dict()["result"][0]["content"] == "Yielded." @pytest.mark.asyncio async def test_non_sub_agent_skips_lookup(self): @@ -73,7 +73,7 @@ async def test_non_sub_agent_skips_lookup(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder) - assert result == "Yielded." + assert result.to_dict()["result"][0]["content"] == "Yielded." @pytest.mark.asyncio async def test_unknown_parent_uuid_caught_gracefully(self): @@ -88,7 +88,7 @@ async def test_unknown_parent_uuid_caught_gracefully(self): with patch.object(AgentService, "_instances", {}): result = await Tool.execute(mock_coder, summary="done") - assert "Summary: done" in result + assert "Summary: done" in str(result) async def test_returns_summary_in_response(self): """When summary provided, response includes it.""" @@ -100,7 +100,7 @@ async def test_returns_summary_in_response(self): mock_coder.files_edited_by_tools = set() result = await Tool.execute(mock_coder, summary="completed successfully") - assert "Summary: completed successfully" in result + assert "Summary: completed successfully" in str(result) @pytest.mark.asyncio async def test_coder_is_none_returns_error(self): @@ -108,4 +108,5 @@ async def test_coder_is_none_returns_error(self): from cecli.tools._yield import Tool result = await Tool.execute(None) - assert "Error" in result + errors = result.to_dict()["errors"] + assert errors diff --git a/tests/tools/test_get_lines.py b/tests/tools/test_get_lines.py index 10a55e713a0..286e04e2556 100644 --- a/tests/tools/test_get_lines.py +++ b/tests/tools/test_get_lines.py @@ -34,6 +34,7 @@ def __init__(self, root): self.abs_fnames = set() self.abs_read_only_fnames = set() self.large_file_token_threshold = 25000 + self.file_read_cache = set() from unittest.mock import MagicMock @@ -75,7 +76,7 @@ def test_pattern_with_zero_line_number_is_allowed(coder_with_file): ) # read_file now returns a new formatted context message - assert "Retrieved context for 1 operation(s)" in result + assert "Retrieved context for 1 operation(s)" in str(result) coder.io.tool_error.assert_not_called() @@ -95,7 +96,7 @@ def test_empty_pattern_uses_line_number(coder_with_file): ) # read_file now returns a static success message - assert "Retrieved context for 1 operation(s)" in result + assert "Retrieved context for 1 operation(s)" in str(result) coder.io.tool_error.assert_not_called() @@ -115,7 +116,7 @@ def test_conflicting_pattern_and_line_number_raise(coder_with_file): ], ) - assert "Provide both 'range_start' and 'range_end'" in result + assert "Provide both 'range_start' and 'range_end'" in str(result) coder.io.tool_error.assert_called() @@ -151,7 +152,7 @@ def test_multiline_pattern_search(coder_with_file): ], ) - assert "Retrieved context for 1 operation(s)" in result + assert "Retrieved context for 1 operation(s)" in str(result) coder.io.tool_error.assert_not_called() @@ -177,7 +178,7 @@ def test_empty_file_includes_edit_hint(tmp_path): ], ) - assert "pubspec.yaml is empty" in result - assert "EditFile" in result - assert "readfile again" in result.lower() + assert "pubspec.yaml is empty" in str(result) + assert "EditFile" in str(result) + assert "readfile again" in str(result).lower() coder.io.tool_error.assert_not_called() diff --git a/tests/tools/test_git_branch.py b/tests/tools/test_git_branch.py index 392f06a5ca5..0d4ddc75ab2 100644 --- a/tests/tools/test_git_branch.py +++ b/tests/tools/test_git_branch.py @@ -29,7 +29,7 @@ def test_gitbranch_show_current_returns_branch_name(): result = git_branch.Tool.execute(coder, show_current=True) - assert result.strip() == "feature" + assert result.to_dict()["result"][0]["content"].strip() == "feature" def test_gitbranch_show_current_handles_detached_head(): @@ -48,4 +48,4 @@ def test_gitbranch_show_current_handles_detached_head(): result = git_branch.Tool.execute(coder, show_current=True) - assert result.strip() == "HEAD (detached)" + assert result.to_dict()["result"][0]["content"].strip() == "HEAD (detached)" diff --git a/tests/tools/test_git_diff.py b/tests/tools/test_git_diff.py index 0daad3d2d01..68c98997073 100644 --- a/tests/tools/test_git_diff.py +++ b/tests/tools/test_git_diff.py @@ -26,4 +26,4 @@ def test_gitdiff_head_argument_includes_working_tree_changes(): result = git_diff.Tool.execute(coder, branch="HEAD") - assert "updated" in result + assert "updated" in str(result) diff --git a/tests/tools/test_grep.py b/tests/tools/test_grep.py index 0228e98f042..9030f883468 100644 --- a/tests/tools/test_grep.py +++ b/tests/tools/test_grep.py @@ -51,13 +51,12 @@ def test_dash_prefixed_pattern_is_searched_literally(search_term, tmp_path, monk ], ) - import json - - data = json.loads(result) - assert "operations" in data - assert len(data["operations"]) == 1 - op = data["operations"][0] - assert op["pattern"] == search_term - assert op["error"] is None - assert "total_files" in op + response_dict = result.to_dict() + operations = response_dict["result"] + assert len(operations) == 1 + op = operations[0] + assert op["_"]["pattern"] == search_term + assert op["_"]["error"] is None + assert "total_files" in op["_"] + assert isinstance(op["_"]["total_files"], int) coder.io.tool_error.assert_not_called() diff --git a/tests/tools/test_insert_block.py b/tests/tools/test_insert_block.py index ffca6f8e77b..856fbaed6c0 100644 --- a/tests/tools/test_insert_block.py +++ b/tests/tools/test_insert_block.py @@ -52,6 +52,7 @@ def __init__(self, root): self.abs_read_only_fnames = set() self.abs_fnames = set() self.edit_allowed = True + self.file_read_cache = set() def abs_root_path(self, file_path): path = Path(file_path) @@ -97,7 +98,7 @@ def test_position_top_succeeds_with_no_patterns(coder_with_file): ], ) - assert result.startswith("Successfully executed EditFile.") + assert "Applied" in result.to_dict()["result"][0]["content"] lines = file_path.read_text().splitlines() # Inserted line replaces first line (inclusive bounds) assert lines[1] == "second line" # Original second line shifts up @@ -121,10 +122,10 @@ def test_mutually_exclusive_parameters_raise(coder_with_file): ], ) - assert result.startswith("Error in EditFile:") - assert "Invalid Edit - Review content ID bounds" in result + assert any("Invalid Edit" in e for e in result.to_dict()["errors"]) + assert "Invalid Edit - Review content ID bounds" in str(result) assert file_path.read_text().startswith("first line") - coder.io.tool_error.assert_called() + coder.io.tool_error.assert_not_called() def test_trailing_newline_preservation(coder_with_file): @@ -221,7 +222,7 @@ def test_line_number_beyond_file_length_appends(coder_with_file): ], ) - assert result.startswith("Successfully executed EditFile.") + assert "Applied" in result.to_dict()["result"][0]["content"] content = file_path.read_text() assert content == "first line\nappended line\n" coder.io.tool_error.assert_not_called() diff --git a/tests/tools/test_read_range_execute.py b/tests/tools/test_read_range_execute.py index 7006f0b0185..7da45f775d7 100644 --- a/tests/tools/test_read_range_execute.py +++ b/tests/tools/test_read_range_execute.py @@ -125,7 +125,7 @@ def _setup(self, mock_coder, mock_file_context, mock_chunks, mock_manager, file_ # Patch hashline_formatted to return (text, json) hl_patch = patch( "cecli.tools.read_file.hashline_formatted", - side_effect=lambda text, file_name, partial, expanded, start_line=1: (text, "{}"), + side_effect=lambda text, file_name, total_lines=0, start_line=1: (text, "{}"), ) hl_patch.start() self.patches.append(hl_patch) @@ -169,11 +169,10 @@ def test_both_digits_valid_range( content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "5", "range_end": "10"}] + show = [{"file_path": self.test_file, "range_start": "@L5", "range_end": "@L10"}] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result - assert "line5" in result - assert "line10" in result + assert "line5" in str(result) + assert "line10" in str(result) finally: self._teardown() @@ -182,9 +181,9 @@ def test_both_digits_same_line(self, mock_coder, mock_file_context, mock_chunks, content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "1", "range_end": "1"}] + show = [{"file_path": self.test_file, "range_start": "@L1", "range_end": "@L1"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result + assert "line1" in str(result) finally: self._teardown() @@ -195,10 +194,10 @@ def test_both_digits_out_of_bounds( content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "1", "range_end": "100"}] + show = [{"file_path": self.test_file, "range_start": "@L1", "range_end": "@L100"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result - assert "line10" in result + assert "line1" in str(result) + assert "line10" in str(result) finally: self._teardown() @@ -209,7 +208,7 @@ def test_both_digits_inverted_order( content = "\n".join(f"line{i}" for i in range(1, 11)) self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "10", "range_end": "5"}] + show = [{"file_path": self.test_file, "range_start": "@L10", "range_end": "@L5"}] result = self.Tool.execute(self.coder, show) # Inverted: start=[9], end=[4], only one each -> swap to (4, 9) assert result is not None @@ -227,8 +226,8 @@ def test_special_start_end(self, mock_coder, mock_file_context, mock_chunks, moc try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result - assert "line5" in result + assert "line1" in str(result) + assert "line5" in str(result) finally: self._teardown() @@ -239,7 +238,7 @@ def test_special_start_at_000(self, mock_coder, mock_file_context, mock_chunks, try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "@000"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result + assert "line1" in str(result) finally: self._teardown() @@ -250,7 +249,7 @@ def test_special_end_at_000(self, mock_coder, mock_file_context, mock_chunks, mo try: show = [{"file_path": self.test_file, "range_start": "000@", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "line5" in result + assert "line5" in str(result) finally: self._teardown() @@ -265,10 +264,10 @@ def test_special_start_digit_end( content = "line1\nline2\nline3\nline4\nline5" self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "3"}] + show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "@L3"}] result = self.Tool.execute(self.coder, show) - assert "line1" in result - assert "line3" in result + assert "line1" in str(result) + assert "line3" in str(result) finally: self._teardown() @@ -279,10 +278,10 @@ def test_digit_start_special_end( content = "line1\nline2\nline3\nline4\nline5" self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, content) try: - show = [{"file_path": self.test_file, "range_start": "2", "range_end": "000@"}] + show = [{"file_path": self.test_file, "range_start": "@L2", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "line2" in result - assert "line5" in result + assert "line2" in str(result) + assert "line5" in str(result) finally: self._teardown() @@ -305,9 +304,8 @@ def test_both_text_patterns(self, mock_coder, mock_file_context, mock_chunks, mo } ] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result - assert "def foo()" in result - assert "def bar()" in result + assert "def foo()" in str(result) + assert "def bar()" in str(result) finally: self._teardown() @@ -324,7 +322,7 @@ def test_text_pattern_not_found(self, mock_coder, mock_file_context, mock_chunks } ] result = self.Tool.execute(self.coder, show) - assert "Errors" in result or "not found" in result + assert "Errors" in str(result) or "not found" in str(result) finally: self._teardown() @@ -335,7 +333,7 @@ def test_text_pattern_multiline(self, mock_coder, mock_file_context, mock_chunks try: show = [{"file_path": self.test_file, "range_start": "def foo", "range_end": "def bar"}] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result + assert "return 1" in str(result) finally: self._teardown() @@ -355,7 +353,7 @@ def test_special_start_text_end(self, mock_coder, mock_file_context, mock_chunks show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "debug_mode"}] result = self.Tool.execute(self.coder, show) # Should find '@000' at start and 'debug_mode' as text - print(f"\n[special_start_text_end] result: {result[:300]}") + print(f"\n[special_start_text_end] result: {str(result)[:300]}") assert result is not None finally: self._teardown() @@ -373,7 +371,7 @@ def test_text_start_special_end(self, mock_coder, mock_file_context, mock_chunks {"file_path": self.test_file, "range_start": "config_value", "range_end": "000@"} ] result = self.Tool.execute(self.coder, show) - print(f"\n[text_start_special_end] result: {result[:300]}") + print(f"\n[text_start_special_end] result: {str(result)[:300]}") assert result is not None finally: self._teardown() @@ -388,7 +386,7 @@ def test_empty_file(self, mock_coder, mock_file_context, mock_chunks, mock_manag try: show = [{"file_path": self.test_file, "range_start": "@000", "range_end": "000@"}] result = self.Tool.execute(self.coder, show) - assert "empty" in result.lower() + assert "empty" in str(result).lower() finally: self._teardown() @@ -396,9 +394,9 @@ def test_single_line_file(self, mock_coder, mock_file_context, mock_chunks, mock """Test with a single line file.""" self._setup(mock_coder, mock_file_context, mock_chunks, mock_manager, "only_line") try: - show = [{"file_path": self.test_file, "range_start": "1", "range_end": "1"}] + show = [{"file_path": self.test_file, "range_start": "@L1", "range_end": "@L1"}] result = self.Tool.execute(self.coder, show) - assert "only_line" in result + assert "only_line" in str(result) finally: self._teardown() @@ -417,9 +415,9 @@ def test_file_not_found(self, mock_coder, mock_file_context, mock_chunks, mock_m from cecli.tools.read_file import Tool - show = [{"file_path": "nonexistent/path.py", "range_start": "1", "range_end": "10"}] + show = [{"file_path": "nonexistent/path.py", "range_start": "@L1", "range_end": "@L10"}] result = Tool.execute(mock_coder, show) - assert "not found" in result or "Errors" in result + assert "not found" in str(result) or "Errors" in str(result) def test_missing_parameters(self, mock_coder, mock_file_context, mock_chunks, mock_manager): """Test with missing range_start and range_end (empty strings).""" @@ -427,7 +425,7 @@ def test_missing_parameters(self, mock_coder, mock_file_context, mock_chunks, mo show = [{"file_path": "some_file.py", "range_start": "", "range_end": ""}] result = Tool.execute(mock_coder, show) - assert "Provide both" in result or "Errors" in result + assert "Provide both" in str(result) or "Errors" in str(result) def test_multiple_show_operations( self, mock_coder, mock_file_context, mock_chunks, mock_manager @@ -451,7 +449,7 @@ def resolve_side_effect(coder, file_path): hl_patch = patch( "cecli.tools.read_file.hashline_formatted", - side_effect=lambda text, file_name, partial, expanded, start_line=1: (text, "{}"), + side_effect=lambda text, file_name, total_lines=0, start_line=1: (text, "{}"), ) hl_patch.start() @@ -478,12 +476,12 @@ def resolve_side_effect(coder, file_path): Tool._last_read_turn = {} show = [ - {"file_path": "file1.py", "range_start": "1", "range_end": "3"}, - {"file_path": "file2.py", "range_start": "2", "range_end": "4"}, + {"file_path": "file1.py", "range_start": "@L1", "range_end": "@L3"}, + {"file_path": "file2.py", "range_start": "@L2", "range_end": "@L4"}, ] result = Tool.execute(mock_coder, show) - assert "line1_1" in result - assert "line2_2" in result + assert "line1_1" in str(result) + assert "line2_2" in str(result) finally: for p in [cs_patch, sh_patch, hl_patch, rp_patch, ip_patch]: p.stop() @@ -524,7 +522,7 @@ def func_f(): } ] result = self.Tool.execute(self.coder, show) - assert "prefixed_contents" in result + assert "def func_a" in str(result) finally: self._teardown() @@ -554,7 +552,7 @@ def func_f(): try: show = [{"file_path": self.test_file, "range_start": "def", "range_end": "def"}] result = self.Tool.execute(self.coder, show) - assert "too broad" in result.lower() + assert "too broad" in str(result).lower() finally: self._teardown() diff --git a/tests/tools/test_remove_mcp_tool.py b/tests/tools/test_remove_mcp_tool.py index ddd2e6e6045..2da2e85e37c 100644 --- a/tests/tools/test_remove_mcp_tool.py +++ b/tests/tools/test_remove_mcp_tool.py @@ -64,7 +64,7 @@ async def test_no_configured_servers(self, coder): """Test when no MCP servers are configured at all.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result == "No MCP servers are configured." + assert result.to_dict()["result"][0]["content"] == "No MCP servers are configured." @pytest.mark.asyncio async def test_server_not_found(self, coder, mock_server): @@ -73,7 +73,7 @@ async def test_server_not_found(self, coder, mock_server): coder.mcp_manager.connected_servers = {"existing": "server"} coder.mcp_manager.get_server.return_value = None result = await ResourceManagerTool.execute(coder, remove_mcp=["nonexistent"]) - assert "MCP server nonexistent does not exist." in result + assert "MCP server nonexistent does not exist." in str(result) @pytest.mark.asyncio async def test_all_servers_not_loaded(self, coder, mock_server): @@ -82,7 +82,7 @@ async def test_all_servers_not_loaded(self, coder, mock_server): coder.mcp_manager.connected_servers = {} coder.mcp_manager.get_server.return_value = mock_server result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Server test-server is not currently connected." in result + assert "Server test-server is not currently connected." in str(result) @pytest.mark.asyncio async def test_successful_removal(self, coder, mock_server): @@ -92,7 +92,7 @@ async def test_successful_removal(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Removed server: test-server" in result + assert "Removed server: test-server" in str(result) @pytest.mark.asyncio async def test_removal_interrupted(self, coder, mock_server): @@ -102,7 +102,7 @@ async def test_removal_interrupted(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, True) result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio async def test_removal_failed(self, coder, mock_server): @@ -112,7 +112,7 @@ async def test_removal_failed(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, False) result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Unable to remove server: test-server" in result + assert "Unable to remove server: test-server" in str(result) @pytest.mark.asyncio async def test_remove_all_servers(self, coder): @@ -128,8 +128,8 @@ async def test_remove_all_servers(self, coder): ) coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) @pytest.mark.asyncio async def test_mixed_results(self, coder): @@ -153,5 +153,5 @@ async def mock_interruptible_func(*args, **kwargs): coder.coroutines.interruptible.side_effect = mock_interruptible_func result = await ResourceManagerTool.execute(coder, remove_mcp=["server1", "server2"]) - assert "Removed server: server1" in result - assert "Unable to remove server: server2" in result + assert "Removed server: server1" in str(result) + assert "Unable to remove server: server2" in str(result) diff --git a/tests/tools/test_tools_load_mcp_tool.py b/tests/tools/test_tools_load_mcp_tool.py index d5a40be0dd8..6e9071cb711 100644 --- a/tests/tools/test_tools_load_mcp_tool.py +++ b/tests/tools/test_tools_load_mcp_tool.py @@ -55,7 +55,7 @@ async def test_no_mcp_servers_found(self, coder): """Test when no MCP servers are configured.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, load_mcp=["test"]) - assert result == "No MCP servers found, nothing to load." + assert result.to_dict()["result"][0]["content"] == "No MCP servers found, nothing to load." @pytest.mark.asyncio async def test_server_not_found(self, coder, mock_server): @@ -63,7 +63,7 @@ async def test_server_not_found(self, coder, mock_server): coder.mcp_manager.servers = [mock_server] coder.mcp_manager.get_server.return_value = None result = await ResourceManagerTool.execute(coder, load_mcp=["nonexistent"]) - assert "MCP server nonexistent does not exist." in result + assert "MCP server nonexistent does not exist." in str(result) @pytest.mark.asyncio async def test_server_already_loaded(self, coder, mock_server): @@ -75,7 +75,7 @@ async def test_server_already_loaded(self, coder, mock_server): # Must return tuple (did_connect, interrupted) coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Server already loaded: test-server" in result + assert "Server already loaded: test-server" in str(result) @pytest.mark.asyncio async def test_server_not_enabled_by_default(self, coder, mock_server): @@ -93,7 +93,7 @@ async def test_successful_load(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Loaded server: test-server" in result + assert "Loaded server: test-server" in str(result) @pytest.mark.asyncio async def test_load_interrupted(self, coder, mock_server): @@ -103,7 +103,7 @@ async def test_load_interrupted(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, True) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio async def test_load_failed(self, coder, mock_server): @@ -113,7 +113,7 @@ async def test_load_failed(self, coder, mock_server): coder.mcp_manager.get_server.return_value = mock_server coder.coroutines.interruptible.return_value = (False, False) result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Unable to load server: test-server" in result + assert "Unable to load server: test-server" in str(result) @pytest.mark.asyncio async def test_load_all_servers(self, coder): @@ -131,8 +131,8 @@ async def test_load_all_servers(self, coder): ) coder.coroutines.interruptible.return_value = (True, False) result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) - assert "Loaded server: server1" in result - assert "Loaded server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Loaded server: server2" in str(result) @pytest.mark.asyncio async def test_mixed_results(self, coder): @@ -161,8 +161,8 @@ async def mock_interruptible_func(*args, **kwargs): coder.coroutines.interruptible.side_effect = mock_interruptible_func result = await ResourceManagerTool.execute(coder, load_mcp=["server1", "server2"]) - assert "Loaded server: server1" in result - assert "Unable to load server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Unable to load server: server2" in str(result) @pytest.mark.asyncio async def test_duplicate_iteration_bug_fix(self, coder, mock_server): @@ -176,7 +176,7 @@ async def test_duplicate_iteration_bug_fix(self, coder, mock_server): result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) # Should only report server already loaded once - assert result.count("Server already loaded: test-server") == 1 + assert str(result).count("Server already loaded: test-server") == 1 # connect_server should not have been called since it was already loaded coder.mcp_manager.connect_server.assert_not_called() @@ -214,5 +214,5 @@ async def mock_interruptible(coro, event): # Should only attempt to load server2 (server1 should be skipped) # Wildcard expansion skips already-connected servers, so server1 is not reported - assert "Loaded server: server2" in result + assert "Loaded server: server2" in str(result) assert connect_calls == ["server2"] # Only server2 should have been connected diff --git a/tests/unit/test_load_mcp.py b/tests/unit/test_load_mcp.py index 1f33511a4b3..17380a33252 100644 --- a/tests/unit/test_load_mcp.py +++ b/tests/unit/test_load_mcp.py @@ -51,7 +51,7 @@ async def test_no_mcp_servers_found(coder): """Test when no MCP servers are configured.""" coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, load_mcp=["test"]) - assert result == "No MCP servers found, nothing to load." + assert result.to_dict()["result"][0]["content"] == "No MCP servers found, nothing to load." @pytest.mark.asyncio @@ -60,7 +60,7 @@ async def test_server_not_found(coder, mock_server): coder.mcp_manager.servers = [mock_server] coder.mcp_manager.get_server.return_value = None result = await ResourceManagerTool.execute(coder, load_mcp=["nonexistent"]) - assert "MCP server nonexistent does not exist." in result + assert "MCP server nonexistent does not exist." in str(result) @pytest.mark.asyncio @@ -78,7 +78,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Server already loaded: test-server" in result + assert "Server already loaded: test-server" in str(result) # connect_server should not have been called since it was already loaded coder.mcp_manager.connect_server.assert_not_called() @@ -93,8 +93,8 @@ async def test_server_not_enabled_by_default(coder, mock_server): coder.mcp_manager.connect_server = AsyncMock() result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) # Non-enabled servers are silently filtered by wildcard expansion - assert result == "" - coder.mcp_manager.connect_server.assert_not_called() + # Result is an empty list since nothing was loaded + assert result.to_dict()["result"] == [] @pytest.mark.asyncio @@ -116,7 +116,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Loaded server: test-server" in result + assert "Loaded server: test-server" in str(result) @pytest.mark.asyncio @@ -138,7 +138,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio @@ -157,7 +157,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Unable to load server: test-server" in result + assert "Unable to load server: test-server" in str(result) @pytest.mark.asyncio @@ -187,8 +187,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) - assert "Loaded server: server1" in result - assert "Loaded server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Loaded server: server2" in str(result) @pytest.mark.asyncio @@ -222,8 +222,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible result = await ResourceManagerTool.execute(coder, load_mcp=["server1", "server2"]) - assert "Loaded server: server1" in result - assert "Unable to load server: server2" in result + assert "Loaded server: server1" in str(result) + assert "Unable to load server: server2" in str(result) @pytest.mark.asyncio @@ -238,7 +238,7 @@ async def test_duplicate_iteration_bug_fix(coder, mock_server): coder.mcp_manager.connect_server = AsyncMock() result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) # Should only report server already loaded once - assert result.count("Server already loaded: test-server") == 1 + assert str(result).count("Server already loaded: test-server") == 1 # connect_server should not have been called since it was already loaded coder.mcp_manager.connect_server.assert_not_called() @@ -276,6 +276,6 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, load_mcp=["*"]) # Should only attempt to load server2 (server1 should be skipped) # server1 is already connected so it's skipped silently by wildcard expansion - assert "Server already loaded: server1" not in result - assert "Loaded server: server2" in result + assert "Server already loaded: server1" not in str(result) + assert "Loaded server: server2" in str(result) assert connect_calls == ["server2"] # Only server2 should have been connected diff --git a/tests/unit/test_remove_mcp.py b/tests/unit/test_remove_mcp.py index 635fddcbbdc..2ac873697de 100644 --- a/tests/unit/test_remove_mcp.py +++ b/tests/unit/test_remove_mcp.py @@ -71,7 +71,7 @@ async def mock_interruptible(coro, event): # Execute result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) # Assertions - assert "Removed server: test-server" in result + assert "Removed server: test-server" in str(result) coder.mcp_manager.disconnect_server.assert_awaited_once_with("test-server") @@ -91,7 +91,7 @@ async def test_remove_mcp_tool_non_existent(): # Execute result = await ResourceManagerTool.execute(coder, remove_mcp=["non-existent-server"]) # Assertions - assert "MCP server non-existent-server does not exist." in result + assert "MCP server non-existent-server does not exist." in str(result) @pytest.mark.asyncio @@ -106,7 +106,7 @@ async def test_remove_mcp_tool_not_connected(): coder.mcp_manager.get_server.return_value = server coder.mcp_manager.connected_servers = {} result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Server test-server is not currently connected." in result + assert "Server test-server is not currently connected." in str(result) @pytest.mark.asyncio @@ -139,8 +139,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) @pytest.mark.asyncio @@ -166,7 +166,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio @@ -192,7 +192,7 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Unable to remove server: test-server" in result + assert "Unable to remove server: test-server" in str(result) @pytest.mark.asyncio @@ -203,7 +203,7 @@ async def test_remove_mcp_tool_no_servers_configured(): coder.mcp_manager = MagicMock() coder.mcp_manager.servers = [] result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result == "No MCP servers are configured." + assert result.to_dict()["result"][0]["content"] == "No MCP servers are configured." @pytest.mark.asyncio @@ -237,8 +237,8 @@ async def mock_interruptible(coro, event): coder.coroutines.interruptible = mock_interruptible coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["server1", "server2"]) - assert "Removed server: server1" in result - assert "Unable to remove server: server2" in result + assert "Removed server: server1" in str(result) + assert "Unable to remove server: server2" in str(result) @pytest.mark.asyncio @@ -269,5 +269,5 @@ async def mock_interruptible(coro, event): coder.interrupt_event = MagicMock() result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) # Should successfully remove both servers using dictionary keys - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) diff --git a/tests/unit/test_unit_load_mcp_tool.py b/tests/unit/test_unit_load_mcp_tool.py index 9458fe5bdf5..7323059bff9 100644 --- a/tests/unit/test_unit_load_mcp_tool.py +++ b/tests/unit/test_unit_load_mcp_tool.py @@ -74,7 +74,7 @@ async def mock_interruptible(coro, event): coder.registered_servers = {"included": set(), "excluded": set()} result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Loaded server: test-server" in result + assert "Loaded server: test-server" in str(result) mock_mcp_manager.connect_server.assert_awaited_once_with("test-server") @@ -88,7 +88,7 @@ async def test_load_mcp_tool_non_existent(mock_mcp_manager): result = await ResourceManagerTool.execute(coder, load_mcp=["non-existent-server"]) - assert "MCP server non-existent-server does not exist." in result + assert "MCP server non-existent-server does not exist." in str(result) mock_mcp_manager.connect_server.assert_not_awaited() @@ -104,13 +104,14 @@ async def test_load_mcp_tool_already_loaded(mock_mcp_manager): result = await ResourceManagerTool.execute(coder, load_mcp=["test-server"]) - assert "Server already loaded: test-server" in result + assert "Server already loaded: test-server" in str(result) mock_mcp_manager.connect_server.assert_not_awaited() @pytest.mark.asyncio async def test_load_mcp_tool_wildcard_and_duplicate_fix(mock_mcp_manager): """Test loading with wildcard and duplicate fix.""" + coder = MagicMock() coder.agent_config = {"include_context_blocks": {"servers"}, "exclude_context_blocks": set()} coder.mcp_manager = mock_mcp_manager @@ -131,7 +132,7 @@ async def mock_interruptible(coro, event): # Check results # Wildcard expansion skips already-connected servers; no "already loaded" message is produced - assert "Loaded server: server2" in result + assert "Loaded server: server2" in str(result) # Non-enabled servers are filtered out silently by wildcard expansion # Verify connect_server was called only once for server2 diff --git a/tests/unit/test_unit_remove_mcp_tool.py b/tests/unit/test_unit_remove_mcp_tool.py index 822841ecb86..a5ab9e49dc9 100644 --- a/tests/unit/test_unit_remove_mcp_tool.py +++ b/tests/unit/test_unit_remove_mcp_tool.py @@ -74,7 +74,7 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) # Assertions - assert "Removed server: test-server" in result + assert "Removed server: test-server" in str(result) coder.mcp_manager.disconnect_server.assert_awaited_once_with("test-server") @@ -96,7 +96,7 @@ async def test_remove_mcp_tool_non_existent(): result = await ResourceManagerTool.execute(coder, remove_mcp=["non-existent-server"]) # Assertions - assert "MCP server non-existent-server does not exist." in result + assert "MCP server non-existent-server does not exist." in str(result) @pytest.mark.asyncio @@ -113,7 +113,7 @@ async def test_remove_mcp_tool_not_connected(): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Server test-server is not currently connected." in result + assert "Server test-server is not currently connected." in str(result) @pytest.mark.asyncio @@ -148,8 +148,8 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result) @pytest.mark.asyncio @@ -177,7 +177,7 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Interrupted: test-server" in result + assert "Interrupted: test-server" in str(result) @pytest.mark.asyncio @@ -205,7 +205,7 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["test-server"]) - assert "Unable to remove server: test-server" in result + assert "Unable to remove server: test-server" in str(result) @pytest.mark.asyncio @@ -218,7 +218,7 @@ async def test_remove_mcp_tool_no_servers_configured(): result = await ResourceManagerTool.execute(coder, remove_mcp=["test"]) - assert result == "No MCP servers are configured." + assert result.to_dict()["result"][0]["content"] == "No MCP servers are configured." @pytest.mark.asyncio @@ -255,8 +255,8 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["server1", "server2"]) - assert "Removed server: server1" in result - assert "Unable to remove server: server2" in result + assert "Removed server: server1" in str(result) + assert "Unable to remove server: server2" in str(result) @pytest.mark.asyncio @@ -289,5 +289,5 @@ async def mock_interruptible(coro, event): result = await ResourceManagerTool.execute(coder, remove_mcp=["*"]) # Should successfully remove both servers using dictionary keys - assert "Removed server: server1" in result - assert "Removed server: server2" in result + assert "Removed server: server1" in str(result) + assert "Removed server: server2" in str(result)