diff --git a/.gitignore b/.gitignore
index b9757b08..7644ce20 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,6 +48,7 @@ tests/*
!tests/test_artifact_analyzer.py
!tests/test_scala_analyzer.py
!tests/test_agent_limits.py
+!tests/test_module_scope.py
# Jupyter
*.ipynb
diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py
index f9e318be..46933d0d 100644
--- a/codewiki/cli/adapters/doc_generator.py
+++ b/codewiki/cli/adapters/doc_generator.py
@@ -278,6 +278,10 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
super_group_modules,
)
from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
+ from codewiki.src.be.module_scope import (
+ get_module_token_count,
+ validate_module_file_coverage,
+ )
from codewiki.src.utils import file_manager
working_dir = str(self.output_dir.absolute())
@@ -298,19 +302,20 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
self.progress_tracker.update_stage(0.5, "Loaded cached module tree")
else:
if self.verbose:
- clustering_tokens = get_clustering_input_token_count(leaf_nodes, components)
+ clustering_tokens = get_module_token_count(components)
self.progress_tracker.update_stage(
0.3,
(
f"Preparing {len(leaf_nodes)} leaf nodes for clustering "
- f"({clustering_tokens} tokens, threshold "
+ f"(code_tokens={clustering_tokens}, leaf_tokens="
+ f"{get_clustering_input_token_count(leaf_nodes, components)}, threshold "
f"{backend_config.max_token_per_module})"
),
)
if clustering_tokens <= backend_config.max_token_per_module:
self.progress_tracker.update_stage(
0.4,
- "Skipping LLM clustering; selected leaf nodes fit within the module token threshold",
+ "Skipping LLM clustering; code size fits within the module token threshold",
)
else:
self.progress_tracker.update_stage(
@@ -342,6 +347,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
from codewiki.src.be.module_naming import dedupe_module_tree_names
module_tree = dedupe_module_tree_names(module_tree)
+ validate_module_file_coverage(module_tree, components)
file_manager.save_json(module_tree, first_module_tree_path)
file_manager.save_json(module_tree, module_tree_path)
diff --git a/codewiki/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py
index 56180845..f74cdc20 100644
--- a/codewiki/src/be/agent_tools/deps.py
+++ b/codewiki/src/be/agent_tools/deps.py
@@ -20,3 +20,17 @@ class CodeWikiDeps:
# (create / str_replace / insert / undo_edit) to a docs file whose
# absolute, resolved path is not in this set. ``None`` = unrestricted.
allowed_write_paths: set[str] | None = None
+ # Entry points belonging to this agent. Sizes expand these IDs to all
+ # components in their files; delegation may never claim a sibling's file.
+ scope_component_ids: list[str] | None = None
+
+ def module_component_ids(self) -> list[str]:
+ if self.scope_component_ids is not None:
+ return self.scope_component_ids
+ if not self.path_to_current_module:
+ return list(self.components)
+ level = self.module_tree
+ for name in self.path_to_current_module:
+ info = level[name]
+ level = info.get("children", {})
+ return info.get("components", [])
diff --git a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py
index 492ac45b..69a3d90b 100644
--- a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py
+++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py
@@ -12,8 +12,12 @@
format_system_prompt,
format_user_prompt,
)
-from codewiki.src.be.utils import is_complex_module, count_tokens
-from codewiki.src.be.cluster_modules import format_potential_core_components
+from codewiki.src.be.utils import is_complex_module
+from codewiki.src.be.module_scope import (
+ component_files,
+ get_module_token_count,
+ prepare_submodule_partition,
+)
import logging
from codewiki.src.be.api_usage import usage_target
@@ -38,6 +42,8 @@ async def generate_sub_module_documentation(
deps = ctx.deps
previous_module_name = deps.current_module_name
+ if deps.current_depth >= deps.max_depth:
+ return "Maximum delegation depth reached; write this module directly."
# Create fallback models from config
fallback_models = create_fallback_models(deps.config)
@@ -64,10 +70,20 @@ async def generate_sub_module_documentation(
if requested_name in name_map
}
- # add the sub-module to the module tree
value = deps.module_tree
for key in deps.path_to_current_module:
value = value[key]["children"]
+ final_specs = prepare_submodule_partition(
+ final_specs, deps.module_component_ids(), deps.components, value
+ )
+ for requested, final_name in list(name_map.items()):
+ if final_name not in final_specs:
+ plan.skipped[requested] = "no unassigned files in the current module scope"
+ del name_map[requested]
+ if not final_specs:
+ return skipped_report(plan.skipped, parent_doc)
+
+ # Reserve every child's complete file scope before running any agent.
for sub_module_name, core_component_ids in final_specs.items():
value[sub_module_name] = {"components": core_component_ids, "children": {}}
@@ -78,14 +94,15 @@ async def generate_sub_module_documentation(
logger.info(f"{indent}{arrow} Generating documentation for sub-module: {sub_module_name}")
- num_tokens = count_tokens(
- format_potential_core_components(core_component_ids, ctx.deps.components)[-1]
+ num_tokens = get_module_token_count(
+ deps.components, component_files(core_component_ids, deps.components)
)
+ child_depth = deps.current_depth + 1
if (
is_complex_module(ctx.deps.components, core_component_ids)
- and ctx.deps.current_depth < ctx.deps.max_depth
- and num_tokens >= ctx.deps.config.max_token_per_leaf_module * ctx.deps.current_depth
+ and child_depth < ctx.deps.max_depth
+ and num_tokens >= ctx.deps.config.max_token_per_leaf_module * child_depth
):
sub_agent = Agent(
model=fallback_models,
@@ -119,6 +136,8 @@ async def generate_sub_module_documentation(
tools=[read_code_components_tool, str_replace_editor_tool],
)
+ previous_scope = deps.scope_component_ids
+ deps.scope_component_ids = core_component_ids
deps.current_module_name = sub_module_name
deps.path_to_current_module.append(sub_module_name)
deps.current_depth += 1
@@ -143,6 +162,7 @@ async def generate_sub_module_documentation(
deps.path_to_current_module.pop()
deps.current_depth -= 1
deps.current_module_name = previous_module_name
+ deps.scope_component_ids = previous_scope
# restore the previous module name
deps.current_module_name = previous_module_name
diff --git a/codewiki/src/be/caw_backend.py b/codewiki/src/be/caw_backend.py
index 6da530f5..86d124d0 100644
--- a/codewiki/src/be/caw_backend.py
+++ b/codewiki/src/be/caw_backend.py
@@ -31,14 +31,19 @@
from codewiki.src.be.agent_tools.deps import CodeWikiDeps
from codewiki.src.be.backend import AgentReply, LLMBackend
-from codewiki.src.be.cluster_modules import format_potential_core_components
+from codewiki.src.be.module_scope import (
+ complete_file_candidates,
+ component_files,
+ get_module_token_count,
+ validate_module_file_coverage,
+)
from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.prompt_template import (
format_leaf_system_prompt,
format_system_prompt,
format_user_prompt,
)
-from codewiki.src.be.utils import count_tokens, is_complex_module, set_main_loop
+from codewiki.src.be.utils import is_complex_module, set_main_loop
from codewiki.src.be.doc_layout import config_layout, find_doc
from codewiki.src.config import MODULE_TREE_FILENAME, OVERVIEW_FILENAME, Config
from codewiki.src.utils import file_manager
@@ -399,21 +404,18 @@ def _run_module_agent_sync(
return module_tree
custom_instructions = config.get_prompt_addition()
+ if not module_path:
+ core_component_ids = complete_file_candidates(core_component_ids, components)
- # Mirror PydanticAIBackend's early-cut: a module is only worth
- # delegating to sub-agents when it spans multiple files AND has enough
- # content to justify the cost AND we still have recursion budget.
- # Without this gate the caw path would give every multi-file sub-module
- # the recursive SYSTEM_PROMPT + delegation tool and fan out one extra
- # agent call per sub-spec even when a single leaf write would suffice.
- # See generate_sub_module_documentation_tool for the pydantic-ai
- # equivalent.
- _, components_with_code = format_potential_core_components(core_component_ids, components)
- num_tokens = count_tokens(components_with_code)
+ # Whole-repository agents may optionally delegate even below the leaf
+ # token budget. Nested modules retain the cost gate and depth limit.
+ num_tokens = get_module_token_count(
+ components, component_files(core_component_ids, components)
+ )
can_delegate = (
is_complex_module(components, core_component_ids)
and start_depth < config.max_depth
- and num_tokens >= config.max_token_per_leaf_module
+ and (not module_path or num_tokens >= config.max_token_per_leaf_module)
)
logger.info(
f"Module {module_name} can delegate: {can_delegate} - is_complex_module: {is_complex_module(components, core_component_ids)} - start_depth: {start_depth} - num_tokens: {num_tokens} - max_depth: {config.max_depth} - max_token_per_leaf_module: {config.max_token_per_leaf_module}"
@@ -440,6 +442,7 @@ def _run_module_agent_sync(
current_depth=start_depth,
config=config,
custom_instructions=custom_instructions,
+ scope_component_ids=core_component_ids,
)
toolkit = CawToolKit(deps=deps, backend=self, allow_subagent=can_delegate)
@@ -521,6 +524,7 @@ def _run_module_agent_sync(
traj.total_tool_calls,
(usage or {}).get("cost_usd", 0.0),
)
+ validate_module_file_coverage(deps.module_tree, components)
file_manager.save_json(deps.module_tree, module_tree_path)
return deps.module_tree
except Exception as e:
diff --git a/codewiki/src/be/caw_toolkit.py b/codewiki/src/be/caw_toolkit.py
index 9a111713..19f06b9e 100644
--- a/codewiki/src/be/caw_toolkit.py
+++ b/codewiki/src/be/caw_toolkit.py
@@ -30,6 +30,7 @@
from codewiki.src.be.agent_tools.deps import CodeWikiDeps
from codewiki.src.be.doc_layout import config_layout, module_doc_file
from codewiki.src.be.module_naming import skipped_report, plan_sub_module_specs, sub_module_report
+from codewiki.src.be.module_scope import prepare_submodule_partition
if TYPE_CHECKING:
from codewiki.src.be.caw_backend import CawBackend
@@ -325,6 +326,17 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str:
value = deps.module_tree
for key in deps.path_to_current_module:
value = value[key]["children"]
+ final_specs = prepare_submodule_partition(
+ final_specs, deps.module_component_ids(), deps.components, value
+ )
+ for requested, final_name in list(name_map.items()):
+ if final_name not in final_specs:
+ plan.skipped[requested] = "no unassigned files in the current module scope"
+ del name_map[requested]
+ if not final_specs:
+ return skipped_report(
+ plan.skipped, module_doc_file(previous_module_name, parent_path, layout)
+ )
for sub_name, core_ids in final_specs.items():
value[sub_name] = {"components": core_ids, "children": {}}
diff --git a/codewiki/src/be/cluster_modules.py b/codewiki/src/be/cluster_modules.py
index c72fe529..0a31d25b 100644
--- a/codewiki/src/be/cluster_modules.py
+++ b/codewiki/src/be/cluster_modules.py
@@ -8,6 +8,14 @@
from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.llm_services import call_llm
from codewiki.src.be.module_naming import resolve_unique_name, sanitize_module_name
+from codewiki.src.be.module_scope import (
+ complete_file_candidates,
+ component_files,
+ file_token_counts,
+ get_module_token_count,
+ normalize_module_partition,
+ validate_module_file_coverage,
+)
from codewiki.src.be.prompt_template import format_cluster_prompt, format_super_group_prompt
from codewiki.src.be.utils import count_tokens
from codewiki.src.config import (
@@ -20,8 +28,7 @@
Completer = Callable[[str], str | None]
-# When whole-repo mode is chosen but leaf entry points touch fewer than this
-# fraction of parsed files, warn that coverage depends on agent exploration.
+# Below this share, log why entry points were expanded before choosing a mode.
LOW_COVERAGE_RATIO = 0.5
@@ -61,7 +68,7 @@ def format_potential_core_components(
def get_clustering_input_token_count(leaf_nodes: list[str], components: dict[str, Node]) -> int:
- """Count the tokens used to decide whether a module needs clustering."""
+ """Count leaf prompt tokens for diagnostics, never for the split decision."""
_, potential_core_components_with_code = format_potential_core_components(
leaf_nodes, components
)
@@ -196,7 +203,7 @@ def _cluster_via_llm(
response_content = response.split("")[1].split("")[
0
]
- module_tree = eval(response_content)
+ module_tree = ast.literal_eval(response_content)
if not isinstance(module_tree, dict):
logger.error(f"Invalid module tree format - expected dict, got {type(module_tree)}")
@@ -213,7 +220,9 @@ def _cluster_via_llm(
logger.error(f"Traceback: {traceback.format_exc()}")
return {}
- return module_tree
+ # Validate each batch before merging untrusted LLM output. File ownership
+ # is reconciled again across batches once every response is available.
+ return normalize_module_partition(module_tree, leaf_nodes, components)
def _merge_module_trees(target: dict[str, Any], addition: dict[str, Any]) -> None:
@@ -261,6 +270,37 @@ def _batch_fallback_name(
return resolve_unique_name(name, None, set(existing))
+def _structural_fallback(
+ candidates: list[str], components: dict[str, Node], threshold: int
+) -> dict[str, Any]:
+ """Pack path-adjacent whole files by code size when the LLM cannot split."""
+ by_file = defaultdict(list)
+ for cid in candidates:
+ by_file[components[cid].relative_path].append(cid)
+ sizes = file_token_counts(components)
+ groups = []
+ current: list[str] = []
+ current_tokens = 0
+ for path in sorted(by_file):
+ if current and current_tokens + sizes[path] > threshold:
+ groups.append(current)
+ current = []
+ current_tokens = 0
+ current.extend(by_file[path])
+ current_tokens += sizes[path]
+ if current:
+ groups.append(current)
+ tree = {}
+ for group in groups:
+ name = _batch_fallback_name(group, components, tree)
+ tree[name] = {
+ "path": _common_path_prefix(sorted(component_files(group, components))),
+ "components": group,
+ "children": {},
+ }
+ return tree
+
+
def cluster_modules(
leaf_nodes: list[str],
components: dict[str, Node],
@@ -269,6 +309,8 @@ def cluster_modules(
current_module_name: str | None = None,
current_module_path: list[str] | None = None,
completer: Completer | None = None,
+ *,
+ scope_files: set[str] | None = None,
) -> dict[str, Any]:
"""
Cluster the potential core components into modules.
@@ -279,59 +321,85 @@ def cluster_modules(
``call_llm``. This is how the LLMBackend abstraction injects
subscription-mode (caw) routing. If ``None``, falls back to
``call_llm`` for backward compatibility with direct callers.
+ scope_files: exact file ownership of this invocation. The repository
+ defaults to all indexed files; a submodule defaults to the files
+ of its supplied entry points. Recursion always passes it explicitly.
"""
current_module_tree = {} if current_module_tree is None else current_module_tree
current_module_path = [] if current_module_path is None else current_module_path
- _, potential_core_components_with_code = format_potential_core_components(
- leaf_nodes, components
- )
- input_tokens = count_tokens(potential_core_components_with_code)
+ if scope_files is None:
+ scope_files = (
+ component_files(list(components), components)
+ if current_module_name is None
+ else component_files(leaf_nodes, components)
+ )
+ leaves = [
+ cid
+ for cid in leaf_nodes
+ if cid in components and components[cid].relative_path in scope_files
+ ]
+ candidates = complete_file_candidates(leaves, components, scope_files)
+ input_tokens = get_module_token_count(components, scope_files)
+ leaf_tokens = get_clustering_input_token_count(leaves, components)
+ leaf_files = component_files(leaves, components)
+ coverage = len(leaf_files) / len(scope_files) if scope_files else 1.0
threshold = config.max_token_per_module
module_label = current_module_name or "repository"
logger.info(
- "Module clustering input for %s: %d leaf nodes, %d tokens, threshold %d",
+ "Module clustering input for %s: code_tokens=%d, leaf_tokens=%d, threshold=%d, "
+ "leaf_files=%d/%d (%.0f%%), candidate_files=%d/%d, candidates=%d",
module_label,
- len(leaf_nodes),
input_tokens,
+ leaf_tokens,
threshold,
+ len(leaf_files),
+ len(scope_files),
+ coverage * 100,
+ len(component_files(candidates, components)),
+ len(scope_files),
+ len(candidates),
)
+ if coverage < LOW_COVERAGE_RATIO:
+ logger.warning(
+ "Leaf coverage for %s is below %.0f%%; added file representatives before "
+ "choosing documentation mode (%d -> %d entry points).",
+ module_label,
+ LOW_COVERAGE_RATIO * 100,
+ len(leaves),
+ len(candidates),
+ )
if input_tokens <= threshold:
logger.info(
- "Skipping LLM module clustering for %s because %d tokens fit within the "
+ "Skipping LLM module clustering for %s because %d code tokens fit within the "
"%d-token threshold; using whole-module documentation mode.",
module_label,
input_tokens,
threshold,
)
- if current_module_name is None:
- leaf_files = {
- components[leaf_node].relative_path
- for leaf_node in leaf_nodes
- if leaf_node in components
- }
- all_files = {c.relative_path for c in components.values()}
- if all_files and len(leaf_files) / len(all_files) < LOW_COVERAGE_RATIO:
- logger.warning(
- "Leaf-node entry points cover only %d of %d parsed files (%.0f%%). "
- "Whole-repository documentation will start from these entry points and "
- "rely on agent exploration to reach the rest of the codebase.",
- len(leaf_files),
- len(all_files),
- 100 * len(leaf_files) / len(all_files),
- )
+ return {}
+
+ if len(scope_files) <= 1:
+ logger.warning(
+ "Module %s has %d code tokens above threshold %d but only %d file; "
+ "keeping the indivisible file without another clustering call.",
+ module_label,
+ input_tokens,
+ threshold,
+ len(scope_files),
+ )
return {}
logger.info(
- "Requesting LLM module clustering for %s because %d tokens exceed the %d-token threshold.",
+ "Requesting LLM module clustering for %s because %d code tokens exceed the %d-token threshold.",
module_label,
input_tokens,
threshold,
)
batches = partition_leaf_nodes_by_structure(
- leaf_nodes, components, lambda ids: _cluster_batch_fits(ids, config)
+ candidates, components, lambda ids: _cluster_batch_fits(ids, config)
)
if len(batches) == 1:
@@ -344,12 +412,10 @@ def cluster_modules(
module_label,
completer,
)
- if not module_tree:
- return {}
else:
logger.info(
- "Partitioned %d leaf nodes for %s into %d structure-based batches for clustering.",
- len(leaf_nodes),
+ "Partitioned %d entry points for %s into %d structure-based batches for clustering.",
+ len(candidates),
module_label,
len(batches),
)
@@ -386,18 +452,17 @@ def cluster_modules(
)
_merge_module_trees(module_tree, partial)
- # check if the module tree is valid
+ module_tree = normalize_module_partition(module_tree, candidates, components)
if len(module_tree) <= 1:
- logger.info(
- "Skipping LLM clustering result for %s because it produced only "
- "%d module(s); using whole-module documentation mode.",
+ logger.warning(
+ "Clustering %s produced only %d usable module(s); using a structural file partition.",
module_label,
len(module_tree),
)
- return {}
+ module_tree = _structural_fallback(candidates, components, threshold)
logger.info(
- "LLM module clustering for %s produced %d top-level modules.",
+ "Module clustering for %s produced %d top-level modules.",
module_label,
len(module_tree),
)
@@ -413,31 +478,26 @@ def cluster_modules(
value[module_name] = module_info
for module_name, module_info in module_tree.items():
- sub_leaf_nodes = module_info.get("components", [])
-
- # Filter sub_leaf_nodes to ensure they exist in components
- valid_sub_leaf_nodes = []
- for node in sub_leaf_nodes:
- if node in components:
- valid_sub_leaf_nodes.append(node)
- else:
- logger.warning(
- f"Skipping invalid sub leaf node '{node}' in module '{module_name}' - not found in components"
- )
+ sub_files = component_files(module_info["components"], components)
+ # Reuse original leaves for coverage diagnostics, not the added file
+ # representatives. The next call reconstructs its complete candidate set.
+ sub_leaves = [cid for cid in leaves if components[cid].relative_path in sub_files]
current_module_path.append(module_name)
module_info["children"] = {}
module_info["children"] = cluster_modules(
- valid_sub_leaf_nodes,
+ sub_leaves,
components,
config,
current_module_tree,
module_name,
current_module_path,
completer=completer,
+ scope_files=sub_files,
)
current_module_path.pop()
+ validate_module_file_coverage(module_tree, components, scope_files)
return module_tree
diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py
index 10670c58..0be19e62 100644
--- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py
+++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py
@@ -20,6 +20,7 @@
from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer
from codewiki.src.be.dependency_analyzer.models.analysis import AnalysisResult
from codewiki.src.be.dependency_analyzer.models.core import Repository
+from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_source_file
from codewiki.src.be.dependency_analyzer.utils.security import assert_safe_path, safe_open_text
if TYPE_CHECKING: # pragma: no cover
@@ -316,6 +317,12 @@ def _analyze_call_graph(
logger.debug(f"Analyzing {len(supported_files)} supported files.")
result = self.call_graph_analyzer.analyze_code_files(supported_files, repo_dir)
+ # Keep readable source files even when their analyzer emits no symbol
+ # (constants, declarations, module-level scripts, or parse failures).
+ # Explicit analyzer exclusions must not be resurrected as file nodes.
+ result["source_files"] = [
+ info for info in supported_files if not is_skipped_source_file(info, repo_dir)
+ ]
result["call_graph"]["supported_languages"] = self._get_supported_languages()
result["call_graph"]["unsupported_files"] = len(code_files) - len(supported_files)
diff --git a/codewiki/src/be/dependency_analyzer/analyzers/go.py b/codewiki/src/be/dependency_analyzer/analyzers/go.py
index e09a1388..7a241812 100644
--- a/codewiki/src/be/dependency_analyzer/analyzers/go.py
+++ b/codewiki/src/be/dependency_analyzer/analyzers/go.py
@@ -51,6 +51,7 @@
from tree_sitter_language_pack import get_parser
from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
+from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_go_file
logger = logging.getLogger(__name__)
@@ -222,12 +223,6 @@ def split_go_package_ref(callee: str) -> tuple[str, str] | None:
return None
-def is_skipped_go_file(relative_path: str) -> bool:
- """`_test.go` files and vendored dependencies yield no components."""
- posix = Path(relative_path).as_posix()
- return posix.endswith("_test.go") or "vendor" in posix.split("/")[:-1]
-
-
def package_dir_of(relative_path: str) -> str:
return posixpath.dirname(Path(relative_path).as_posix())
diff --git a/codewiki/src/be/dependency_analyzer/analyzers/php.py b/codewiki/src/be/dependency_analyzer/analyzers/php.py
index 4536b418..97eda2fa 100644
--- a/codewiki/src/be/dependency_analyzer/analyzers/php.py
+++ b/codewiki/src/be/dependency_analyzer/analyzers/php.py
@@ -13,6 +13,11 @@
from tree_sitter import Parser, Language
import tree_sitter_php
from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
+from codewiki.src.be.dependency_analyzer.utils.file_selection import (
+ PHP_TEMPLATE_DIRECTORIES,
+ PHP_TEMPLATE_PATTERNS,
+ is_php_template_file,
+)
logger = logging.getLogger(__name__)
@@ -63,8 +68,8 @@
}
# Template file patterns to skip
-TEMPLATE_PATTERNS: Set[str] = {".blade.php", ".phtml", ".twig.php"}
-TEMPLATE_DIRECTORIES: Set[str] = {"views", "templates", "resources/views"}
+TEMPLATE_PATTERNS = PHP_TEMPLATE_PATTERNS
+TEMPLATE_DIRECTORIES = PHP_TEMPLATE_DIRECTORIES
# Maximum recursion depth to prevent stack overflow
MAX_RECURSION_DEPTH = 100
@@ -138,19 +143,7 @@ def __init__(self, file_path: str, content: str, repo_path: str = None):
def _is_template_file(self) -> bool:
"""Check if file is a PHP template that should be skipped."""
- file_str = str(self.file_path)
-
- # Check extension patterns
- for pattern in TEMPLATE_PATTERNS:
- if file_str.endswith(pattern):
- return True
-
- # Check directory patterns
- for dir_pattern in TEMPLATE_DIRECTORIES:
- if f"/{dir_pattern}/" in file_str or f"\\{dir_pattern}\\" in file_str:
- return True
-
- return False
+ return is_php_template_file(str(self.file_path))
def _get_module_path(self) -> str:
"""Get module path for the file."""
diff --git a/codewiki/src/be/dependency_analyzer/analyzers/zig.py b/codewiki/src/be/dependency_analyzer/analyzers/zig.py
index 2c2cba7a..cd5f1b7c 100644
--- a/codewiki/src/be/dependency_analyzer/analyzers/zig.py
+++ b/codewiki/src/be/dependency_analyzer/analyzers/zig.py
@@ -50,6 +50,7 @@
from tree_sitter_language_pack import get_parser
from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node
+from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_zig_file
logger = logging.getLogger(__name__)
@@ -61,7 +62,6 @@
"opaque": ("class", "opaque"),
}
-_SKIPPED_DIRS = {"zig-cache", ".zig-cache", "zig-out"}
_EXTERNAL_IMPORTS = {"std", "builtin", "root"}
# Constructors by convention: their result is a value of the owning type.
_CONSTRUCTOR_NAMES = {"init", "initCapacity", "create", "new", "open", "parse"}
@@ -84,10 +84,6 @@
_UNKNOWN = ("unknown",)
-def is_skipped_zig_file(relative_path: str) -> bool:
- return any(part in _SKIPPED_DIRS for part in Path(relative_path).parts[:-1])
-
-
@lru_cache(maxsize=64)
def _parse_build_zig(path: str, mtime_ns: int) -> tuple[tuple[str, str], ...]:
try:
diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py
index 9f5a6a1c..e5f67c40 100644
--- a/codewiki/src/be/dependency_analyzer/ast_parser.py
+++ b/codewiki/src/be/dependency_analyzer/ast_parser.py
@@ -1,10 +1,12 @@
import json
import logging
import os
+from pathlib import Path
from typing import TYPE_CHECKING
from codewiki.src.be.dependency_analyzer.analysis.analysis_service import AnalysisService
from codewiki.src.be.dependency_analyzer.models.core import Node
+from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text
if TYPE_CHECKING: # pragma: no cover
from codewiki.src.be.dependency_analyzer.analyzers.artifact import ArtifactOptions
@@ -69,10 +71,45 @@ def parse_repository(self, filtered_folders: list[str] | None = None) -> dict[st
self.artifact_index = call_graph_result.get("artifact_index")
self._build_components_from_analysis(call_graph_result)
+ self._retain_symbol_free_files(call_graph_result.get("source_files", []))
logger.debug(f"Found {len(self.components)} components across {len(self.modules)} modules")
return self.components
+ def _retain_symbol_free_files(self, source_files: list[dict]) -> None:
+ # An artifact head (e.g. setup.py) may be capped. It does not stand
+ # in for the full source of a supported code file without symbols.
+ covered = {
+ node.relative_path
+ for node in self.components.values()
+ if node.component_type != "artifact"
+ }
+ base = Path(self.repo_path)
+ for info in source_files:
+ relative = info["path"]
+ if relative in covered:
+ continue
+ try:
+ source = safe_open_text(base, base / relative)
+ except OSError as exc:
+ logger.warning("Cannot retain source file %s: %s", relative, exc)
+ continue
+ cid = f"{relative}::__file__"
+ self.components[cid] = Node(
+ id=cid,
+ name="__file__",
+ component_type="file",
+ file_path=str(base / relative),
+ relative_path=relative,
+ source_code=source,
+ start_line=1,
+ end_line=len(source.splitlines()),
+ language=info.get("language"),
+ )
+ self.modules.add(relative)
+ covered.add(relative)
+ logger.info("Retained source file without indexed symbols: %s", relative)
+
def _build_components_from_analysis(self, call_graph_result: dict):
functions = call_graph_result.get("functions", [])
relationships = call_graph_result.get("relationships", [])
diff --git a/codewiki/src/be/dependency_analyzer/utils/file_selection.py b/codewiki/src/be/dependency_analyzer/utils/file_selection.py
new file mode 100644
index 00000000..02fa5942
--- /dev/null
+++ b/codewiki/src/be/dependency_analyzer/utils/file_selection.py
@@ -0,0 +1,37 @@
+"""Explicit source-file exclusions, without importing optional parser grammars."""
+
+from pathlib import Path
+
+PHP_TEMPLATE_PATTERNS = {".blade.php", ".phtml", ".twig.php"}
+PHP_TEMPLATE_DIRECTORIES = {"views", "templates", "resources/views"}
+ZIG_SKIPPED_DIRS = {"zig-cache", ".zig-cache", "zig-out"}
+
+
+def is_skipped_go_file(relative_path: str) -> bool:
+ """`_test.go` files and vendored dependencies yield no components."""
+ posix = Path(relative_path).as_posix()
+ return posix.endswith("_test.go") or "vendor" in posix.split("/")[:-1]
+
+
+def is_skipped_zig_file(relative_path: str) -> bool:
+ return any(part in ZIG_SKIPPED_DIRS for part in Path(relative_path).parts[:-1])
+
+
+def is_php_template_file(file_path: str) -> bool:
+ file_str = str(file_path)
+ return any(file_str.endswith(pattern) for pattern in PHP_TEMPLATE_PATTERNS) or any(
+ f"/{directory}/" in file_str or f"\\{directory}\\" in file_str
+ for directory in PHP_TEMPLATE_DIRECTORIES
+ )
+
+
+def is_skipped_source_file(file_info: dict, base_dir: str) -> bool:
+ """Apply the same exclusions for parsing and symbol-free file retention."""
+ language = file_info.get("language")
+ if language == "go":
+ return is_skipped_go_file(file_info["path"])
+ if language == "zig":
+ return is_skipped_zig_file(file_info["path"])
+ if language == "php":
+ return is_php_template_file(str(Path(base_dir) / file_info["path"]))
+ return False
diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py
index eb5238f7..94f7e9e2 100644
--- a/codewiki/src/be/documentation_generator.py
+++ b/codewiki/src/be/documentation_generator.py
@@ -28,6 +28,11 @@
find_missing_module_docs,
resolve_module_doc_path,
)
+from codewiki.src.be.module_scope import (
+ complete_file_candidates,
+ get_module_token_count,
+ validate_module_file_coverage,
+)
from codewiki.src.be.prompt_template import (
MODULE_OVERVIEW_PROMPT,
REPO_OVERVIEW_ARTIFACT_ADDENDUM,
@@ -322,12 +327,13 @@ async def _generate_modules_in_order(
final_module_tree = await self.backend.run_module_agent(
module_name=repo_name,
components=components,
- core_component_ids=leaf_nodes,
+ core_component_ids=complete_file_candidates(leaf_nodes, components),
module_path=[],
working_dir=working_dir,
)
# save final_module_tree to module_tree.json
+ validate_module_file_coverage(final_module_tree, components)
file_manager.save_json(
final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME)
)
@@ -337,6 +343,10 @@ async def _generate_modules_in_order(
if os.path.exists(repo_overview_path):
os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME))
+ # Also check resumed trees whose agents returned early because their
+ # pages already existed. Do not silently bless an old partial tree.
+ validate_module_file_coverage(file_manager.load_json(module_tree_path), components)
+
# Move pages an agent saved in the wrong folder and fix links between pages
organize_docs(working_dir, config_layout(self.config))
@@ -485,11 +495,13 @@ async def run(self) -> None:
file_manager.save_json(module_tree, module_tree_path)
else:
logger.debug(f"Module tree not found at {module_tree_path}, clustering modules")
- clustering_tokens = get_clustering_input_token_count(leaf_nodes, components)
+ clustering_tokens = get_module_token_count(components)
logger.info(
- "Preparing %d leaf nodes for module clustering (%d tokens, threshold %d)",
+ "Preparing %d leaf nodes for module clustering "
+ "(code_tokens=%d, leaf_tokens=%d, threshold=%d)",
len(leaf_nodes),
clustering_tokens,
+ get_clustering_input_token_count(leaf_nodes, components),
self.config.max_token_per_module,
)
# Bind cluster_model into the completer so the backend uses the
@@ -517,6 +529,7 @@ async def run(self) -> None:
# Only freshly clustered trees are deduped: renaming a cached
# key whose .md already exists would orphan the doc.
module_tree = dedupe_module_tree_names(module_tree)
+ validate_module_file_coverage(module_tree, components)
file_manager.save_json(module_tree, first_module_tree_path)
file_manager.save_json(module_tree, module_tree_path)
diff --git a/codewiki/src/be/module_scope.py b/codewiki/src/be/module_scope.py
new file mode 100644
index 00000000..5f6df990
--- /dev/null
+++ b/codewiki/src/be/module_scope.py
@@ -0,0 +1,196 @@
+"""Language-independent module sizes, file coverage, and file ownership.
+
+Core components are entry points, not a size estimate. A module owns files;
+its size includes every indexed component in those files, including methods
+which leaf selection intentionally omits. Parent lists may aggregate children,
+but a file must have only one leaf owner.
+"""
+
+from __future__ import annotations
+
+import logging
+import posixpath
+from collections import Counter, defaultdict
+from typing import Any
+
+from codewiki.src.be.dependency_analyzer.models.core import Node
+from codewiki.src.be.utils import count_tokens
+
+logger = logging.getLogger(__name__)
+
+
+def component_files(ids: list[str], components: dict[str, Node]) -> set[str]:
+ return {components[cid].relative_path for cid in ids if cid in components}
+
+
+def file_token_counts(components: dict[str, Node]) -> dict[str, int]:
+ """Sum component source tokens, deliberately retaining overlapping source."""
+ totals: dict[str, int] = defaultdict(int)
+ for node in components.values():
+ totals[node.relative_path] += count_tokens(node.source_code or "")
+ return dict(totals)
+
+
+def get_module_token_count(components: dict[str, Node], scope_files: set[str] | None = None) -> int:
+ return sum(
+ count_tokens(node.source_code or "")
+ for node in components.values()
+ if scope_files is None or node.relative_path in scope_files
+ )
+
+
+def complete_file_candidates(
+ ids: list[str], components: dict[str, Node], scope_files: set[str] | None = None
+) -> list[str]:
+ """Keep entry points and add one stable representative per uncovered file."""
+ scope = (
+ {node.relative_path for node in components.values()} if scope_files is None else scope_files
+ )
+ selected = {cid for cid in ids if cid in components and components[cid].relative_path in scope}
+ covered = component_files(list(selected), components)
+ for cid in sorted(components):
+ node = components[cid]
+ path = node.relative_path
+ # A supported source file may also have capped artifact entry points.
+ # Retain its full-source representative for documentation as code.
+ if path in scope and (path not in covered or node.component_type == "file"):
+ selected.add(cid)
+ covered.add(path)
+ return sorted(selected, key=lambda cid: (components[cid].relative_path, cid))
+
+
+def normalize_module_partition(
+ proposed: dict[str, Any], candidates: list[str], components: dict[str, Node]
+) -> dict[str, Any]:
+ """Turn an untrusted flat proposal into a complete, disjoint file partition.
+
+ Votes use distinct known IDs in the current file scope. All candidates of
+ a file follow its winning group, including IDs omitted from the response.
+ Unassigned files prefer the nearest directory, then the smallest group.
+ An entirely unusable proposal returns {}, letting clustering use its
+ deterministic structural fallback instead.
+ """
+ by_file: dict[str, list[str]] = defaultdict(list)
+ for cid in candidates:
+ if cid in components:
+ by_file[components[cid].relative_path].append(cid)
+ votes: dict[str, Counter] = defaultdict(Counter)
+ for name, info in proposed.items():
+ if not isinstance(name, str) or not isinstance(info, dict):
+ continue
+ ids = info.get("components", [])
+ if not isinstance(ids, list):
+ continue
+ for cid in set(cid for cid in ids if isinstance(cid, str)):
+ if cid in components and components[cid].relative_path in by_file:
+ votes[components[cid].relative_path][name] += 1
+ if not votes:
+ return {}
+
+ missing_count = len(by_file.keys() - votes.keys())
+ conflicting_count = sum(len(groups) > 1 for groups in votes.values())
+ if missing_count or conflicting_count:
+ logger.warning(
+ "Reconciling module file assignments: %d omitted files, %d files assigned to multiple "
+ "groups; restoring complete file ownership.",
+ missing_count,
+ conflicting_count,
+ )
+
+ assigned: dict[str, list[str]] = defaultdict(list)
+ for path in sorted(votes):
+ winner = min(votes[path], key=lambda name: (-votes[path][name], name))
+ assigned[winner].append(path)
+ sizes = file_token_counts(components)
+ loads = {name: sum(sizes[path] for path in paths) for name, paths in assigned.items()}
+
+ def proximity(left: str, right: str) -> int:
+ left_parts = left.split("/")[:-1]
+ right_parts = right.split("/")[:-1]
+ depth = 0
+ for a, b in zip(left_parts, right_parts):
+ if a != b:
+ break
+ depth += 1
+ return depth
+
+ for path in sorted(by_file.keys() - votes.keys()):
+ winner = min(
+ assigned,
+ key=lambda name: (
+ -max(proximity(path, other) for other in assigned[name]),
+ loads[name],
+ name,
+ ),
+ )
+ assigned[winner].append(path)
+ loads[winner] += sizes[path]
+
+ return {
+ name: {
+ "path": posixpath.commonpath(paths),
+ "components": [cid for path in sorted(paths) for cid in sorted(set(by_file[path]))],
+ "children": {},
+ }
+ for name, paths in assigned.items()
+ }
+
+
+def module_file_owners(tree: dict[str, Any], components: dict[str, Node]) -> dict[str, list[str]]:
+ owners: dict[str, list[str]] = defaultdict(list)
+
+ def walk(level: dict[str, Any], parent: tuple[str, ...]) -> None:
+ for name, info in level.items():
+ path = (*parent, name)
+ children = info.get("children") or {}
+ if children:
+ walk(children, path)
+ else:
+ for file in component_files(info.get("components", []), components):
+ owners[file].append("/".join(path))
+
+ walk(tree, ())
+ return dict(owners)
+
+
+def validate_module_file_coverage(
+ tree: dict[str, Any],
+ components: dict[str, Node],
+ scope_files: set[str] | None = None,
+) -> None:
+ """Fail before saving a partial/overlapping tree; {} is intentional one-page mode."""
+ if not tree:
+ return
+ expected = (
+ scope_files if scope_files is not None else component_files(list(components), components)
+ )
+ owners = module_file_owners(tree, components)
+ missing = sorted(expected - owners.keys())
+ duplicates = {path: modules for path, modules in owners.items() if len(modules) > 1}
+ outside = sorted(owners.keys() - expected)
+ if missing or duplicates or outside:
+ raise ValueError(
+ f"Invalid module file coverage: missing={missing}, duplicate_owners={duplicates}, "
+ f"outside_scope={outside}"
+ )
+ logger.info(
+ "Module file coverage: %d/%d files owned, 0 missing, 0 duplicate owners",
+ len(owners),
+ len(expected),
+ )
+
+
+def prepare_submodule_partition(
+ specs: dict[str, list[str]],
+ parent_ids: list[str],
+ components: dict[str, Node],
+ existing_children: dict[str, Any],
+) -> dict[str, list[str]]:
+ """Partition the remaining parent files without stealing an existing child's files."""
+ locked = set(module_file_owners(existing_children, components))
+ scope = component_files(parent_ids, components) - locked
+ candidates = complete_file_candidates(parent_ids, components, scope)
+ normalized = normalize_module_partition(
+ {name: {"components": ids} for name, ids in specs.items()}, candidates, components
+ )
+ return {name: info["components"] for name, info in normalized.items()}
diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py
index fd274261..775e98d1 100644
--- a/codewiki/src/be/prompt_template.py
+++ b/codewiki/src/be/prompt_template.py
@@ -150,12 +150,12 @@
MODULE_OVERVIEW_PROMPT += "\n\n" + _MERMAID_TEMPLATE_GUIDANCE
CLUSTER_REPO_PROMPT = """
-Here is list of all potential core components of the repository (It's normal that some components are not essential to the repository):
+Here are the core entry points of the repository, grouped by source file:
{potential_core_components}
-Please group the components into groups such that each group is a set of components that are closely related to each other and together they form a module. DO NOT include components that are not essential to the repository.
+Group related source files into modules. Assign EVERY provided component ID exactly once. All IDs from the same file must stay together in one module. Each file represents its entire implementation, including methods and functions absent from this entry-point list. A `__file__` entry represents a source file without indexed symbols; include it too.
Files marked `(artifact: )` are build, CI, container, packaging, manifest, configuration, schema or script files. Their components describe how the system is built, packaged, shipped, configured and tested. They ARE essential: group them into a dedicated build/deployment/configuration module, or attach them to the module they configure. Never drop them.
@@ -192,12 +192,12 @@
{module_tree}
-Here is list of all potential core components of the module {module_name} (It's normal that some components are not essential to the module):
+Here are the core entry points of the module {module_name}, grouped by source file:
{potential_core_components}
-Please group the components into groups such that each group is a set of components that are closely related to each other and together they form a smaller module. DO NOT include components that are not essential to the module.
+Group related source files into smaller modules. Assign EVERY provided component ID exactly once. All IDs from the same file must stay together in one module. Each file represents its entire implementation, including methods and functions absent from this entry-point list. A `__file__` entry represents a source file without indexed symbols; include it too.
Files marked `(artifact: )` are build, CI, container, packaging, manifest, configuration, schema or script files. Their components describe how the system is built, packaged, shipped, configured and tested. They ARE essential: group them into a dedicated build/deployment/configuration module, or attach them to the module they configure. Never drop them.
diff --git a/codewiki/src/be/pydantic_ai_backend.py b/codewiki/src/be/pydantic_ai_backend.py
index 704b7ee2..3304c0b6 100644
--- a/codewiki/src/be/pydantic_ai_backend.py
+++ b/codewiki/src/be/pydantic_ai_backend.py
@@ -27,6 +27,7 @@
from codewiki.src.be.backend import AgentReply, LLMBackend, UsageLedger, usage_to_dict
from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.llm_services import call_llm, create_fallback_models, pop_last_usage
+from codewiki.src.be.module_scope import complete_file_candidates, validate_module_file_coverage
from codewiki.src.be.prompt_template import (
format_leaf_system_prompt,
format_system_prompt,
@@ -144,7 +145,9 @@ async def run_module_agent(
logger.info("✓ Module docs already exists at %s", docs_path)
return module_tree
- if is_complex_module(components, core_component_ids):
+ if not module_path:
+ core_component_ids = complete_file_candidates(core_component_ids, components)
+ if is_complex_module(components, core_component_ids) and config.max_depth > 1:
agent = Agent(
self._fallback_models,
name=module_name,
@@ -183,6 +186,7 @@ async def run_module_agent(
current_depth=1,
config=config,
custom_instructions=self._custom_instructions,
+ scope_component_ids=core_component_ids,
)
try:
@@ -202,6 +206,7 @@ async def run_module_agent(
)
finally:
self.last_usage = tracked.totals()
+ validate_module_file_coverage(deps.module_tree, components)
file_manager.save_json(deps.module_tree, module_tree_path)
return deps.module_tree
except Exception as e:
diff --git a/codewiki/src/be/updater/orchestrator.py b/codewiki/src/be/updater/orchestrator.py
index a895107d..4e149d97 100644
--- a/codewiki/src/be/updater/orchestrator.py
+++ b/codewiki/src/be/updater/orchestrator.py
@@ -10,6 +10,7 @@
from codewiki.src.be.backend import LLMBackend
from codewiki.src.be.cluster_modules import cluster_modules, get_clustering_input_token_count
+from codewiki.src.be.module_scope import get_module_token_count
from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.updater import pages as P
from codewiki.src.be.updater import tree as T
@@ -334,9 +335,11 @@ async def _run(
# --include and then updated against the full repo does not, and
# feeding that to the single-page agent runs unbounded (issue #113).
scope = [c for c in leaf_nodes if c in new_graph]
- tokens = get_clustering_input_token_count(scope, new_graph)
+ tokens = get_module_token_count(new_graph)
+ leaf_tokens = get_clustering_input_token_count(scope, new_graph)
threshold = self.config.max_token_per_module
ratios["clustering_tokens"] = tokens
+ ratios["leaf_tokens"] = leaf_tokens
ratios["tau_cluster"] = threshold
if tokens <= threshold:
ratios["fired"] = False
diff --git a/tests/test_agent_limits.py b/tests/test_agent_limits.py
index eaa1eb63..21c004df 100644
--- a/tests/test_agent_limits.py
+++ b/tests/test_agent_limits.py
@@ -11,6 +11,7 @@
import codewiki.src.be.llm_services as llm_services
from codewiki.cli.models.config import Configuration
from codewiki.src.be.agent_tools.deps import CodeWikiDeps
+from codewiki.src.be.dependency_analyzer.models.core import Node
from codewiki.src.be.prompt_template import format_overview_system_prompt
from codewiki.src.config import DEFAULT_AGENT_RETRIES, DEFAULT_REQUEST_LIMIT, Config
@@ -95,11 +96,21 @@ async def run(self, prompt, deps, **kwargs):
monkeypatch.setattr(sub_mod, "Agent", FakeAgent)
monkeypatch.setattr(sub_mod, "create_fallback_models", lambda config: None)
+ source = tmp_path / "b.py"
+ source.write_text("def fb(): pass\n")
+ component = Node(
+ id="b.py::fb",
+ name="fb",
+ component_type="function",
+ file_path=str(source),
+ relative_path="b.py",
+ source_code=source.read_text(),
+ )
deps = CodeWikiDeps(
absolute_docs_path=str(tmp_path),
absolute_repo_path=str(tmp_path),
registry={},
- components={},
+ components={component.id: component},
path_to_current_module=[],
current_module_name="root",
module_tree={},
diff --git a/tests/test_cluster_partitioning.py b/tests/test_cluster_partitioning.py
index 06141b8a..cdd435d9 100644
--- a/tests/test_cluster_partitioning.py
+++ b/tests/test_cluster_partitioning.py
@@ -63,6 +63,7 @@ def _max_count_fits(limit: int):
# partition_leaf_nodes_by_structure
# ---------------------------------------------------------------------------
+
def test_partition_single_batch_when_everything_fits():
ids = [f"lib/a.py::C{i}" for i in range(5)]
components = _components(ids)
@@ -80,9 +81,7 @@ def test_partition_drops_unknown_ids():
def test_partition_splits_by_top_level_directory():
- ids = [f"alpha/f{i}.py::A{i}" for i in range(4)] + [
- f"beta/f{i}.py::B{i}" for i in range(4)
- ]
+ ids = [f"alpha/f{i}.py::A{i}" for i in range(4)] + [f"beta/f{i}.py::B{i}" for i in range(4)]
components = _components(ids)
batches = partition_leaf_nodes_by_structure(ids, components, _max_count_fits(4))
assert len(batches) == 2
@@ -111,15 +110,10 @@ def test_partition_chunks_unsplittable_single_file():
def test_partition_coalesces_root_files_and_is_deterministic():
- ids = (
- ["setup.py::Setup", "conftest.py::Conf"]
- + [f"src/f{i}.py::S{i}" for i in range(3)]
- )
+ ids = ["setup.py::Setup", "conftest.py::Conf"] + [f"src/f{i}.py::S{i}" for i in range(3)]
components = _components(ids)
first = partition_leaf_nodes_by_structure(ids, components, _max_count_fits(4))
- second = partition_leaf_nodes_by_structure(
- list(reversed(ids)), components, _max_count_fits(4)
- )
+ second = partition_leaf_nodes_by_structure(list(reversed(ids)), components, _max_count_fits(4))
assert first == second
assert all(len(batch) <= 4 for batch in first)
# Root-level files must not sit alone in single-node batches.
@@ -142,14 +136,14 @@ def test_cluster_batch_fits_respects_node_count_and_token_budget():
# None-robustness
# ---------------------------------------------------------------------------
-def test_cluster_modules_none_response_returns_empty_tree():
+
+def test_cluster_modules_none_response_keeps_file_partition():
ids = [f"lib/f{i}.py::C{i}" for i in range(3)]
components = _components(ids)
config = _config(max_token_per_module=1)
- result = cluster_modules(
- ids, components, config, {}, None, [], completer=lambda prompt: None
- )
- assert result == {}
+ result = cluster_modules(ids, components, config, {}, None, [], completer=lambda prompt: None)
+ assert len(result) == 3
+ assert sorted(cid for info in result.values() for cid in info["components"]) == ids
def test_parse_super_group_response_handles_none_and_garbage():
@@ -159,27 +153,18 @@ def test_parse_super_group_response_handles_none_and_garbage():
def test_super_group_modules_none_response_keeps_tree():
- tree = {
- f"Module{i}": {"path": f"m{i}", "components": [f"m{i}/f.py::C"]}
- for i in range(5)
- }
+ tree = {f"Module{i}": {"path": f"m{i}", "components": [f"m{i}/f.py::C"]} for i in range(5)}
config = _config()
assert super_group_modules(tree, config, completer=lambda prompt: None) == tree
def test_extract_content_returns_none_on_truncated_empty_response():
response = SimpleNamespace(
- choices=[
- SimpleNamespace(
- message=SimpleNamespace(content=None), finish_reason="length"
- )
- ]
+ choices=[SimpleNamespace(message=SimpleNamespace(content=None), finish_reason="length")]
)
assert _extract_content(response, "test-model") is None
ok = SimpleNamespace(
- choices=[
- SimpleNamespace(message=SimpleNamespace(content="hi"), finish_reason="stop")
- ]
+ choices=[SimpleNamespace(message=SimpleNamespace(content="hi"), finish_reason="stop")]
)
assert _extract_content(ok, "test-model") == "hi"
@@ -188,6 +173,7 @@ def test_extract_content_returns_none_on_truncated_empty_response():
# Batched clustering end-to-end (scripted completer)
# ---------------------------------------------------------------------------
+
def _grouped(tree: dict) -> str:
return f"{tree!r}"
@@ -204,20 +190,22 @@ def completer(prompt: str) -> str:
in_alpha = alpha[0] in prompt
in_beta = beta[0] in prompt
if in_alpha and not in_beta:
- return _grouped({
- "Alpha": {"path": "alpha", "components": alpha[:2]},
- "Shared": {"path": "alpha", "components": [alpha[2]]},
- })
+ return _grouped(
+ {
+ "Alpha": {"path": "alpha", "components": alpha[:2]},
+ "Shared": {"path": "alpha", "components": [alpha[2]]},
+ }
+ )
if in_beta and not in_alpha:
- return _grouped({
- "Beta": {"path": "beta", "components": beta[:2]},
- "Shared": {"path": "beta", "components": [beta[2]]},
- })
+ return _grouped(
+ {
+ "Beta": {"path": "beta", "components": beta[:2]},
+ "Shared": {"path": "beta", "components": [beta[2]]},
+ }
+ )
return None
- tree = cluster_modules(
- alpha + beta, components, config, {}, None, [], completer=completer
- )
+ tree = cluster_modules(alpha + beta, components, config, {}, None, [], completer=completer)
assert set(tree) == {"Alpha", "Shared", "Beta"}
# Name collision across batches merges components and generalizes the path.
assert tree["Shared"]["components"] == [alpha[2], beta[2]]
@@ -237,15 +225,15 @@ def completer(prompt: str) -> str:
if beta[0] in prompt and alpha[0] not in prompt:
return None # this batch's call "truncates"
if alpha[0] in prompt and beta[0] not in prompt:
- return _grouped({
- "Alpha": {"path": "alpha", "components": alpha[:2]},
- "AlphaExtra": {"path": "alpha", "components": [alpha[2]]},
- })
+ return _grouped(
+ {
+ "Alpha": {"path": "alpha", "components": alpha[:2]},
+ "AlphaExtra": {"path": "alpha", "components": [alpha[2]]},
+ }
+ )
return None
- tree = cluster_modules(
- alpha + beta, components, config, {}, None, [], completer=completer
- )
+ tree = cluster_modules(alpha + beta, components, config, {}, None, [], completer=completer)
assert "Alpha" in tree and "AlphaExtra" in tree
fallback = tree["beta"]
assert fallback["components"] == beta
diff --git a/tests/test_module_scope.py b/tests/test_module_scope.py
new file mode 100644
index 00000000..4b648c46
--- /dev/null
+++ b/tests/test_module_scope.py
@@ -0,0 +1,460 @@
+"""Regression tests for sizes and complete, disjoint source-file ownership."""
+
+from __future__ import annotations
+
+import asyncio
+import builtins
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from codewiki.src.be.cluster_modules import cluster_modules, get_clustering_input_token_count
+from codewiki.src.be.dependency_analyzer.dependency_graphs_builder import DependencyGraphBuilder
+from codewiki.src.be.dependency_analyzer.models.core import Node
+from codewiki.src.be.module_scope import (
+ complete_file_candidates,
+ component_files,
+ get_module_token_count,
+ module_file_owners,
+ normalize_module_partition,
+ prepare_submodule_partition,
+ validate_module_file_coverage,
+)
+from codewiki.src.be.utils import count_tokens
+from codewiki.src.config import Config
+
+
+def _node(cid: str, source: str = "code", kind: str = "method") -> Node:
+ path, name = cid.split("::")
+ return Node(
+ id=cid,
+ name=name,
+ component_type=kind,
+ file_path=path,
+ relative_path=path,
+ source_code=source,
+ )
+
+
+def _config(repo: Path, out: Path, threshold: int = 36369) -> Config:
+ return Config.from_cli(
+ repo_path=str(repo),
+ output_dir=str(out),
+ llm_base_url="http://localhost",
+ llm_api_key="test",
+ main_model="test",
+ cluster_model="test",
+ max_token_per_module=threshold,
+ artifacts_enabled=False,
+ )
+
+
+def test_go_method_heavy_package_splits_with_sparse_leaves(tmp_path, caplog):
+ caplog.set_level("INFO")
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ (repo / "go.mod").write_text("module example.org/engine\n\ngo 1.22\n")
+ for batch in range(3):
+ types = ["package engine\n"]
+ methods = ["package engine\n"]
+ for i in range(batch * 135, (batch + 1) * 135):
+ field = f"previous *S{i - 1}" if i else "value int"
+ types.append(f"type S{i} struct {{ {field} }}\n")
+ for name in ("Step", "Reset"):
+ methods.append(
+ f"func (s *S{i}) {name}() int {{ value := {i}; "
+ "for n := 0; n < 8; n++ { value += n }; return value }\n"
+ )
+ (repo / f"types_{batch}.go").write_text("".join(types))
+ (repo / f"methods_{batch}.go").write_text("".join(methods))
+ (repo / "entry.go").write_text("package engine\ntype Entry struct {}\n")
+ (repo / "constants.go").write_text("package engine\nconst Capacity = 4096\n")
+ (repo / "ignored_test.go").write_text("package engine\ntype TestOnly struct {}\n")
+ config = _config(repo, tmp_path / "out", 4000)
+ components, leaves = DependencyGraphBuilder(config).build_dependency_graph()
+ files = component_files(list(components), components)
+ assert len(leaves) < 10
+ assert len(component_files(leaves, components)) / len(files) < 0.5
+ assert get_clustering_input_token_count(leaves, components) < config.max_token_per_module
+ assert get_module_token_count(components) > config.max_token_per_module
+ assert components["constants.go::__file__"].component_type == "file"
+ assert "ignored_test.go" not in files
+ candidates = complete_file_candidates(leaves, components)
+ calls = []
+
+ def completer(prompt):
+ calls.append(prompt)
+ # Include omissions, a duplicate file assignment, an invalid ID, and
+ # a method which was not a leaf. Reconciliation must retain all files.
+ first = candidates[0]
+ other = next(cid for cid in candidates if components[cid].component_type == "method")
+ return f'{{"Core": {{"components": {[first, "bad::id"]!r}}}, "Execution": {{"components": {[first, other]!r}}}}}'
+
+ tree = cluster_modules(leaves, components, config, completer=completer)
+ assert calls
+ assert len(tree) >= 2
+ owners = module_file_owners(tree, components)
+ expected = {p.name for p in repo.glob("*.go") if not p.name.endswith("_test.go")}
+ assert owners.keys() == expected
+ assert all(len(owner) == 1 for owner in owners.values())
+ validate_module_file_coverage(tree, components)
+ assert f"code_tokens={get_module_token_count(components)}" in caplog.text
+ assert f"leaf_tokens={get_clustering_input_token_count(leaves, components)}" in caplog.text
+ assert "threshold=4000" in caplog.text
+ assert f"candidate_files={len(files)}/{len(files)}" in caplog.text
+
+
+def test_small_repository_keeps_one_page_with_all_file_entry_points(tmp_path):
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ (repo / "types.go").write_text("package tiny\ntype Buffer struct { value int }\n")
+ (repo / "methods.go").write_text(
+ "package tiny\nfunc (b *Buffer) Read() int { return b.value }\n"
+ )
+ (repo / "constants.go").write_text("package tiny\nconst Capacity = 16\n")
+ config = _config(repo, tmp_path / "out")
+ components, leaves = DependencyGraphBuilder(config).build_dependency_graph()
+ assert get_module_token_count(components) < config.max_token_per_module
+ assert len(component_files(leaves, components)) == 1
+ calls = []
+ assert cluster_modules(leaves, components, config, completer=lambda p: calls.append(p)) == {}
+ assert calls == []
+ assert component_files(complete_file_candidates(leaves, components), components) == {
+ "types.go",
+ "methods.go",
+ "constants.go",
+ }
+
+
+def test_size_includes_non_leaf_components_in_same_and_other_files():
+ extension = "go"
+ leaf = _node(f"types.{extension}::T", "type T {}", "struct")
+ method = _node(f"types.{extension}::T.work", "work();\n" * 80)
+ other = _node(f"methods.{extension}::T.read", "read();\n" * 100)
+ components = {node.id: node for node in (leaf, method, other)}
+ assert get_module_token_count(components) == sum(
+ count_tokens(n.source_code) for n in components.values()
+ )
+ assert get_module_token_count(components, {leaf.relative_path}) == count_tokens(
+ leaf.source_code
+ ) + count_tokens(method.source_code)
+
+
+@pytest.mark.parametrize(
+ ("extension", "source"),
+ [
+ ("py", "CAPACITY = 16\n"),
+ ("js", "export const capacity = 16;\n"),
+ ("ts", "export const capacity: number = 16;\n"),
+ ("java", "package engine;\n"),
+ ("c", "#define CAPACITY 16\n"),
+ ("cpp", "#pragma once\n#define CAPACITY 16\n"),
+ ("cs", "using System;\n"),
+ ("go", "package engine\nconst Capacity = 16\n"),
+ ("rs", "const CAPACITY: usize = 16;\n"),
+ ("rb", "CAPACITY = 16\n"),
+ ("dart", "const capacity = 16;\n"),
+ ("scala", "package engine\nval capacity = 16\n"),
+ ("kt", "package engine\nconst val CAPACITY = 16\n"),
+ ("php", "Engine
\n"),
+ ],
+)
+def test_files_without_architectural_symbols_survive_each_language_parser(
+ tmp_path, extension, source
+):
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ path = f"source.{extension}"
+ (repo / path).write_text(source)
+ components, leaves = DependencyGraphBuilder(
+ _config(repo, tmp_path / "out")
+ ).build_dependency_graph()
+ assert component_files(list(components), components) == {path}
+ assert component_files(complete_file_candidates(leaves, components), components) == {path}
+ assert get_module_token_count(components) > 0
+
+
+def test_capped_artifact_does_not_hide_symbol_free_source_code(tmp_path):
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ source = "CAPACITY = " + repr("x" * 30000) + "\n"
+ (repo / "setup.py").write_text(source)
+ config = _config(repo, tmp_path / "out")
+ config.artifacts_enabled = True
+ components, leaves = DependencyGraphBuilder(config).build_dependency_graph()
+ assert components["setup.py::__file__"].source_code == source
+ assert len(components["setup.py::setup.py"].source_code) < len(source)
+ assert "setup.py::__file__" in complete_file_candidates(leaves, components)
+ assert get_module_token_count(components) >= count_tokens(source)
+
+
+@pytest.mark.parametrize(
+ ("language", "included", "excluded"),
+ [
+ ("go", "source.go", "vendor/source.go"),
+ ("zig", "source.zig", "zig-out/source.zig"),
+ ("php", "source.php", "templates/source.php"),
+ ],
+)
+def test_file_exclusions_do_not_import_parsers(monkeypatch, language, included, excluded):
+ from codewiki.src.be.dependency_analyzer.utils.file_selection import is_skipped_source_file
+
+ original_import = builtins.__import__
+
+ def without_grammars(name, *args, **kwargs):
+ if name.startswith(("tree_sitter", "codewiki.src.be.dependency_analyzer.analyzers")):
+ raise ModuleNotFoundError(name)
+ return original_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", without_grammars)
+ assert not is_skipped_source_file({"language": language, "path": included}, "/repo")
+ assert is_skipped_source_file({"language": language, "path": excluded}, "/repo")
+
+
+def test_symbol_free_file_survives_missing_php_grammar(tmp_path, monkeypatch):
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ source = "")[1].split(
+ ""
+ )[0]
+ batch = [line.strip() for line in section.splitlines() if line.startswith("\t")]
+ calls.append(batch)
+ return f'{{"Group{len(calls)}": {{"components": {batch!r}}}}}'
+
+ tree = cluster_modules(ids, components, config, completer=completer)
+ assert len(calls) > 1
+ assert all(len(batch) <= 3 for batch in calls)
+ assert len(tree) == 2
+ validate_module_file_coverage(tree, components)
+ assert all(len(owners) == 1 for owners in module_file_owners(tree, components).values())
+
+
+def test_partition_restores_omissions_and_moves_whole_files():
+ ids = ["src/a.py::A", "src/a.py::A.m", "src/b.py::B", "other/c.py::C"]
+ components = {cid: _node(cid) for cid in ids}
+ proposal = {
+ "Zulu": {"components": [ids[0], ids[0], "unknown::bad"]},
+ "Alpha": {"components": [ids[1]]},
+ "Other": {"components": [ids[3]]},
+ "Broken": None,
+ }
+ tree = normalize_module_partition(proposal, ids, components)
+ assert tree["Alpha"]["components"] == ids[:3]
+ assert "Zulu" not in tree
+ validate_module_file_coverage(tree, components)
+ with pytest.raises(ValueError, match="missing="):
+ validate_module_file_coverage({"A": {"components": [ids[0]]}}, components)
+ with pytest.raises(ValueError, match="duplicate_owners"):
+ validate_module_file_coverage(
+ {"A": {"components": ids}, "B": {"components": [ids[1]]}}, components
+ )
+
+
+def test_delegation_covers_parent_without_stealing_siblings():
+ ids = ["a.go::A", "a.go::A.m", "b.go::B.m", "sibling.go::S"]
+ components = {cid: _node(cid) for cid in ids}
+ specs = {"First": [ids[0]], "Second": [ids[1], ids[3]]}
+ normalized = prepare_submodule_partition(specs, ids[:3], components, {})
+ assert sorted(cid for group in normalized.values() for cid in group) == sorted(ids[:3])
+ assert len(normalized) == 1
+ assert (
+ prepare_submodule_partition(
+ specs, ids[:3], components, {"Existing": {"components": ids[:3]}}
+ )
+ == {}
+ )
+
+
+@pytest.mark.parametrize("backend_kind", ["pydantic", "caw"])
+@pytest.mark.parametrize("max_depth", [1, 2])
+def test_small_whole_repo_agent_has_all_files_and_optional_delegate(
+ tmp_path, monkeypatch, backend_kind, max_depth
+):
+ from codewiki.src.be.backend import UsageLedger
+
+ repo = tmp_path / "repo"
+ out = tmp_path / "out"
+ repo.mkdir()
+ out.mkdir()
+ components = {}
+ for path, name in (("types.go", "Buffer"), ("methods.go", "Buffer.Read")):
+ file = repo / path
+ file.write_text("package tiny\n")
+ node = _node(f"{path}::{name}", "small")
+ node.file_path = str(file)
+ components[node.id] = node
+ (out / "module_tree.json").write_text("{}")
+ config = _config(repo, out)
+ config.max_depth = max_depth
+ captured = {}
+
+ if backend_kind == "pydantic":
+ from codewiki.src.be import pydantic_ai_backend as mod
+
+ class FakeAgent:
+ def __init__(self, *args, **kwargs):
+ captured["can_delegate"] = any(
+ tool.name == "generate_sub_module_documentation" for tool in kwargs["tools"]
+ )
+
+ async def run(self, prompt, deps, **kwargs):
+ captured["ids"] = deps.module_component_ids()
+ return SimpleNamespace(output="ok")
+
+ monkeypatch.setattr(mod, "Agent", FakeAgent)
+ backend = object.__new__(mod.PydanticAIBackend)
+ backend._fallback_models = None
+ backend._custom_instructions = ""
+ backend._usage_ledger = UsageLedger()
+ else:
+ pytest.importorskip("caw")
+ from codewiki.src.be import caw_backend as mod
+
+ class FakeCawAgent:
+ def __init__(self, **kwargs):
+ toolkit = kwargs["tool_servers"][0]
+ captured["can_delegate"] = toolkit._allow_subagent
+ captured["ids"] = toolkit._deps.module_component_ids()
+
+ def completion(self, prompt):
+ return SimpleNamespace(
+ total_usage=None, model="test", num_turns=1, total_tool_calls=0
+ )
+
+ monkeypatch.setattr(mod, "CawAgent", FakeCawAgent)
+ backend = object.__new__(mod.CawBackend)
+ backend._caw_provider = "codex"
+ backend._model = "test"
+ backend._repo_root = str(repo)
+
+ backend._config = config
+ asyncio.run(backend.run_module_agent("tiny", components, ["types.go::Buffer"], [], str(out)))
+ assert captured["can_delegate"] == (max_depth > 1)
+ assert component_files(captured["ids"], components) == {"types.go", "methods.go"}
+ assert json.loads((out / "module_tree.json").read_text()) == {}
+
+
+@pytest.mark.parametrize("backend_kind", ["pydantic", "caw"])
+def test_delegate_normalizes_files_before_running_subagents(tmp_path, monkeypatch, backend_kind):
+ from codewiki.src.be.agent_tools.deps import CodeWikiDeps
+ from codewiki.src.be.agent_tools import generate_sub_module_documentations as tool
+
+ ids = ["a.go::A", "a.go::A.work", "b.go::B.work", "other.go::Other"]
+ components = {cid: _node(cid) for cid in ids}
+ config = _config(tmp_path, tmp_path / "out")
+ tree = {"Parent": {"components": ids[:3], "children": {}}, "Sibling": {"components": ids[3:]}}
+ deps = CodeWikiDeps(
+ absolute_docs_path=str(tmp_path),
+ absolute_repo_path=str(tmp_path),
+ registry={},
+ components=components,
+ path_to_current_module=["Parent"],
+ current_module_name="Parent",
+ module_tree=tree,
+ max_depth=2,
+ current_depth=1,
+ config=config,
+ scope_component_ids=ids[:3],
+ custom_instructions="",
+ )
+ seen = []
+ specs = {"One": [ids[0]], "Two": [ids[1], ids[3]]}
+ if backend_kind == "pydantic":
+
+ class FakeAgent:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ async def run(self, prompt, deps, **kwargs):
+ seen.extend(deps.module_component_ids())
+
+ monkeypatch.setattr(tool, "Agent", FakeAgent)
+ monkeypatch.setattr(tool, "create_fallback_models", lambda config: None)
+ asyncio.run(tool.generate_sub_module_documentation(SimpleNamespace(deps=deps), specs))
+ else:
+ pytest.importorskip("caw")
+ from codewiki.src.be.caw_toolkit import CawToolKit
+
+ class Backend:
+ def _run_module_agent_sync(self, **kwargs):
+ seen.extend(kwargs["core_component_ids"])
+
+ CawToolKit(deps, Backend(), True)._run_sub_modules(specs)
+ assert sorted(seen) == sorted(ids[:3])
+ assert deps.scope_component_ids == ids[:3]
+ assert deps.current_module_name == "Parent" and deps.current_depth == 1
+ validate_module_file_coverage(tree, components)
diff --git a/tests/test_sub_module_dedupe.py b/tests/test_sub_module_dedupe.py
index 0c0e0728..4e1eb7c6 100644
--- a/tests/test_sub_module_dedupe.py
+++ b/tests/test_sub_module_dedupe.py
@@ -78,12 +78,8 @@ def test_repeat_request_is_skipped_not_suffixed(tmp_path):
assert (tmp_path / "approval.md").exists() and (tmp_path / "billing.md").exists()
assert set(deps.module_tree) == {"approval", "billing"}
- # Round 2: same names again -> parent prefix, once (issue #76 behaviour).
- _call(deps, specs)
- assert (tmp_path / "overview_approval.md").exists()
- assert (tmp_path / "overview_billing.md").exists()
-
- # Round 3: plain and prefixed both exist -> skipped, nothing new anywhere.
+ # Repeating under parent-prefixed names must not assign the same files
+ # to a second leaf or run additional agents.
files_before = sorted(p.name for p in tmp_path.iterdir())
tree_before = dict(deps.module_tree)
runs_before = len(FakeAgent.runs)
diff --git a/tests/test_updater_orchestrator.py b/tests/test_updater_orchestrator.py
index a0754891..b4db3fb8 100644
--- a/tests/test_updater_orchestrator.py
+++ b/tests/test_updater_orchestrator.py
@@ -294,6 +294,22 @@ def test_whole_repo_baseline_falls_back_when_scope_needs_clustering(tmp_path):
assert (docs / "overview.md").exists()
+def test_whole_repo_growth_counts_methods_outside_leaf_entry_points(tmp_path):
+ docs, config, prev = _setup(tmp_path, tree={})
+ backend = FakeBackend()
+ graph = graph_r2()
+ method_id = "src/engine.go::Engine.execute"
+ graph[method_id] = node(method_id, "method", "execute();\n" * 20000)
+ updater = IncrementalUpdater(config, backend, _generator(config, backend), UpdateOptions())
+ record = asyncio.run(
+ updater.run(prev, graph, sorted(tracked_r2()), {"old_commit": "old", "new_commit": "new"})
+ )
+ assert record.fallback["leaf_tokens"] < config.max_token_per_module
+ assert record.fallback["clustering_tokens"] > config.max_token_per_module
+ assert record.outcome == "full_fallback"
+ assert backend.module_calls == [] and backend.update_calls == []
+
+
# ------------------------------------------------------------ ownership closure
HELPER = "src/misc/util.py::helper" # fresh dir, no tracked neighbour: rules 1-3 fail