Enforce minimum module sizes and reject non-splits

.gitignore

@@ -49,6 +49,7 @@ tests/*
 !tests/test_scala_analyzer.py
 !tests/test_agent_limits.py
 !tests/test_module_scope.py
+!tests/test_min_module_size.py
 !tests/test_doc_write_guard.py
 !tests/test_overview_generation.py
 

codewiki/cli/adapters/doc_generator.py

@@ -20,7 +20,7 @@
 # Import backend modules
 from codewiki.src.be.documentation_generator import DocumentationGenerator
 from codewiki.src.be.doc_layout import list_doc_files
-from codewiki.src.config import DEFAULT_LAYOUT
+from codewiki.src.config import DEFAULT_LAYOUT, DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
 from codewiki.src.config import Config as BackendConfig
 from codewiki.src.config import set_cli_context
 
@@ -154,8 +154,11 @@ def generate(self) -> DocumentationJob:
                 google_project=self.config.get("google_project", ""),
                 google_location=self.config.get("google_location", ""),
                 max_tokens=self.config.get("max_tokens", 32768),
+                min_token_per_module=self.config.get("min_token_per_module", 1000),
                 max_token_per_module=self.config.get("max_token_per_module", 36369),
-                max_token_per_leaf_module=self.config.get("max_token_per_leaf_module", 16000),
+                max_token_per_leaf_module=self.config.get(
+                    "max_token_per_leaf_module", DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
+                ),
                 max_leaf_nodes_per_cluster=self.config.get("max_leaf_nodes_per_cluster", 600),
                 max_depth=self.config.get("max_depth", 2),
                 request_limit=self.config.get("request_limit", 100),
@@ -341,7 +344,12 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
                 # Artifact nodes the clustering LLM dropped get a fixed module
                 # so build/CI/config coverage does not depend on the LLM.
                 if getattr(backend_config, "artifacts_enabled", True):
-                    module_tree = ensure_artifact_module(module_tree, leaf_nodes, components)
+                    module_tree = ensure_artifact_module(
+                        module_tree,
+                        leaf_nodes,
+                        components,
+                        min_tokens=backend_config.min_token_per_module,
+                    )
                 # Only freshly clustered trees are deduped: renaming a cached
                 # key whose .md already exists would orphan the doc.
                 from codewiki.src.be.module_naming import dedupe_module_tree_names

codewiki/cli/commands/config.py

@@ -11,6 +11,7 @@
 from codewiki.cli.config_manager import ConfigManager
 from codewiki.cli.models.config import AgentInstructions
 from codewiki.src.language import normalize_language
+from codewiki.src.config import DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
 from codewiki.cli.utils.errors import ConfigurationError, handle_error, EXIT_CONFIG_ERROR
 from codewiki.cli.utils.validation import (
     validate_url,
@@ -145,13 +146,21 @@ def pricing_remove(model, provider):
 @click.option("--cluster-model", type=str, help="Model for module clustering (recommend top-tier)")
 @click.option("--fallback-model", type=str, help="Fallback model for documentation generation")
 @click.option("--max-tokens", type=int, help="Maximum tokens for LLM response (default: 32768)")
+@click.option(
+    "--min-token-per-module",
+    type=click.IntRange(min=1),
+    default=None,
+    help="Minimum code tokens per standalone module (default: 1000)",
+)
 @click.option(
     "--max-token-per-module",
     type=int,
     help="Maximum tokens per module for clustering (default: 36369)",
 )
 @click.option(
-    "--max-token-per-leaf-module", type=int, help="Maximum tokens per leaf module (default: 16000)"
+    "--max-token-per-leaf-module",
+    type=int,
+    help=f"Base code-token threshold to allow delegation (default: {DEFAULT_MAX_TOKEN_PER_LEAF_MODULE})",
 )
 @click.option(
     "--max-depth", type=int, help="Maximum depth for hierarchical decomposition (default: 2)"
@@ -223,6 +232,7 @@ def config_set(
     max_token_per_module: Optional[int],
     max_token_per_leaf_module: Optional[int],
     max_depth: Optional[int],
+    min_token_per_module: Optional[int] = None,
     request_limit: Optional[int] = None,
     agent_retries: Optional[int] = None,
     provider: Optional[str] = None,
@@ -303,6 +313,7 @@ def config_set(
                 cluster_model,
                 fallback_model,
                 max_tokens,
+                min_token_per_module,
                 max_token_per_module,
                 max_token_per_leaf_module,
                 max_depth,
@@ -369,6 +380,9 @@ def config_set(
                 raise ConfigurationError("max_tokens must be a positive integer")
             validated_data["max_tokens"] = max_tokens
 
+        if min_token_per_module is not None:
+            validated_data["min_token_per_module"] = min_token_per_module
+
         if max_token_per_module is not None:
             if max_token_per_module < 1:
                 raise ConfigurationError("max_token_per_module must be a positive integer")
@@ -425,6 +439,7 @@ def config_set(
             cluster_model=validated_data.get("cluster_model"),
             fallback_model=validated_data.get("fallback_model"),
             max_tokens=validated_data.get("max_tokens"),
+            min_token_per_module=validated_data.get("min_token_per_module"),
             max_token_per_module=validated_data.get("max_token_per_module"),
             max_token_per_leaf_module=validated_data.get("max_token_per_leaf_module"),
             max_depth=validated_data.get("max_depth"),
@@ -476,6 +491,9 @@ def config_set(
         if max_tokens:
             click.secho(f"✓ Max tokens: {max_tokens}", fg="green")
 
+        if min_token_per_module is not None:
+            click.secho(f"✓ Min token per module: {min_token_per_module}", fg="green")
+
         if max_token_per_module:
             click.secho(f"✓ Max token per module: {max_token_per_module}", fg="green")
 
@@ -570,8 +588,11 @@ def config_show(output_json: bool):
                 "fallback_model": config.fallback_model if config else "glm-4p5",
                 "default_output": config.default_output if config else "docs",
                 "max_tokens": config.max_tokens if config else 32768,
+                "min_token_per_module": config.min_token_per_module if config else 1000,
                 "max_token_per_module": config.max_token_per_module if config else 36369,
-                "max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000,
+                "max_token_per_leaf_module": config.max_token_per_leaf_module
+                if config
+                else DEFAULT_MAX_TOKEN_PER_LEAF_MODULE,
                 "max_depth": config.max_depth if config else 2,
                 "request_limit": config.request_limit if config else 100,
                 "agent_retries": config.agent_retries if config else 3,
@@ -644,6 +665,7 @@ def config_show(output_json: bool):
             click.secho("Token Settings", fg="cyan", bold=True)
             if config:
                 click.echo(f"  Max Tokens:              {config.max_tokens}")
+                click.echo(f"  Min Token/Module:        {config.min_token_per_module}")
                 click.echo(f"  Max Token/Module:        {config.max_token_per_module}")
                 click.echo(f"  Max Token/Leaf Module:   {config.max_token_per_leaf_module}")
                 click.echo(f"  Prompt Caching:          {config.prompt_caching}")

codewiki/cli/commands/generate.py

@@ -302,6 +302,12 @@ def _find_affected(tree, parent_names=None):
     default=None,
     help="Maximum tokens for LLM response (overrides config)",
 )
+@click.option(
+    "--min-token-per-module",
+    type=click.IntRange(min=1),
+    default=None,
+    help="Minimum code tokens per standalone module (default: 1000)",
+)
 @click.option(
     "--max-token-per-module",
     type=int,
@@ -312,7 +318,7 @@ def _find_affected(tree, parent_names=None):
     "--max-token-per-leaf-module",
     type=int,
     default=None,
-    help="Maximum tokens per leaf module (overrides config)",
+    help="Base code-token threshold to allow delegation (overrides config)",
 )
 @click.option(
     "--max-depth",
@@ -469,6 +475,7 @@ def generate_command(
     request_limit: int | None,
     agent_retries: int | None,
     prompt_caching: bool | None,
+    min_token_per_module: int | None = None,
     main_model: str | None = None,
     cluster_model: str | None = None,
     fallback_model: str | None = None,
@@ -745,6 +752,9 @@ def generate_command(
             )
             logger.debug(f"Main model: {main_model or config.main_model}")
             logger.debug(f"Cluster model: {cluster_model or config.cluster_model}")
+            logger.debug(
+                f"Min token/module: {min_token_per_module if min_token_per_module is not None else getattr(config, 'min_token_per_module', 1000)}"
+            )
             logger.debug(f"Max tokens: {effective_max_tokens}")
             logger.debug(f"Max token/module: {effective_max_token_per_module}")
             logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}")
@@ -796,6 +806,9 @@ def generate_command(
                 "agent_instructions": agent_instructions_dict,
                 # Max token settings (runtime overrides take precedence)
                 "max_tokens": max_tokens if max_tokens is not None else config.max_tokens,
+                "min_token_per_module": min_token_per_module
+                if min_token_per_module is not None
+                else getattr(config, "min_token_per_module", 1000),
                 "max_token_per_module": max_token_per_module
                 if max_token_per_module is not None
                 else config.max_token_per_module,

codewiki/cli/config_manager.py

@@ -147,6 +147,7 @@ def save(
         use_gitignore: Optional[bool] = None,
         prompt_caching: Optional[bool] = None,
         pricing: Optional[dict] = None,
+        min_token_per_module: Optional[int] = None,
     ):
         """
         Save configuration to file and keyring.
@@ -159,6 +160,7 @@ def save(
             fallback_model: Fallback model
             default_output: Default output directory
             max_tokens: Maximum tokens for LLM response
+            min_token_per_module: Minimum code tokens for a standalone module
             max_token_per_module: Maximum tokens per module for clustering
             max_token_per_leaf_module: Maximum tokens per leaf module
             max_depth: Maximum depth for hierarchical decomposition
@@ -220,6 +222,14 @@ def save(
             self._config.default_output = default_output
         if max_tokens is not None:
             self._config.max_tokens = max_tokens
+        if min_token_per_module is not None:
+            from codewiki.src.config import validate_min_module_tokens
+
+            try:
+                validate_min_module_tokens(min_token_per_module)
+            except ValueError as exc:
+                raise ConfigurationError(str(exc)) from exc
+            self._config.min_token_per_module = min_token_per_module
         if max_token_per_module is not None:
             self._config.max_token_per_module = max_token_per_module
         if max_token_per_leaf_module is not None:

codewiki/cli/models/config.py

@@ -9,6 +9,11 @@
 from dataclasses import dataclass, field
 
 from codewiki.src.language import format_language_directive
+from codewiki.src.config import (
+    DEFAULT_MAX_TOKEN_PER_LEAF_MODULE,
+    DEFAULT_MIN_TOKEN_PER_MODULE,
+    validate_min_module_tokens,
+)
 from codewiki.cli.utils.validation import (
     validate_model_name,
     validate_url,
@@ -154,8 +159,9 @@ class Configuration:
         google_project: Google Cloud project for google-vertex
         google_location: Vertex AI location for google-vertex
         max_tokens: Maximum tokens for LLM response (default: 32768)
+        min_token_per_module: Minimum code tokens per standalone module (default: 1000)
         max_token_per_module: Maximum tokens per module for clustering (default: 36369)
-        max_token_per_leaf_module: Maximum tokens per leaf module (default: 16000)
+        max_token_per_leaf_module: Base token threshold to allow delegation (default: 8000)
         max_depth: Maximum depth for hierarchical decomposition (default: 2)
         request_limit: Maximum model requests per agent run (default: 100)
         agent_retries: Retries for a failing tool call (default: 3)
@@ -176,8 +182,9 @@ class Configuration:
     google_project: str = ""
     google_location: str = ""
     max_tokens: int = 32768
+    min_token_per_module: int = DEFAULT_MIN_TOKEN_PER_MODULE
     max_token_per_module: int = 36369
-    max_token_per_leaf_module: int = 16000
+    max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
     max_depth: int = 2
     request_limit: int = 100
     agent_retries: int = 3
@@ -192,6 +199,7 @@ def __post_init__(self):
 
         try:
             validate_pricing(self.pricing)
+            validate_min_module_tokens(self.min_token_per_module)
         except ValueError as exc:
             raise ConfigurationError(str(exc)) from exc
 
@@ -207,6 +215,8 @@ def validate(self):
         """
         from codewiki.src.be.backend import is_caw_provider, needs_base_url
 
+        validate_min_module_tokens(self.min_token_per_module)
+
         if is_caw_provider(self.provider):
             validate_model_name(self.main_model)
             return
@@ -230,6 +240,7 @@ def to_dict(self) -> dict:
             "google_project": self.google_project,
             "google_location": self.google_location,
             "max_tokens": self.max_tokens,
+            "min_token_per_module": self.min_token_per_module,
             "max_token_per_module": self.max_token_per_module,
             "max_token_per_leaf_module": self.max_token_per_leaf_module,
             "max_depth": self.max_depth,
@@ -272,8 +283,11 @@ def from_dict(cls, data: dict) -> "Configuration":
             google_project=data.get("google_project", ""),
             google_location=data.get("google_location", ""),
             max_tokens=data.get("max_tokens", 32768),
+            min_token_per_module=data.get("min_token_per_module", DEFAULT_MIN_TOKEN_PER_MODULE),
             max_token_per_module=data.get("max_token_per_module", 36369),
-            max_token_per_leaf_module=data.get("max_token_per_leaf_module", 16000),
+            max_token_per_leaf_module=data.get(
+                "max_token_per_leaf_module", DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
+            ),
             max_depth=data.get("max_depth", 2),
             request_limit=data.get("request_limit", 100),
             agent_retries=data.get("agent_retries", 3),
@@ -342,6 +356,7 @@ def to_backend_config(
             google_project=self.google_project,
             google_location=self.google_location,
             max_tokens=self.max_tokens,
+            min_token_per_module=self.min_token_per_module,
             max_token_per_module=self.max_token_per_module,
             max_token_per_leaf_module=self.max_token_per_leaf_module,
             max_depth=self.max_depth,