diff --git a/.gitignore b/.gitignore index e05007a9058..a81d4afa99d 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ cecli/_version.py cecli/website/Gemfile.lock *.pyc env/ +__pycache__/ # Ignore Folders cecli/website/_site/* \ No newline at end of file diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 62c590f9adb..41331965417 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -150,6 +150,28 @@ def _get_agent_config(self): ): try: config = json.loads(self.args.agent_config) + + # Validate that array fields are lists, wrap scalars in lists + array_fields = [ + "skills_paths", + "skills_includelist", + "skills_excludelist", + "skills_init", + "subagent_paths", + "tools_paths", + "tools_includelist", + "tools_excludelist", + "servers_includelist", + "servers_excludelist", + "allowed_commands", + ] + for field in array_fields: + if field in config and not isinstance(config[field], list): + self.start_up_errors.append( + f"agent-config field '{field}' should be a list but got " + f"{type(config[field]).__name__}, wrapping in list" + ) + config[field] = [config[field]] except (json.JSONDecodeError, TypeError) as e: self.start_up_errors.append(f"Failed to parse agent-config JSON: {e}") return {} diff --git a/cecli/helpers/nested.py b/cecli/helpers/nested.py index c77be72b717..1153d94d478 100644 --- a/cecli/helpers/nested.py +++ b/cecli/helpers/nested.py @@ -1,3 +1,6 @@ +import copy +import json +import os from typing import Any, Dict, List, Union @@ -83,16 +86,190 @@ def getter( return default -def deep_merge(dict1, dict2): +DEEP_MERGE_LIST_FIELDS: frozenset[str] = frozenset( + { + # ── Top-level argparse action="append" args ────────────────── + # These are excluded from configargparse for .cecli.conf.yml files + # (to prevent shallow overwrite), so they must be deep-merged here. + "rules", + "file", + "read", + "mcp_servers_files", + "set_env", + "api_key", + "alias", + "exempt_paths", + "lint_cmd", + # ── Nested agent-config array fields ───────────────────────── + "skills_paths", + "skills_includelist", + "skills_excludelist", + "skills_init", + "subagent_paths", + "tools_paths", + "tools_includelist", + "tools_excludelist", + "servers_includelist", + "servers_excludelist", + "allowed_commands", + } +) + +DEEP_MERGE_JSON_FIELDS: frozenset[str] = frozenset( + { + "agent_config", + "mcp_servers", + "hooks", + "model_providers", + "security_config", + "retries", + "custom", + "tui_config", + } +) + + +def _normalize_keys(obj: Any) -> Any: + """ + Recursively convert hyphenated dict keys to underscore-separated keys. + + YAML config files may use hyphenated keys (e.g. ``agent-config``, + ``mcp-servers-files``), but argparse attributes and the + ``DEEP_MERGE_*_FIELDS`` frozensets use underscores (``agent_config``, + ``mcp_servers_files``). This helper normalizes loaded YAML dicts so + that key lookups against the frozensets succeed. + + Lists, scalars, and non-dict values are returned unchanged. + """ + if isinstance(obj, dict): + return {key.replace("-", "_"): _normalize_keys(value) for key, value in obj.items()} + if isinstance(obj, list): + return [_normalize_keys(item) for item in obj] + return obj + + +def _deduplicate_list(merged_list: list, new_list: list) -> list: + """ + Append elements from new_list to merged_list, skipping duplicates. + + Deduplication is value-based: + - Primitives (str, int, float, bool, None): compared with ``==`` + - Dicts: compared via ``json.dumps(obj, sort_keys=True)`` for stable + structural equality + - Other types: compared with ``==`` + + First-occurrence order is preserved: elements already in merged_list + keep their position; new unique elements are appended at the end. + + Each appended element is deep-copied via ``copy.deepcopy()`` to + prevent reference sharing between the merged result and the source + lists. + """ + # Build a set of canonical representations for fast lookup + seen: set = set() + for item in merged_list: + seen.add(_canonical_repr(item)) + + for item in new_list: + key = _canonical_repr(item) + if key not in seen: + seen.add(key) + merged_list.append(copy.deepcopy(item)) + + return merged_list + + +def _canonical_repr(item: Any) -> Any: + """ + Return a hashable, equality-comparable canonical representation + of *item* for deduplication purposes. + + - Primitives (str, int, float, bool, None, tuple): returned as-is + - Dicts: serialized to a JSON string with sorted keys + - Lists: recursively converted to a tuple of canonical representations + - Everything else: returned as-is (falls back to ``==``) """ - Recursively merges dict2 into dict1. - If a key exists in both and both values are dicts, it merges the sub-dicts. - Otherwise, the value from dict2 overwrites the value from dict1. + if isinstance(item, dict): + return json.dumps(item, sort_keys=True, default=str) + if isinstance(item, list): + return tuple(_canonical_repr(e) for e in item) + return item + + +def deep_merge(dict1, dict2, deep_merge_arrays=True): + """ + Recursively merges *dict2* into *dict1*. + + Parameters + ---------- + dict1 : dict + The base dictionary (earlier / lower-precedence config). + dict2 : dict + The overlay dictionary (later / higher-precedence config). + deep_merge_arrays : bool, optional + When ``True`` (the default), list values are concatenated with + deduplication instead of being overwritten. When ``False``, + the existing shallow-overwrite behaviour is preserved. + + Returns + ------- + dict + A new dictionary (deep-copied from *dict1*) with *dict2* merged + in. Neither *dict1* nor *dict2* is mutated. + + Merge rules (per key) + --------------------- + * Both values are dicts → recursively ``deep_merge`` the sub-dicts. + * Both values are lists **and** ``deep_merge_arrays=True`` → + concatenate with deduplication (first-occurrence order preserved). + * Otherwise → *dict2*'s value overwrites *dict1*'s value. """ - merged = dict1.copy() # Create a copy to avoid modifying original dict1 in place + merged = copy.deepcopy(dict1) for key, value in dict2.items(): if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): - merged[key] = deep_merge(merged[key], value) + merged[key] = deep_merge(merged[key], value, deep_merge_arrays=deep_merge_arrays) + elif ( + deep_merge_arrays + and key in merged + and isinstance(merged[key], list) + and isinstance(value, list) + ): + merged[key] = _deduplicate_list(merged[key], value) else: - merged[key] = value + merged[key] = copy.deepcopy(value) + return merged + + +def is_cecli_conf_file(filepath: str) -> bool: + """ + Return ``True`` if *filepath* refers to a ``.cecli.conf.yml`` file + (which should receive deep-merge treatment), ``False`` otherwise + (e.g. ``.cecli/conf.yml`` or unknown patterns). + + The check is based solely on the basename of the path. + """ + basename = os.path.basename(filepath) + return basename == ".cecli.conf.yml" + + +def deep_merge_config_dicts( + config_dicts: List[Dict[str, Any]], +) -> Dict[str, Any]: + """ + Cumulatively deep-merge a list of raw config dictionaries. + + Each successive dict is merged into the accumulator using + ``deep_merge(acc, next_dict, deep_merge_arrays=True)``. The + accumulator is deep-copied before each merge to prevent mutation + of intermediate results. + + Returns the final merged dictionary. If *config_dicts* is empty, + returns an empty dict. + """ + if not config_dicts: + return {} + + merged = copy.deepcopy(config_dicts[0]) + for next_dict in config_dicts[1:]: + merged = deep_merge(merged, next_dict, deep_merge_arrays=True) return merged diff --git a/cecli/main.py b/cecli/main.py index aeaf49f1acd..b7982129193 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -1,5 +1,12 @@ +import json +import logging import os import sys +from typing import Any + +import yaml + +from cecli.helpers import nested try: if sys.platform == "win32": @@ -18,7 +25,6 @@ pass import asyncio -import json import os import re import shutil @@ -573,14 +579,27 @@ async def main_async( else: git_root = get_git_root() conf_fname = handle_core_files(Path(".cecli.conf.yml")) - default_config_files = [ + # Split config files into two groups: + # - conf_yml_files: .cecli/conf.yml (shallow merge via configargparse) + # - cecli_conf_yml_files: .cecli.conf.yml (deep merge handled by us) + # configargparse must NOT see .cecli.conf.yml files because it would + # shallow-merge (overwrite) them, destroying values from earlier files + # before our deep-merge code can run. + all_config_paths = [ str(Path.home() / ".cecli" / "conf.yml"), str(Path.home() / ".cecli.conf.yml"), str(Path(".cecli.conf.yml")), ] if git_root: - default_config_files.append(str(Path(git_root) / ".cecli.conf.yml")) - parser = get_parser(default_config_files, git_root) + all_config_paths.append(str(Path(git_root) / ".cecli.conf.yml")) + + conf_yml_files = [ + p for p in all_config_paths if p.endswith("conf.yml") and not p.endswith(".cecli.conf.yml") + ] + cecli_conf_yml_files = [p for p in all_config_paths if p.endswith(".cecli.conf.yml")] + + # Only pass .cecli/conf.yml files to configargparse for shallow merge + parser = get_parser(conf_yml_files, git_root) args, unknown = parser.parse_known_args(argv) # Load dotenv files and re-parse args before workspace logic @@ -615,15 +634,133 @@ async def main_async( if git_root: git_conf = Path(git_root) / conf_fname - if git_conf not in default_config_files: - default_config_files.append(str(git_conf)) + if git_conf not in all_config_paths: + all_config_paths.append(str(git_conf)) + cecli_conf_yml_files.append(str(git_conf)) if uses_workspace: - parser = get_parser(default_config_files, git_root) + parser = get_parser(conf_yml_files, git_root) args, unknown = parser.parse_known_args(argv) set_args_error_data(args) + # ── Deep merge .cecli.conf.yml files ────────────────────────────────── + # configargparse has already shallow-merged .cecli/conf.yml files onto + # args (correct behaviour for those files). We now read every + # .cecli.conf.yml file directly from disk, deep-merge them together, + # and then merge the result ON TOP OF the existing args values (which + # came from .cecli/conf.yml). This way .cecli/conf.yml values are + # preserved and .cecli.conf.yml values are concatenated/deduped on top. + + existing_cecli_conf = [cf for cf in cecli_conf_yml_files if os.path.exists(cf)] + + if existing_cecli_conf: + cecli_conf_dicts: list[dict[str, Any]] = [] + for config_file in existing_cecli_conf: + try: + with open(config_file, "r") as f: + config_data = yaml.safe_load(f) + if isinstance(config_data, dict): + cecli_conf_dicts.append(nested._normalize_keys(config_data)) + except yaml.YAMLError as e: + logging.warning("Could not parse YAML file %s: %s", config_file, e) + + if cecli_conf_dicts: + # Deep-merge .cecli.conf.yml files together + merged_cecli_conf = nested.deep_merge_config_dicts(cecli_conf_dicts) + + # ── List fields: concatenate with existing args values ────── + for key in nested.DEEP_MERGE_LIST_FIELDS: + if key in merged_cecli_conf and hasattr(args, key): + cecli_val = merged_cecli_conf[key] + if not isinstance(cecli_val, list): + logging.warning( + "Merged .cecli.conf.yml value for %s is not a list (type: %s), " + "expected list", + key, + type(cecli_val).__name__, + ) + continue + existing_val = getattr(args, key) + if isinstance(existing_val, list): + # Concatenate: .cecli/conf.yml values first, then + # .cecli.conf.yml values (deduplicated) + setattr(args, key, nested._deduplicate_list(list(existing_val), cecli_val)) + else: + setattr(args, key, cecli_val) + + # ── JSON fields: deep-merge into existing args values ─────── + for key in nested.DEEP_MERGE_JSON_FIELDS: + if key in merged_cecli_conf and hasattr(args, key): + cecli_val = merged_cecli_conf[key] + if not isinstance(cecli_val, dict): + logging.warning( + "Merged .cecli.conf.yml value for %s is not a dict (type: %s), " + "expected dict", + key, + type(cecli_val).__name__, + ) + continue + existing_val = getattr(args, key) + if existing_val is not None: + # Parse existing value (may be JSON string from + # configargparse) and deep-merge .cecli.conf.yml + # values into it + if isinstance(existing_val, str): + try: + existing_val = json.loads(existing_val) + except json.JSONDecodeError: + try: + existing_val = yaml.safe_load(existing_val) + except yaml.YAMLError: + logging.warning( + "Could not parse existing %s value as JSON/YAML, " + "overwriting with .cecli.conf.yml value", + key, + ) + existing_val = {} + if isinstance(existing_val, dict): + merged_val = nested.deep_merge( + existing_val, cecli_val, deep_merge_arrays=True + ) + else: + merged_val = cecli_val + else: + merged_val = cecli_val + try: + setattr(args, key, json.dumps(merged_val)) + except (TypeError, ValueError) as e: + logging.warning("Could not serialize merged config for %s: %s", key, e) + else: + logging.debug( + "No .cecli.conf.yml files found — shallow-merge-only mode active " + "(configargparse handles .cecli/conf.yml)" + ) + + # ── Normalize array fields to never be None ─────────────────────────── + # After merging, ensure every DEEP_MERGE_LIST_FIELDS attribute is at + # minimum an empty list [] and every DEEP_MERGE_JSON_FIELDS attribute + # is at minimum an empty JSON object "{}". Downstream code should never + # have to deal with None for these fields. + for key in nested.DEEP_MERGE_LIST_FIELDS: + if hasattr(args, key): + val = getattr(args, key) + if val is None or (isinstance(val, str) and val.strip() == ""): + setattr(args, key, []) + elif not isinstance(val, list): + logging.warning( + "args.%s is not a list (type: %s), coercing to empty list", + key, + type(val).__name__, + ) + setattr(args, key, []) + + for key in nested.DEEP_MERGE_JSON_FIELDS: + if hasattr(args, key): + val = getattr(args, key) + if val is None or (isinstance(val, str) and val.strip() == ""): + setattr(args, key, "{}") + if len(unknown): print("Unknown Args: ", unknown) diff --git a/cecli/website/docs/config/conf.md b/cecli/website/docs/config/conf.md index ad3c1a8578c..14c69820d03 100644 --- a/cecli/website/docs/config/conf.md +++ b/cecli/website/docs/config/conf.md @@ -23,6 +23,135 @@ You can also specify the `--config ` parameter, which will only load t Lists of values can be specified either as a bulleted list: +## Deep merge behavior for `.cecli.conf.yml` + +There are two types of YAML config files with different merge behaviors: + +| Config File | Merge Behavior | Array Fields | +|-------------|----------------|--------------| +| `.cecli/conf.yml` | Shallow merge (later values replace earlier) | All fields | +| `.cecli.conf.yml` | Deep merge (arrays are concatenated with deduplication) | Array fields only | + +### How deep merge works + +When you have multiple `.cecli.conf.yml` files (e.g., in your home directory and git root), array fields are **concatenated** rather than replaced. Duplicate values are automatically removed, and the order of entries is preserved (earlier config files keep their position). + +```yaml +# ~/.cecli.conf.yml +read: + - ~/.cecli/rules/global.txt + +# project/.cecli.conf.yml +read: + - ./project-rules.txt + +# Result: ["~/.cecli/rules/global.txt", "./project-rules.txt"] +``` + +### Array fields that support deep merge + +The following **top-level** array fields are deep-merged: + +- `rules` - Rules files to load +- `file` - Files to edit +- `read` - Read-only files +- `mcp-servers-files` - MCP server configuration files +- `set-env` - Environment variables +- `api-key` - API keys +- `alias` - Model aliases +- `exempt-paths` - Exempt paths +- `lint-cmd` - Lint commands + +The following **nested** array fields (inside `agent-config`) are also deep-merged: + +- `skills_paths` - Directories to search for skills +- `skills_includelist` - Skills to include (whitelist) +- `skills_excludelist` - Skills to exclude (blacklist) +- `skills_init` - Skills to load on startup +- `subagent_paths` - Directories for sub-agent definitions +- `tools_paths` - Directories for tools +- `tools_includelist` - Tools to include +- `tools_excludelist` - Tools to exclude +- `servers_includelist` - MCP servers to include +- `servers_excludelist` - MCP servers to exclude +- `allowed_commands` - Allowed shell commands + +### JSON/YAML string fields + +The following fields are stored as JSON/YAML strings but are **internally deep-merged** at the dict/list level: + +- `agent-config` - Agent configuration +- `mcp-servers` - MCP server definitions +- `hooks` - Hook configurations +- `model-providers` - Model provider configurations +- `security-config` - Security settings +- `retries` - Retry configuration +- `custom` - Custom configurations +- `tui-config` - TUI configuration + +### Examples + +#### Adding skills directories + +```yaml +# ~/.cecli.conf.yml +agent-config: '{"skills_paths": [".cecli/skills"]}' + +# project/.cecli.conf.yml +agent-config: '{"skills_paths": ["./project-skills"]}' + +# Result: skills_paths contains both ".cecli/skills" and "./project-skills" +``` + +#### Extending allowed commands + +```yaml +# ~/.cecli.conf.yml +allowed-commands: + - git + - npm + +# project/.cecli.conf.yml +allowed-commands: + - docker + - git + +# Result: ["git", "npm", "docker"] (git appears only once, order preserved) +``` + +#### Adding lint commands + +```yaml +# ~/.cecli.conf.yml +lint-cmd: + - python: flake8 + +# project/.cecli.conf.yml +lint-cmd: + - javascript: eslint + +# Result: Both lint commands are active +``` + +### Deduplication + +Entries are deduplicated based on value equality. For primitive values (strings, numbers), equality is checked with `==`. For dicts, `json.dumps` comparison is used. The first occurrence of a value is preserved, ensuring consistent ordering. + +```yaml +# config1 +skills-includelist: + - skill-a + - skill-b + +# config2 +skills-includelist: + - skill-b + - skill-c + +# Result: ["skill-a", "skill-b", "skill-c"] (skill-b not duplicated) +``` + + ``` read: - CONVENTIONS.md diff --git a/tests/integration/test_config_loading.py b/tests/integration/test_config_loading.py new file mode 100644 index 00000000000..185f005af7c --- /dev/null +++ b/tests/integration/test_config_loading.py @@ -0,0 +1,203 @@ +import json +import os +import sys +import unittest + +import yaml + +# Mock the configargparse before it's imported by other modules +# This is a bit of a hack, but necessary for this kind of integration test + + +class MockArgumentParser: + def __init__(self, *args, **kwargs): + self.args = {} + self.default_config_files = [] + + def add_argument(self, *args, **kwargs): + action = kwargs.get("action") + dest = kwargs.get("dest") + if not dest: + for arg in args: + if arg.startswith("--"): + dest = arg.lstrip("-").replace("-", "_") + break + if action == "append": + self.args[dest] = [] + else: + self.args[dest] = None + + def parse_known_args(self, argv=None): + # A very simplified parser that just recognizes file paths + # In a real scenario, this would be much more complex + # For this test, we assume the args are pre-populated by mock config files + + # Simulate configargparse loading files and setting args + all_configs = {} + for f in self.default_config_files: + if os.path.exists(f): + with open(f, "r") as stream: + config = yaml.safe_load(stream) + # Shallow merge for simulation + all_configs.update(config) + + for key, value in all_configs.items(): + if key in self.args: + if isinstance(self.args[key], list): + self.args[key].extend(value) + else: + self.args[key] = value + + # Convert to a namespace-like object + class ArgsNamespace: + def __init__(self, d): + self.__dict__.update(d) + + return ArgsNamespace(self.args), [] + + +sys.modules["configargparse"].ArgumentParser = MockArgumentParser + + +from cecli.helpers.nested import ( # noqa: E402 + deep_merge_config_dicts, +) + + +class TestConfigLoading(unittest.TestCase): + + def setUp(self): + # Create mock config files + self.conf_dir = "temp_test_conf" + os.makedirs(self.conf_dir, exist_ok=True) + + self.git_root_conf_path = os.path.join(self.conf_dir, ".cecli.conf.yml") + self.user_conf_dir = os.path.join(self.conf_dir, "user") + os.makedirs(self.user_conf_dir, exist_ok=True) + self.user_conf_path = os.path.join(self.user_conf_dir, ".cecli.conf.yml") + self.legacy_conf_path = os.path.join(self.conf_dir, ".cecli", "conf.yml") + os.makedirs(os.path.dirname(self.legacy_conf_path), exist_ok=True) + + def tearDown(self): + import shutil + + shutil.rmtree(self.conf_dir) + + def test_multiple_config_files_deep_merge(self): + # 1. Create mock config files with content + legacy_content = {"skills_paths": ["/legacy/skills"], "model": "gpt-3.5"} + user_content = {"skills_paths": ["/user/skills"], "model": "gpt-4", "temperature": 0.5} + git_root_content = {"skills_paths": ["./project/skills"], "temperature": 0.7} + + with open(self.legacy_conf_path, "w") as f: + yaml.dump(legacy_content, f) + with open(self.user_conf_path, "w") as f: + yaml.dump(user_content, f) + with open(self.git_root_conf_path, "w") as f: + yaml.dump(git_root_content, f) + + # 2. Simulate the loading and merging process from main.py. + # The fix reads ALL config files (both .cecli/conf.yml and + # .cecli.conf.yml) from disk and deep-merges them together, + # replacing configargparse's shallow-merged result for array fields. + + all_config_dicts = [ + legacy_content, + user_content, + git_root_content, + ] + merged_config = deep_merge_config_dicts(all_config_dicts) + + # Non-array fields: last file wins (deep_merge overwrites scalars) + # Array fields: concatenated with dedup across ALL config files + final_config = merged_config + + # 3. Assert the final merged config is correct + # skills_paths: all three sources concatenated (deep merge) + # model: last .cecli.conf.yml wins (scalar overwrite) + # temperature: last .cecli.conf.yml wins (scalar overwrite) + expected = { + "skills_paths": ["/legacy/skills", "/user/skills", "./project/skills"], + "model": "gpt-4", + "temperature": 0.7, + } + self.assertEqual(final_config["skills_paths"], expected["skills_paths"]) + self.assertEqual(final_config["model"], expected["model"]) + self.assertEqual(final_config["temperature"], expected["temperature"]) + + def test_config_precedence_order(self): + # Files are loaded in order, later files override earlier ones + # .cecli/conf.yml (shallow) < .cecli.conf.yml (deep) < git_root/.cecli.conf.yml (deep) + + # Create files + with open(self.legacy_conf_path, "w") as f: + yaml.dump({"a": 1, "b": [10]}, f) + with open(self.user_conf_path, "w") as f: + yaml.dump({"a": 2, "b": [20]}, f) + with open(self.git_root_conf_path, "w") as f: + yaml.dump({"a": 3, "b": [30]}, f) + + # Simulate loading + # Configargparse shallow merges all first. The last value for 'a' and 'b' wins. + # Our manual deep merge will then be applied. + + # 1. Base from shallow file + base_config = yaml.safe_load(open(self.legacy_conf_path)) + + # 2. Deep merge the others + + # Manually simulate the process + # Start with shallow merge result + temp_merged = {} + temp_merged.update(base_config) + temp_merged.update(yaml.safe_load(open(self.user_conf_path))) + temp_merged.update(yaml.safe_load(open(self.git_root_conf_path))) + + # Now, apply our deep merge logic for array fields + final_config = deep_merge_config_dicts( + [ + base_config, + yaml.safe_load(open(self.user_conf_path)), + yaml.safe_load(open(self.git_root_conf_path)), + ] + ) + + # 'a' is a scalar, so last one wins + self.assertEqual(final_config["a"], 3) + # 'b' is an array, should be deep merged + self.assertEqual(final_config["b"], [10, 20, 30]) + + def test_agent_config_json_parsing(self): + # This tests that a JSON string field like 'agent_config' is correctly merged + conf1 = {"agent_config": {"skills_paths": ["/path1"], "tools_includelist": ["tool-a"]}} + conf2 = { + "agent_config": {"skills_paths": ["/path2"], "tools_includelist": ["tool-b", "tool-a"]} + } + + # Simulate loading from two .cecli.conf.yml files + merged = deep_merge_config_dicts([conf1, conf2]) + + # The result is a Python dict. In main.py, this would be re-serialized to JSON. + expected_agent_config = { + "skills_paths": ["/path1", "/path2"], + "tools_includelist": ["tool-a", "tool-b"], + } + + self.assertEqual(merged["agent_config"], expected_agent_config) + + # Test the re-serialization step + final_json_string = json.dumps(merged["agent_config"]) + parsed_final = json.loads(final_json_string) + + # Sort lists for comparison since order inside JSON can be tricky + self.assertEqual( + sorted(parsed_final["skills_paths"]), sorted(expected_agent_config["skills_paths"]) + ) + self.assertEqual( + sorted(parsed_final["tools_includelist"]), + sorted(expected_agent_config["tools_includelist"]), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_deep_merge.py b/tests/unit/test_deep_merge.py new file mode 100644 index 00000000000..c2a77b49c2b --- /dev/null +++ b/tests/unit/test_deep_merge.py @@ -0,0 +1,100 @@ +import json +import unittest + +from cecli.helpers.nested import ( + _deduplicate_list, + deep_merge, + deep_merge_config_dicts, + is_cecli_conf_file, +) + + +class TestDeepMergeArrays(unittest.TestCase): + def test_deep_merge_simple_arrays(self): + dict1 = { + "skills_paths": [".cecli/skills"], + "subagent_paths": [".cecli/subagents"], + "tools_paths": [".cecli/tools"], + "allowed_commands": ["git", "npm"], + } + dict2 = { + "skills_paths": ["./project-skills"], + "subagent_paths": ["./custom-subagents"], + "tools_paths": ["./project-tools"], + "allowed_commands": ["docker", "git"], + } + expected = { + "skills_paths": [".cecli/skills", "./project-skills"], + "subagent_paths": [".cecli/subagents", "./custom-subagents"], + "tools_paths": [".cecli/tools", "./project-tools"], + "allowed_commands": ["git", "npm", "docker"], + } + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_deep_merge_with_duplicates(self): + dict1 = {"skills_includelist": ["skill-a", "skill-b"]} + dict2 = {"skills_includelist": ["skill-b", "skill-c"]} + expected = {"skills_includelist": ["skill-a", "skill-b", "skill-c"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_deep_merge_order_preservation(self): + dict1 = {"tools_includelist": ["tool-a", "tool-b"]} + dict2 = {"tools_includelist": ["tool-c", "tool-d"]} + expected = {"tools_includelist": ["tool-a", "tool-b", "tool-c", "tool-d"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_deep_merge_nested_arrays(self): + dict1 = {"mcp_servers": [{"server-a": {"command": ["cmd1"]}}]} + dict2 = {"mcp_servers": [{"server-b": {"command": ["cmd2"]}}]} + expected = { + "mcp_servers": [ + {"server-a": {"command": ["cmd1"]}}, + {"server-b": {"command": ["cmd2"]}}, + ] + } + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_json_field_serialization(self): + dict1 = {"mcp_servers": {"server-a": {"command": ["cmd1"]}}} + dict2 = {"mcp_servers": {"server-b": {"command": ["cmd2"]}}} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + try: + json.dumps(merged) + except TypeError: + self.fail("deep_merge result is not JSON serializable") + + def test_json_field_deep_merge(self): + dict1 = {"mcp_servers": {"server-a": {"command": ["cmd1"]}}} + dict2 = {"mcp_servers": {"server-b": {"command": ["cmd2"]}}} + expected = { + "mcp_servers": {"server-a": {"command": ["cmd1"]}, "server-b": {"command": ["cmd2"]}} + } + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + +class TestConfigHelpers(unittest.TestCase): + def test_is_cecli_conf_file(self): + self.assertTrue(is_cecli_conf_file("/path/to/.cecli.conf.yml")) + self.assertFalse(is_cecli_conf_file("/path/to/.cecli/conf.yml")) + self.assertFalse(is_cecli_conf_file("conf.yml")) + + def test_deep_merge_config_dicts(self): + config1 = {"a": [1], "b": 2} + config2 = {"a": [3], "c": 4} + config3 = {"a": [1, 5], "b": 6} + configs = [config1, config2, config3] + expected = {"a": [1, 3, 5], "b": 6, "c": 4} + merged = deep_merge_config_dicts(configs) + self.assertEqual(merged, expected) + + def test_deduplicate_list(self): + list1 = [1, 2, {"a": 1}] + list2 = [2, 3, {"a": 1}, {"b": 2}] + expected = [1, 2, {"a": 1}, 3, {"b": 2}] + result = _deduplicate_list(list1, list2) + self.assertEqual(result, expected) diff --git a/tests/unit/test_edge_cases.py b/tests/unit/test_edge_cases.py new file mode 100644 index 00000000000..0bdd1da643d --- /dev/null +++ b/tests/unit/test_edge_cases.py @@ -0,0 +1,50 @@ +import unittest + +import yaml + +from cecli.helpers.nested import deep_merge + + +class TestEdgeCases(unittest.TestCase): + def test_empty_arrays(self): + dict1 = {"skills_paths": []} + dict2 = {"skills_paths": ["./project-skills"]} + expected = {"skills_paths": ["./project-skills"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_null_and_missing_values(self): + dict1 = {"skills_paths": ["./project-skills"]} + dict2 = {"skills_paths": None} + expected = {"skills_paths": None} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + dict1 = {"skills_paths": ["./project-skills"]} + dict2 = {} + expected = {"skills_paths": ["./project-skills"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_non_array_fields_unchanged(self): + dict1 = {"model": "gpt-4", "temperature": 0.5} + dict2 = {"model": "claude-2", "dark_mode": True} + expected = {"model": "claude-2", "temperature": 0.5, "dark_mode": True} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_malformed_yaml(self): + """Verify graceful error handling with invalid YAML syntax.""" + malformed_yaml = "skills_paths: [unclosed quote\n" + with self.assertRaises(yaml.YAMLError): + yaml.safe_load(malformed_yaml) + + # Verify the try-except pattern works without crashing + try: + yaml.safe_load(malformed_yaml) + except yaml.YAMLError: + pass # Expected — graceful handling confirmed + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_performance.py b/tests/unit/test_performance.py new file mode 100644 index 00000000000..83518e45344 --- /dev/null +++ b/tests/unit/test_performance.py @@ -0,0 +1,27 @@ +import time +import unittest + +from cecli.helpers.nested import deep_merge + + +class TestPerformance(unittest.TestCase): + def test_large_arrays_performance(self): + dict1 = {"large_array": list(range(1000))} + dict2 = {"large_array": list(range(1000, 2000))} + start_time = time.time() + deep_merge(dict1, dict2, deep_merge_arrays=True) + end_time = time.time() + self.assertLess(end_time - start_time, 0.1) + + def test_deep_copy_behavior(self): + dict1 = {"a": [1], "b": {"c": [2]}} + dict2 = {"a": [3], "b": {"c": [4]}} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + merged["a"].append(4) + merged["b"]["c"].append(5) + self.assertEqual(dict1["a"], [1]) + self.assertEqual(dict1["b"]["c"], [2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_shallow_merge.py b/tests/unit/test_shallow_merge.py new file mode 100644 index 00000000000..cf439002efc --- /dev/null +++ b/tests/unit/test_shallow_merge.py @@ -0,0 +1,16 @@ +import unittest + +from cecli.helpers.nested import deep_merge + + +class TestShallowMerge(unittest.TestCase): + def test_shallow_merge_replaces_arrays(self): + dict1 = {"skills_paths": ["./old-skills"]} + dict2 = {"skills_paths": ["./new-skills"]} + expected = {"skills_paths": ["./new-skills"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=False) + self.assertEqual(merged, expected) + + +if __name__ == "__main__": + unittest.main()