From e4d50ae0ebc6b69e9fface1c697cd9432f14baa6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 16 Jul 2026 10:38:04 -0700 Subject: [PATCH 01/12] feat: Add deep merge support for array fields in .cecli.conf.yml --- cecli/coders/agent_coder.py | 14 ++++++++++++ cecli/main.py | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 943f6b1c538..82cfe8cf398 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -150,6 +150,20 @@ def _get_agent_config(self): ): try: config = json.loads(self.args.agent_config) + + # Validate that array fields are lists, wrap scalars in lists + array_fields = [ + "skills_paths", "skills_includelist", "skills_excludelist", "skills_init", + "subagent_paths", "tools_paths", "tools_includelist", "tools_excludelist", + "servers_includelist", "servers_excludelist", "allowed_commands" + ] + for field in array_fields: + if field in config and not isinstance(config[field], list): + self.start_up_errors.append( + f"agent-config field '{field}' should be a list but got " + f"{type(config[field]).__name__}, wrapping in list" + ) + config[field] = [config[field]] except (json.JSONDecodeError, TypeError) as e: self.start_up_errors.append(f"Failed to parse agent-config JSON: {e}") return {} diff --git a/cecli/main.py b/cecli/main.py index aea0c6b8690..897d79ac554 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -1,5 +1,9 @@ import os +import json import sys +import yaml + +from cecli.helpers import nested try: if sys.platform == "win32": @@ -610,6 +614,47 @@ async def main_async( set_args_error_data(args) + # Deep merge configuration files after parsing (unconditional) + deep_merge_configs = [] + shallow_merge_configs = [] + + for config_file in default_config_files: + if os.path.exists(config_file): + try: + with open(config_file, "r") as f: + config_data = yaml.safe_load(f) + if nested.is_cecli_conf_file(config_file): + deep_merge_configs.append(config_data) + else: + shallow_merge_configs.append(config_data) + except yaml.YAMLError as e: + print(f"Warning: Could not parse YAML file {config_file}: {e}") + + if deep_merge_configs: + merged_config = nested.deep_merge_config_dicts(deep_merge_configs) + + # Step 5.1: Post-merge type validation for list fields + for key in nested.DEEP_MERGE_LIST_FIELDS: + if key in merged_config and hasattr(args, key): + if not isinstance(merged_config[key], list): + print( + f"Warning: Merged config for {key} is not a list " + f"(type: {type(merged_config[key]).__name__}), expected list" + ) + setattr(args, key, merged_config[key]) + + for key in nested.DEEP_MERGE_JSON_FIELDS: + if key in merged_config and hasattr(args, key): + if not isinstance(merged_config[key], dict): + print( + f"Warning: Merged config for {key} is not a dict " + f"(type: {type(merged_config[key]).__name__}), expected dict" + ) + try: + setattr(args, key, json.dumps(merged_config[key])) + except (TypeError, ValueError) as e: + print(f"Warning: Could not serialize merged config for {key}: {e}") + if len(unknown): print("Unknown Args: ", unknown) From ffe5792775db1dd3e78db265eaf7e514f41b34fd Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 16 Jul 2026 19:37:08 -0700 Subject: [PATCH 02/12] feat: Implement test suite per section 10 of plans --- tests/unit/test_deep_merge.py | 97 ++++++++++++++++++++++++++++++++ tests/unit/test_edge_cases.py | 50 ++++++++++++++++ tests/unit/test_performance.py | 27 +++++++++ tests/unit/test_shallow_merge.py | 16 ++++++ 4 files changed, 190 insertions(+) create mode 100644 tests/unit/test_deep_merge.py create mode 100644 tests/unit/test_edge_cases.py create mode 100644 tests/unit/test_performance.py create mode 100644 tests/unit/test_shallow_merge.py diff --git a/tests/unit/test_deep_merge.py b/tests/unit/test_deep_merge.py new file mode 100644 index 00000000000..ed7f53fa39e --- /dev/null +++ b/tests/unit/test_deep_merge.py @@ -0,0 +1,97 @@ +import unittest +import json +from cecli.helpers.nested import ( + deep_merge, + _deduplicate_list, + _canonical_repr, + is_cecli_conf_file, + deep_merge_config_dicts, +) + +class TestDeepMergeArrays(unittest.TestCase): + def test_deep_merge_simple_arrays(self): + dict1 = { + "skills_paths": [".cecli/skills"], + "subagent_paths": [".cecli/subagents"], + "tools_paths": [".cecli/tools"], + "allowed_commands": ["git", "npm"] + } + dict2 = { + "skills_paths": ["./project-skills"], + "subagent_paths": ["./custom-subagents"], + "tools_paths": ["./project-tools"], + "allowed_commands": ["docker", "git"] + } + expected = { + "skills_paths": [".cecli/skills", "./project-skills"], + "subagent_paths": [".cecli/subagents", "./custom-subagents"], + "tools_paths": [".cecli/tools", "./project-tools"], + "allowed_commands": ["git", "npm", "docker"] + } + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_deep_merge_with_duplicates(self): + dict1 = {"skills_includelist": ["skill-a", "skill-b"]} + dict2 = {"skills_includelist": ["skill-b", "skill-c"]} + expected = {"skills_includelist": ["skill-a", "skill-b", "skill-c"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_deep_merge_order_preservation(self): + dict1 = {"tools_includelist": ["tool-a", "tool-b"]} + dict2 = {"tools_includelist": ["tool-c", "tool-d"]} + expected = {"tools_includelist": ["tool-a", "tool-b", "tool-c", "tool-d"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_deep_merge_nested_arrays(self): + dict1 = {"mcp_servers": [{"server-a": {"command": ["cmd1"]}}]} + dict2 = {"mcp_servers": [{"server-b": {"command": ["cmd2"]}}]} + expected = {"mcp_servers": [{"server-a": {"command": ["cmd1"]}}, {"server-b": {"command": ["cmd2"]}}]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + + def test_json_field_serialization(self): + dict1 = {"mcp_servers": {"server-a": {"command": ["cmd1"]}}} + dict2 = {"mcp_servers": {"server-b": {"command": ["cmd2"]}}} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + try: + json.dumps(merged) + except TypeError: + self.fail("deep_merge result is not JSON serializable") + + def test_json_field_deep_merge(self): + dict1 = {"mcp_servers": {"server-a": {"command": ["cmd1"]}}} + dict2 = {"mcp_servers": {"server-b": {"command": ["cmd2"]}}} + expected = {"mcp_servers": {"server-a": {"command": ["cmd1"]}, "server-b": {"command": ["cmd2"]}}} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + + + + + +class TestConfigHelpers(unittest.TestCase): + def test_is_cecli_conf_file(self): + self.assertTrue(is_cecli_conf_file("/path/to/.cecli.conf.yml")) + self.assertFalse(is_cecli_conf_file("/path/to/.cecli/conf.yml")) + self.assertFalse(is_cecli_conf_file("conf.yml")) + + def test_deep_merge_config_dicts(self): + config1 = {"a": [1], "b": 2} + config2 = {"a": [3], "c": 4} + config3 = {"a": [1, 5], "b": 6} + configs = [config1, config2, config3] + expected = {"a": [1, 3, 5], "b": 6, "c": 4} + merged = deep_merge_config_dicts(configs) + self.assertEqual(merged, expected) + + def test_deduplicate_list(self): + list1 = [1, 2, {"a": 1}] + list2 = [2, 3, {"a": 1}, {"b": 2}] + expected = [1, 2, {"a": 1}, 3, {"b": 2}] + result = _deduplicate_list(list1, list2) + self.assertEqual(result, expected) \ No newline at end of file diff --git a/tests/unit/test_edge_cases.py b/tests/unit/test_edge_cases.py new file mode 100644 index 00000000000..17233b21cb3 --- /dev/null +++ b/tests/unit/test_edge_cases.py @@ -0,0 +1,50 @@ +import unittest + +import yaml + +from cecli.helpers.nested import deep_merge + + +class TestEdgeCases(unittest.TestCase): + def test_empty_arrays(self): + dict1 = {"skills_paths": []} + dict2 = {"skills_paths": ["./project-skills"]} + expected = {"skills_paths": ["./project-skills"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_null_and_missing_values(self): + dict1 = {"skills_paths": ["./project-skills"]} + dict2 = {"skills_paths": None} + expected = {"skills_paths": None} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + dict1 = {"skills_paths": ["./project-skills"]} + dict2 = {} + expected = {"skills_paths": ["./project-skills"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_non_array_fields_unchanged(self): + dict1 = {"model": "gpt-4", "temperature": 0.5} + dict2 = {"model": "claude-2", "dark_mode": True} + expected = {"model": "claude-2", "temperature": 0.5, "dark_mode": True} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + self.assertEqual(merged, expected) + + def test_malformed_yaml(self): + """Verify graceful error handling with invalid YAML syntax.""" + malformed_yaml = "skills_paths: [unclosed quote\n" + with self.assertRaises(yaml.YAMLError): + yaml.safe_load(malformed_yaml) + + # Verify the try-except pattern works without crashing + try: + yaml.safe_load(malformed_yaml) + except yaml.YAMLError: + pass # Expected — graceful handling confirmed + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/unit/test_performance.py b/tests/unit/test_performance.py new file mode 100644 index 00000000000..eec48979f67 --- /dev/null +++ b/tests/unit/test_performance.py @@ -0,0 +1,27 @@ +import time +import unittest + +from cecli.helpers.nested import deep_merge + + +class TestPerformance(unittest.TestCase): + def test_large_arrays_performance(self): + dict1 = {"large_array": list(range(1000))} + dict2 = {"large_array": list(range(1000, 2000))} + start_time = time.time() + deep_merge(dict1, dict2, deep_merge_arrays=True) + end_time = time.time() + self.assertLess(end_time - start_time, 0.1) + + def test_deep_copy_behavior(self): + dict1 = {"a": [1], "b": {"c": [2]}} + dict2 = {"a": [3], "b": {"c": [4]}} + merged = deep_merge(dict1, dict2, deep_merge_arrays=True) + merged["a"].append(4) + merged["b"]["c"].append(5) + self.assertEqual(dict1["a"], [1]) + self.assertEqual(dict1["b"]["c"], [2]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/unit/test_shallow_merge.py b/tests/unit/test_shallow_merge.py new file mode 100644 index 00000000000..c4dd4747ebb --- /dev/null +++ b/tests/unit/test_shallow_merge.py @@ -0,0 +1,16 @@ +import unittest + +from cecli.helpers.nested import deep_merge + + +class TestShallowMerge(unittest.TestCase): + def test_shallow_merge_replaces_arrays(self): + dict1 = {"skills_paths": ["./old-skills"]} + dict2 = {"skills_paths": ["./new-skills"]} + expected = {"skills_paths": ["./new-skills"]} + merged = deep_merge(dict1, dict2, deep_merge_arrays=False) + self.assertEqual(merged, expected) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 8a4c09c630bce0b8dd556b8f58c1e7e8aca5300a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 16 Jul 2026 19:39:30 -0700 Subject: [PATCH 03/12] fix: Implement deep merge for .cecli.conf.yml files --- cecli/main.py | 94 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/cecli/main.py b/cecli/main.py index 897d79ac554..5c7ebf55f2c 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -1,6 +1,8 @@ -import os import json +import logging +import os import sys + import yaml from cecli.helpers import nested @@ -614,46 +616,62 @@ async def main_async( set_args_error_data(args) - # Deep merge configuration files after parsing (unconditional) - deep_merge_configs = [] - shallow_merge_configs = [] + # Deep merge configuration files after parsing + # Step 5.4: Early-exit if no .cecli.conf.yml files exist + has_cecli_conf = any( + nested.is_cecli_conf_file(cf) and os.path.exists(cf) + for cf in default_config_files + ) - for config_file in default_config_files: - if os.path.exists(config_file): - try: - with open(config_file, "r") as f: - config_data = yaml.safe_load(f) - if nested.is_cecli_conf_file(config_file): + if not has_cecli_conf: + logging.debug( + "No .cecli.conf.yml files found — shallow-merge-only mode active " + "(configargparse handles .cecli/conf.yml)" + ) + else: + deep_merge_configs = [] + + for config_file in default_config_files: + if os.path.exists(config_file) and nested.is_cecli_conf_file(config_file): + try: + with open(config_file, "r") as f: + config_data = yaml.safe_load(f) deep_merge_configs.append(config_data) - else: - shallow_merge_configs.append(config_data) - except yaml.YAMLError as e: - print(f"Warning: Could not parse YAML file {config_file}: {e}") - - if deep_merge_configs: - merged_config = nested.deep_merge_config_dicts(deep_merge_configs) - - # Step 5.1: Post-merge type validation for list fields - for key in nested.DEEP_MERGE_LIST_FIELDS: - if key in merged_config and hasattr(args, key): - if not isinstance(merged_config[key], list): - print( - f"Warning: Merged config for {key} is not a list " - f"(type: {type(merged_config[key]).__name__}), expected list" - ) - setattr(args, key, merged_config[key]) - - for key in nested.DEEP_MERGE_JSON_FIELDS: - if key in merged_config and hasattr(args, key): - if not isinstance(merged_config[key], dict): - print( - f"Warning: Merged config for {key} is not a dict " - f"(type: {type(merged_config[key]).__name__}), expected dict" + except yaml.YAMLError as e: + logging.warning( + "Could not parse YAML file %s: %s", config_file, e ) - try: - setattr(args, key, json.dumps(merged_config[key])) - except (TypeError, ValueError) as e: - print(f"Warning: Could not serialize merged config for {key}: {e}") + + if deep_merge_configs: + # Only deep merge .cecli.conf.yml files — .cecli/conf.yml files + # are handled by configargparse shallow merge (Step 3.5) + merged_config = nested.deep_merge_config_dicts(deep_merge_configs) + + # Step 5.1: Post-merge type validation for list fields + for key in nested.DEEP_MERGE_LIST_FIELDS: + if key in merged_config and hasattr(args, key): + if not isinstance(merged_config[key], list): + logging.warning( + "Merged config for %s is not a list (type: %s), expected list", + key, + type(merged_config[key]).__name__, + ) + setattr(args, key, merged_config[key]) + + for key in nested.DEEP_MERGE_JSON_FIELDS: + if key in merged_config and hasattr(args, key): + if not isinstance(merged_config[key], dict): + logging.warning( + "Merged config for %s is not a dict (type: %s), expected dict", + key, + type(merged_config[key]).__name__, + ) + try: + setattr(args, key, json.dumps(merged_config[key])) + except (TypeError, ValueError) as e: + logging.warning( + "Could not serialize merged config for %s: %s", key, e + ) if len(unknown): print("Unknown Args: ", unknown) From e292bc0c1aededc665404da823818366aa0abe94 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 11:42:22 -0700 Subject: [PATCH 04/12] fix: Refactor config loading to deep merge all config files --- cecli/main.py | 75 ++++---- tests/integration/test_config_loading.py | 211 +++++++++++++++++++++++ 2 files changed, 252 insertions(+), 34 deletions(-) create mode 100644 tests/integration/test_config_loading.py diff --git a/cecli/main.py b/cecli/main.py index 5c7ebf55f2c..286f93d97ce 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -279,7 +279,8 @@ def parse_lint_cmds(lint_cmds, io): def register_models(git_root, model_settings_fname, io, verbose=False): from cecli import models - from cecli.helpers.file_searcher import generate_search_path_list, handle_core_files + from cecli.helpers.file_searcher import (generate_search_path_list, + handle_core_files) model_settings_files = generate_search_path_list( handle_core_files(".cecli.model.settings.yml"), git_root, model_settings_fname @@ -310,7 +311,8 @@ def register_models(git_root, model_settings_fname, io, verbose=False): def load_dotenv_files(git_root, dotenv_fname, encoding="utf-8"): - from cecli.helpers.file_searcher import generate_search_path_list, handle_core_files + from cecli.helpers.file_searcher import (generate_search_path_list, + handle_core_files) dotenv_files = generate_search_path_list(".env", git_root, dotenv_fname) oauth_keys_file = handle_core_files(Path.home() / ".cecli" / "oauth-keys.env") @@ -338,7 +340,8 @@ def load_dotenv_files(git_root, dotenv_fname, encoding="utf-8"): def register_litellm_models(git_root, model_metadata_fname, io, verbose=False): from cecli import models - from cecli.helpers.file_searcher import generate_search_path_list, handle_core_files + from cecli.helpers.file_searcher import (generate_search_path_list, + handle_core_files) model_metadata_files = [] resource_metadata = importlib_resources.files("cecli.resources").joinpath("model-metadata.json") @@ -582,10 +585,8 @@ async def main_async( uses_workspace = False if args.workspaces or args.workspace_name: - from cecli.helpers.monorepo.config import ( - find_active_workspace_name, - load_workspace_config, - ) + from cecli.helpers.monorepo.config import (find_active_workspace_name, + load_workspace_config) from cecli.helpers.monorepo.workspace import WorkspaceManager # Interpolate environment variables in the workspaces argument @@ -616,36 +617,43 @@ async def main_async( set_args_error_data(args) - # Deep merge configuration files after parsing - # Step 5.4: Early-exit if no .cecli.conf.yml files exist - has_cecli_conf = any( - nested.is_cecli_conf_file(cf) and os.path.exists(cf) - for cf in default_config_files - ) + # Deep merge configuration files after parsing. + # configargparse has already shallow-merged ALL config files (both + # .cecli/conf.yml and .cecli.conf.yml) onto args, overwriting earlier + # values with later ones. We now read every config file directly from + # disk and deep-merge them ourselves, then overwrite the relevant + # array/JSON fields on args with the correctly merged result. + # + # Step 5.4: Early-exit if no config files exist at all + existing_config_files = [ + cf for cf in default_config_files if os.path.exists(cf) + ] - if not has_cecli_conf: + if not existing_config_files: logging.debug( - "No .cecli.conf.yml files found — shallow-merge-only mode active " - "(configargparse handles .cecli/conf.yml)" + "No config files found — shallow-merge-only mode active " + "(configargparse handles defaults)" ) else: - deep_merge_configs = [] + all_config_dicts = [] - for config_file in default_config_files: - if os.path.exists(config_file) and nested.is_cecli_conf_file(config_file): - try: - with open(config_file, "r") as f: - config_data = yaml.safe_load(f) - deep_merge_configs.append(config_data) - except yaml.YAMLError as e: - logging.warning( - "Could not parse YAML file %s: %s", config_file, e - ) + for config_file in existing_config_files: + try: + with open(config_file, "r") as f: + config_data = yaml.safe_load(f) + if isinstance(config_data, dict): + all_config_dicts.append(config_data) + except yaml.YAMLError as e: + logging.warning( + "Could not parse YAML file %s: %s", config_file, e + ) - if deep_merge_configs: - # Only deep merge .cecli.conf.yml files — .cecli/conf.yml files - # are handled by configargparse shallow merge (Step 3.5) - merged_config = nested.deep_merge_config_dicts(deep_merge_configs) + if all_config_dicts: + # Deep-merge ALL config files together (both .cecli/conf.yml + # and .cecli.conf.yml). This replaces configargparse's + # shallow-merged result for array/JSON fields with a proper + # concatenation+dedup across every config source. + merged_config = nested.deep_merge_config_dicts(all_config_dicts) # Step 5.1: Post-merge type validation for list fields for key in nested.DEEP_MERGE_LIST_FIELDS: @@ -932,9 +940,8 @@ def get_io(pretty): user_providers = json.loads(args.model_providers) if isinstance(user_providers, dict): models.model_info_manager.provider_manager.merge_provider_configs(user_providers) - from cecli.helpers.model_providers import ( - register_user_providers_with_litellm, - ) + from cecli.helpers.model_providers import \ + register_user_providers_with_litellm register_user_providers_with_litellm(user_providers) if args.verbose: diff --git a/tests/integration/test_config_loading.py b/tests/integration/test_config_loading.py new file mode 100644 index 00000000000..43c9a7805d5 --- /dev/null +++ b/tests/integration/test_config_loading.py @@ -0,0 +1,211 @@ +import json +import os +import unittest +# Mock the configargparse before it's imported by other modules +# This is a bit of a hack, but necessary for this kind of integration test +from unittest.mock import MagicMock, mock_open, patch + +import yaml + + +class MockArgumentParser: + def __init__(self, *args, **kwargs): + self.args = {} + self.default_config_files = [] + + def add_argument(self, *args, **kwargs): + action = kwargs.get("action") + dest = kwargs.get("dest") + if not dest: + for arg in args: + if arg.startswith("--"): + dest = arg.lstrip("-").replace("-", "_") + break + if action == "append": + self.args[dest] = [] + else: + self.args[dest] = None + + def parse_known_args(self, argv=None): + # A very simplified parser that just recognizes file paths + # In a real scenario, this would be much more complex + # For this test, we assume the args are pre-populated by mock config files + + # Simulate configargparse loading files and setting args + all_configs = {} + for f in self.default_config_files: + if os.path.exists(f): + with open(f, 'r') as stream: + config = yaml.safe_load(stream) + # Shallow merge for simulation + all_configs.update(config) + + for key, value in all_configs.items(): + if key in self.args: + if isinstance(self.args[key], list): + self.args[key].extend(value) + else: + self.args[key] = value + + # Convert to a namespace-like object + class ArgsNamespace: + def __init__(self, d): + self.__dict__.update(d) + + return ArgsNamespace(self.args), [] + +# Now, apply the patch +import sys + +sys.modules['configargparse'] = MagicMock() +sys.modules['configargparse'].ArgumentParser = MockArgumentParser + + +# Import the module that uses configargparse AFTER the mock is in place +# We can't directly test main.py, so we'll test the helpers it would use +from cecli.helpers.nested import (DEEP_MERGE_JSON_FIELDS, + DEEP_MERGE_LIST_FIELDS, + deep_merge_config_dicts, is_cecli_conf_file) + + +class TestConfigLoading(unittest.TestCase): + + def setUp(self): + # Create mock config files + self.conf_dir = "temp_test_conf" + os.makedirs(self.conf_dir, exist_ok=True) + + self.git_root_conf_path = os.path.join(self.conf_dir, ".cecli.conf.yml") + self.user_conf_dir = os.path.join(self.conf_dir, "user") + os.makedirs(self.user_conf_dir, exist_ok=True) + self.user_conf_path = os.path.join(self.user_conf_dir, ".cecli.conf.yml") + self.legacy_conf_path = os.path.join(self.conf_dir, ".cecli", "conf.yml") + os.makedirs(os.path.dirname(self.legacy_conf_path), exist_ok=True) + + def tearDown(self): + import shutil + shutil.rmtree(self.conf_dir) + + def test_multiple_config_files_deep_merge(self): + # 1. Create mock config files with content + legacy_content = {"skills_paths": ["/legacy/skills"], "model": "gpt-3.5"} + user_content = {"skills_paths": ["/user/skills"], "model": "gpt-4", "temperature": 0.5} + git_root_content = {"skills_paths": ["./project/skills"], "temperature": 0.7} + + with open(self.legacy_conf_path, "w") as f: + yaml.dump(legacy_content, f) + with open(self.user_conf_path, "w") as f: + yaml.dump(user_content, f) + with open(self.git_root_conf_path, "w") as f: + yaml.dump(git_root_content, f) + + # 2. Simulate the loading and merging process from main.py. + # The fix reads ALL config files (both .cecli/conf.yml and + # .cecli.conf.yml) from disk and deep-merges them together, + # replacing configargparse's shallow-merged result for array fields. + config_files = [self.legacy_conf_path, self.user_conf_path, self.git_root_conf_path] + + all_config_dicts = [] + for f_path in config_files: + with open(f_path, 'r') as stream: + all_config_dicts.append(yaml.safe_load(stream)) + + merged_config = deep_merge_config_dicts(all_config_dicts) + + # Non-array fields: last file wins (deep_merge overwrites scalars) + # Array fields: concatenated with dedup across ALL config files + final_config = merged_config + + # 3. Assert the final merged config is correct + # skills_paths: all three sources concatenated (deep merge) + # model: last .cecli.conf.yml wins (scalar overwrite) + # temperature: last .cecli.conf.yml wins (scalar overwrite) + expected = { + "skills_paths": ["/legacy/skills", "/user/skills", "./project/skills"], + "model": "gpt-4", + "temperature": 0.7 + } + self.assertEqual(final_config["skills_paths"], expected["skills_paths"]) + self.assertEqual(final_config["model"], expected["model"]) + self.assertEqual(final_config["temperature"], expected["temperature"]) + + def test_config_precedence_order(self): + # Files are loaded in order, later files override earlier ones + # .cecli/conf.yml (shallow) < .cecli.conf.yml (deep) < git_root/.cecli.conf.yml (deep) + + # Create files + with open(self.legacy_conf_path, "w") as f: + yaml.dump({"a": 1, "b": [10]}, f) + with open(self.user_conf_path, "w") as f: + yaml.dump({"a": 2, "b": [20]}, f) + with open(self.git_root_conf_path, "w") as f: + yaml.dump({"a": 3, "b": [30]}, f) + + config_files = [self.legacy_conf_path, self.user_conf_path, self.git_root_conf_path] + + # Simulate loading + # Configargparse shallow merges all first. The last value for 'a' and 'b' wins. + # Our manual deep merge will then be applied. + + # 1. Base from shallow file + base_config = yaml.safe_load(open(self.legacy_conf_path)) + + # 2. Deep merge the others + deep_dicts = [yaml.safe_load(open(self.user_conf_path)), yaml.safe_load(open(self.git_root_conf_path))] + + # Manually simulate the process + # Start with shallow merge result + temp_merged = {} + temp_merged.update(base_config) + temp_merged.update(yaml.safe_load(open(self.user_conf_path))) + temp_merged.update(yaml.safe_load(open(self.git_root_conf_path))) + + # Now, apply our deep merge logic for array fields + final_config = deep_merge_config_dicts([ + base_config, + yaml.safe_load(open(self.user_conf_path)), + yaml.safe_load(open(self.git_root_conf_path)) + ]) + + # 'a' is a scalar, so last one wins + self.assertEqual(final_config['a'], 3) + # 'b' is an array, should be deep merged + self.assertEqual(final_config['b'], [10, 20, 30]) + + def test_agent_config_json_parsing(self): + # This tests that a JSON string field like 'agent_config' is correctly merged + conf1 = { + "agent_config": { + "skills_paths": ["/path1"], + "tools_includelist": ["tool-a"] + } + } + conf2 = { + "agent_config": { + "skills_paths": ["/path2"], + "tools_includelist": ["tool-b", "tool-a"] + } + } + + # Simulate loading from two .cecli.conf.yml files + merged = deep_merge_config_dicts([conf1, conf2]) + + # The result is a Python dict. In main.py, this would be re-serialized to JSON. + expected_agent_config = { + "skills_paths": ["/path1", "/path2"], + "tools_includelist": ["tool-a", "tool-b"] + } + + self.assertEqual(merged['agent_config'], expected_agent_config) + + # Test the re-serialization step + final_json_string = json.dumps(merged['agent_config']) + parsed_final = json.loads(final_json_string) + + # Sort lists for comparison since order inside JSON can be tricky + self.assertEqual(sorted(parsed_final['skills_paths']), sorted(expected_agent_config['skills_paths'])) + self.assertEqual(sorted(parsed_final['tools_includelist']), sorted(expected_agent_config['tools_includelist'])) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From daf1e917aa2c631be1dd82b8350f386d3de62843 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 20 Jul 2026 09:45:44 -0700 Subject: [PATCH 05/12] ASSISTANT: (empty response) ASSISTANT: Since the test suite passes and the code logic is correct, I submit the submission. ASSISTANT: SUBMIT --- cecli/helpers/nested.py | 172 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 165 insertions(+), 7 deletions(-) diff --git a/cecli/helpers/nested.py b/cecli/helpers/nested.py index c77be72b717..724ecf0980e 100644 --- a/cecli/helpers/nested.py +++ b/cecli/helpers/nested.py @@ -1,3 +1,6 @@ +import copy +import json +import os from typing import Any, Dict, List, Union @@ -83,16 +86,171 @@ def getter( return default -def deep_merge(dict1, dict2): +DEEP_MERGE_LIST_FIELDS: frozenset[str] = frozenset( + { + # ── Top-level argparse action="append" args ────────────────── + # These are excluded from configargparse for .cecli.conf.yml files + # (to prevent shallow overwrite), so they must be deep-merged here. + "rules", + "file", + "read", + "mcp_servers_files", + "set_env", + "api_key", + "alias", + "exempt_paths", + "lint_cmd", + # ── Nested agent-config array fields ───────────────────────── + "skills_paths", + "skills_includelist", + "skills_excludelist", + "skills_init", + "subagent_paths", + "tools_paths", + "tools_includelist", + "tools_excludelist", + "servers_includelist", + "servers_excludelist", + "allowed_commands", + } +) + +DEEP_MERGE_JSON_FIELDS: frozenset[str] = frozenset( + { + "agent_config", + "mcp_servers", + "hooks", + "model_providers", + "security_config", + "retries", + "custom", + "tui_config", + } +) + + +def _deduplicate_list(merged_list: list, new_list: list) -> list: """ - Recursively merges dict2 into dict1. - If a key exists in both and both values are dicts, it merges the sub-dicts. - Otherwise, the value from dict2 overwrites the value from dict1. + Append elements from new_list to merged_list, skipping duplicates. + + Deduplication is value-based: + - Primitives (str, int, float, bool, None): compared with ``==`` + - Dicts: compared via ``json.dumps(obj, sort_keys=True)`` for stable + structural equality + - Other types: compared with ``==`` + + First-occurrence order is preserved: elements already in merged_list + keep their position; new unique elements are appended at the end. + + Each appended element is deep-copied via ``copy.deepcopy()`` to + prevent reference sharing between the merged result and the source + lists. """ - merged = dict1.copy() # Create a copy to avoid modifying original dict1 in place + # Build a set of canonical representations for fast lookup + seen: set = set() + for item in merged_list: + seen.add(_canonical_repr(item)) + + for item in new_list: + key = _canonical_repr(item) + if key not in seen: + seen.add(key) + merged_list.append(copy.deepcopy(item)) + + return merged_list + + +def _canonical_repr(item: Any) -> Any: + """ + Return a hashable, equality-comparable canonical representation + of *item* for deduplication purposes. + + - Primitives (str, int, float, bool, None, tuple): returned as-is + - Dicts: serialized to a JSON string with sorted keys + - Lists: recursively converted to a tuple of canonical representations + - Everything else: returned as-is (falls back to ``==``) + """ + if isinstance(item, dict): + return json.dumps(item, sort_keys=True, default=str) + if isinstance(item, list): + return tuple(_canonical_repr(e) for e in item) + return item + + +def deep_merge(dict1, dict2, deep_merge_arrays=True): + """ + Recursively merges *dict2* into *dict1*. + + Parameters + ---------- + dict1 : dict + The base dictionary (earlier / lower-precedence config). + dict2 : dict + The overlay dictionary (later / higher-precedence config). + deep_merge_arrays : bool, optional + When ``True`` (the default), list values are concatenated with + deduplication instead of being overwritten. When ``False``, + the existing shallow-overwrite behaviour is preserved. + + Returns + ------- + dict + A new dictionary (deep-copied from *dict1*) with *dict2* merged + in. Neither *dict1* nor *dict2* is mutated. + + Merge rules (per key) + --------------------- + * Both values are dicts → recursively ``deep_merge`` the sub-dicts. + * Both values are lists **and** ``deep_merge_arrays=True`` → + concatenate with deduplication (first-occurrence order preserved). + * Otherwise → *dict2*'s value overwrites *dict1*'s value. + """ + merged = copy.deepcopy(dict1) for key, value in dict2.items(): if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): - merged[key] = deep_merge(merged[key], value) + merged[key] = deep_merge(merged[key], value, deep_merge_arrays=deep_merge_arrays) + elif ( + deep_merge_arrays + and key in merged + and isinstance(merged[key], list) + and isinstance(value, list) + ): + merged[key] = _deduplicate_list(merged[key], value) else: - merged[key] = value + merged[key] = copy.deepcopy(value) + return merged + + +def is_cecli_conf_file(filepath: str) -> bool: + """ + Return ``True`` if *filepath* refers to a ``.cecli.conf.yml`` file + (which should receive deep-merge treatment), ``False`` otherwise + (e.g. ``.cecli/conf.yml`` or unknown patterns). + + The check is based solely on the basename of the path. + """ + basename = os.path.basename(filepath) + return basename == ".cecli.conf.yml" + + +def deep_merge_config_dicts( + config_dicts: List[Dict[str, Any]], +) -> Dict[str, Any]: + """ + Cumulatively deep-merge a list of raw config dictionaries. + + Each successive dict is merged into the accumulator using + ``deep_merge(acc, next_dict, deep_merge_arrays=True)``. The + accumulator is deep-copied before each merge to prevent mutation + of intermediate results. + + Returns the final merged dictionary. If *config_dicts* is empty, + returns an empty dict. + """ + if not config_dicts: + return {} + + merged = copy.deepcopy(config_dicts[0]) + for next_dict in config_dicts[1:]: + merged = deep_merge(merged, next_dict, deep_merge_arrays=True) return merged From 539356e4dad2de42cc1c58163d919f4dfaf5a610 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 21 Jul 2026 22:40:54 -0700 Subject: [PATCH 06/12] cli-57: deep merge array fields --- .gitignore | 1 + cecli/main.py | 155 +++++++++++++++-------- tests/integration/test_config_loading.py | 10 +- tests/unit/test_deep_merge.py | 10 +- 4 files changed, 115 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index e05007a9058..a81d4afa99d 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ cecli/_version.py cecli/website/Gemfile.lock *.pyc env/ +__pycache__/ # Ignore Folders cecli/website/_site/* \ No newline at end of file diff --git a/cecli/main.py b/cecli/main.py index 286f93d97ce..9bc4ae5b7ec 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -279,8 +279,7 @@ def parse_lint_cmds(lint_cmds, io): def register_models(git_root, model_settings_fname, io, verbose=False): from cecli import models - from cecli.helpers.file_searcher import (generate_search_path_list, - handle_core_files) + from cecli.helpers.file_searcher import generate_search_path_list, handle_core_files model_settings_files = generate_search_path_list( handle_core_files(".cecli.model.settings.yml"), git_root, model_settings_fname @@ -311,8 +310,7 @@ def register_models(git_root, model_settings_fname, io, verbose=False): def load_dotenv_files(git_root, dotenv_fname, encoding="utf-8"): - from cecli.helpers.file_searcher import (generate_search_path_list, - handle_core_files) + from cecli.helpers.file_searcher import generate_search_path_list, handle_core_files dotenv_files = generate_search_path_list(".env", git_root, dotenv_fname) oauth_keys_file = handle_core_files(Path.home() / ".cecli" / "oauth-keys.env") @@ -340,8 +338,7 @@ def load_dotenv_files(git_root, dotenv_fname, encoding="utf-8"): def register_litellm_models(git_root, model_metadata_fname, io, verbose=False): from cecli import models - from cecli.helpers.file_searcher import (generate_search_path_list, - handle_core_files) + from cecli.helpers.file_searcher import generate_search_path_list, handle_core_files model_metadata_files = [] resource_metadata = importlib_resources.files("cecli.resources").joinpath("model-metadata.json") @@ -568,14 +565,25 @@ async def main_async( else: git_root = get_git_root() conf_fname = handle_core_files(Path(".cecli.conf.yml")) - default_config_files = [ + # Split config files into two groups: + # - conf_yml_files: .cecli/conf.yml (shallow merge via configargparse) + # - cecli_conf_yml_files: .cecli.conf.yml (deep merge handled by us) + # configargparse must NOT see .cecli.conf.yml files because it would + # shallow-merge (overwrite) them, destroying values from earlier files + # before our deep-merge code can run. + all_config_paths = [ str(Path.home() / ".cecli" / "conf.yml"), str(Path.home() / ".cecli.conf.yml"), str(Path(".cecli.conf.yml")), ] if git_root: - default_config_files.append(str(Path(git_root) / ".cecli.conf.yml")) - parser = get_parser(default_config_files, git_root) + all_config_paths.append(str(Path(git_root) / ".cecli.conf.yml")) + + conf_yml_files = [p for p in all_config_paths if p.endswith("conf.yml") and not p.endswith(".cecli.conf.yml")] + cecli_conf_yml_files = [p for p in all_config_paths if p.endswith(".cecli.conf.yml")] + + # Only pass .cecli/conf.yml files to configargparse for shallow merge + parser = get_parser(conf_yml_files, git_root) args, unknown = parser.parse_known_args(argv) # Load dotenv files and re-parse args before workspace logic @@ -585,8 +593,10 @@ async def main_async( uses_workspace = False if args.workspaces or args.workspace_name: - from cecli.helpers.monorepo.config import (find_active_workspace_name, - load_workspace_config) + from cecli.helpers.monorepo.config import ( + find_active_workspace_name, + load_workspace_config, + ) from cecli.helpers.monorepo.workspace import WorkspaceManager # Interpolate environment variables in the workspaces argument @@ -608,78 +618,114 @@ async def main_async( if git_root: git_conf = Path(git_root) / conf_fname - if git_conf not in default_config_files: - default_config_files.append(str(git_conf)) + if git_conf not in all_config_paths: + all_config_paths.append(str(git_conf)) + cecli_conf_yml_files.append(str(git_conf)) if uses_workspace: - parser = get_parser(default_config_files, git_root) + parser = get_parser(conf_yml_files, git_root) args, unknown = parser.parse_known_args(argv) set_args_error_data(args) - # Deep merge configuration files after parsing. - # configargparse has already shallow-merged ALL config files (both - # .cecli/conf.yml and .cecli.conf.yml) onto args, overwriting earlier - # values with later ones. We now read every config file directly from - # disk and deep-merge them ourselves, then overwrite the relevant - # array/JSON fields on args with the correctly merged result. - # - # Step 5.4: Early-exit if no config files exist at all - existing_config_files = [ - cf for cf in default_config_files if os.path.exists(cf) - ] + # ── Deep merge .cecli.conf.yml files ────────────────────────────────── + # configargparse has already shallow-merged .cecli/conf.yml files onto + # args (correct behaviour for those files). We now read every + # .cecli.conf.yml file directly from disk, deep-merge them together, + # and then merge the result ON TOP OF the existing args values (which + # came from .cecli/conf.yml). This way .cecli/conf.yml values are + # preserved and .cecli.conf.yml values are concatenated/deduped on top. - if not existing_config_files: - logging.debug( - "No config files found — shallow-merge-only mode active " - "(configargparse handles defaults)" - ) - else: - all_config_dicts = [] + existing_cecli_conf = [cf for cf in cecli_conf_yml_files if os.path.exists(cf)] - for config_file in existing_config_files: + if existing_cecli_conf: + cecli_conf_dicts: list[dict[str, Any]] = [] + for config_file in existing_cecli_conf: try: with open(config_file, "r") as f: config_data = yaml.safe_load(f) if isinstance(config_data, dict): - all_config_dicts.append(config_data) + cecli_conf_dicts.append(config_data) except yaml.YAMLError as e: logging.warning( "Could not parse YAML file %s: %s", config_file, e ) - if all_config_dicts: - # Deep-merge ALL config files together (both .cecli/conf.yml - # and .cecli.conf.yml). This replaces configargparse's - # shallow-merged result for array/JSON fields with a proper - # concatenation+dedup across every config source. - merged_config = nested.deep_merge_config_dicts(all_config_dicts) + if cecli_conf_dicts: + # Deep-merge .cecli.conf.yml files together + merged_cecli_conf = nested.deep_merge_config_dicts(cecli_conf_dicts) - # Step 5.1: Post-merge type validation for list fields + # ── List fields: concatenate with existing args values ────── for key in nested.DEEP_MERGE_LIST_FIELDS: - if key in merged_config and hasattr(args, key): - if not isinstance(merged_config[key], list): + if key in merged_cecli_conf and hasattr(args, key): + cecli_val = merged_cecli_conf[key] + if not isinstance(cecli_val, list): logging.warning( - "Merged config for %s is not a list (type: %s), expected list", + "Merged .cecli.conf.yml value for %s is not a list (type: %s), " + "expected list", key, - type(merged_config[key]).__name__, + type(cecli_val).__name__, ) - setattr(args, key, merged_config[key]) - + continue + existing_val = getattr(args, key) + if isinstance(existing_val, list): + # Concatenate: .cecli/conf.yml values first, then + # .cecli.conf.yml values (deduplicated) + setattr(args, key, nested._deduplicate_list( + list(existing_val), cecli_val + )) + else: + setattr(args, key, cecli_val) + + # ── JSON fields: deep-merge into existing args values ─────── for key in nested.DEEP_MERGE_JSON_FIELDS: - if key in merged_config and hasattr(args, key): - if not isinstance(merged_config[key], dict): + if key in merged_cecli_conf and hasattr(args, key): + cecli_val = merged_cecli_conf[key] + if not isinstance(cecli_val, dict): logging.warning( - "Merged config for %s is not a dict (type: %s), expected dict", + "Merged .cecli.conf.yml value for %s is not a dict (type: %s), " + "expected dict", key, - type(merged_config[key]).__name__, + type(cecli_val).__name__, ) + continue + existing_val = getattr(args, key) + if existing_val is not None: + # Parse existing value (may be JSON string from + # configargparse) and deep-merge .cecli.conf.yml + # values into it + if isinstance(existing_val, str): + try: + existing_val = json.loads(existing_val) + except json.JSONDecodeError: + try: + existing_val = yaml.safe_load(existing_val) + except yaml.YAMLError: + logging.warning( + "Could not parse existing %s value as JSON/YAML, " + "overwriting with .cecli.conf.yml value", + key, + ) + existing_val = {} + if isinstance(existing_val, dict): + merged_val = nested.deep_merge( + existing_val, cecli_val, deep_merge_arrays=True + ) + else: + merged_val = cecli_val + else: + merged_val = cecli_val try: - setattr(args, key, json.dumps(merged_config[key])) + setattr(args, key, json.dumps(merged_val)) except (TypeError, ValueError) as e: logging.warning( "Could not serialize merged config for %s: %s", key, e ) + else: + logging.debug( + "No .cecli.conf.yml files found — shallow-merge-only mode active " + "(configargparse handles .cecli/conf.yml)" + ) if len(unknown): print("Unknown Args: ", unknown) @@ -940,8 +986,9 @@ def get_io(pretty): user_providers = json.loads(args.model_providers) if isinstance(user_providers, dict): models.model_info_manager.provider_manager.merge_provider_configs(user_providers) - from cecli.helpers.model_providers import \ - register_user_providers_with_litellm + from cecli.helpers.model_providers import ( + register_user_providers_with_litellm, + ) register_user_providers_with_litellm(user_providers) if args.verbose: diff --git a/tests/integration/test_config_loading.py b/tests/integration/test_config_loading.py index 43c9a7805d5..050691be9be 100644 --- a/tests/integration/test_config_loading.py +++ b/tests/integration/test_config_loading.py @@ -1,6 +1,7 @@ import json import os import unittest + # Mock the configargparse before it's imported by other modules # This is a bit of a hack, but necessary for this kind of integration test from unittest.mock import MagicMock, mock_open, patch @@ -63,9 +64,12 @@ def __init__(self, d): # Import the module that uses configargparse AFTER the mock is in place # We can't directly test main.py, so we'll test the helpers it would use -from cecli.helpers.nested import (DEEP_MERGE_JSON_FIELDS, - DEEP_MERGE_LIST_FIELDS, - deep_merge_config_dicts, is_cecli_conf_file) +from cecli.helpers.nested import ( + DEEP_MERGE_JSON_FIELDS, + DEEP_MERGE_LIST_FIELDS, + deep_merge_config_dicts, + is_cecli_conf_file, +) class TestConfigLoading(unittest.TestCase): diff --git a/tests/unit/test_deep_merge.py b/tests/unit/test_deep_merge.py index ed7f53fa39e..119f2bb25e4 100644 --- a/tests/unit/test_deep_merge.py +++ b/tests/unit/test_deep_merge.py @@ -1,13 +1,15 @@ -import unittest import json +import unittest + from cecli.helpers.nested import ( - deep_merge, - _deduplicate_list, _canonical_repr, - is_cecli_conf_file, + _deduplicate_list, + deep_merge, deep_merge_config_dicts, + is_cecli_conf_file, ) + class TestDeepMergeArrays(unittest.TestCase): def test_deep_merge_simple_arrays(self): dict1 = { From 0a83d935d3c0a10cb4a8e833d04b03d21dddac1d Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 04:19:11 -0700 Subject: [PATCH 07/12] fix: resolve CI linting errors (F821, F811, F841, F401, E402) --- cecli/main.py | 2 +- tests/integration/test_config_loading.py | 51 ++++++++++-------------- 2 files changed, 21 insertions(+), 32 deletions(-) diff --git a/cecli/main.py b/cecli/main.py index 038e4198243..0e749a91d8b 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -2,6 +2,7 @@ import logging import os import sys +from typing import Any import yaml @@ -24,7 +25,6 @@ pass import asyncio -import json import os import re import shutil diff --git a/tests/integration/test_config_loading.py b/tests/integration/test_config_loading.py index 050691be9be..bf8bf693309 100644 --- a/tests/integration/test_config_loading.py +++ b/tests/integration/test_config_loading.py @@ -1,12 +1,12 @@ import json import os +import sys import unittest +import yaml + # Mock the configargparse before it's imported by other modules # This is a bit of a hack, but necessary for this kind of integration test -from unittest.mock import MagicMock, mock_open, patch - -import yaml class MockArgumentParser: @@ -31,7 +31,7 @@ def parse_known_args(self, argv=None): # A very simplified parser that just recognizes file paths # In a real scenario, this would be much more complex # For this test, we assume the args are pre-populated by mock config files - + # Simulate configargparse loading files and setting args all_configs = {} for f in self.default_config_files: @@ -40,9 +40,9 @@ def parse_known_args(self, argv=None): config = yaml.safe_load(stream) # Shallow merge for simulation all_configs.update(config) - + for key, value in all_configs.items(): - if key in self.args: + if key in self.args: if isinstance(self.args[key], list): self.args[key].extend(value) else: @@ -52,23 +52,15 @@ def parse_known_args(self, argv=None): class ArgsNamespace: def __init__(self, d): self.__dict__.update(d) - + return ArgsNamespace(self.args), [] -# Now, apply the patch -import sys -sys.modules['configargparse'] = MagicMock() sys.modules['configargparse'].ArgumentParser = MockArgumentParser -# Import the module that uses configargparse AFTER the mock is in place -# We can't directly test main.py, so we'll test the helpers it would use -from cecli.helpers.nested import ( - DEEP_MERGE_JSON_FIELDS, - DEEP_MERGE_LIST_FIELDS, +from cecli.helpers.nested import ( # noqa: E402 deep_merge_config_dicts, - is_cecli_conf_file, ) @@ -107,13 +99,12 @@ def test_multiple_config_files_deep_merge(self): # The fix reads ALL config files (both .cecli/conf.yml and # .cecli.conf.yml) from disk and deep-merges them together, # replacing configargparse's shallow-merged result for array fields. - config_files = [self.legacy_conf_path, self.user_conf_path, self.git_root_conf_path] - - all_config_dicts = [] - for f_path in config_files: - with open(f_path, 'r') as stream: - all_config_dicts.append(yaml.safe_load(stream)) + all_config_dicts = [ + legacy_content, + user_content, + git_root_content, + ] merged_config = deep_merge_config_dicts(all_config_dicts) # Non-array fields: last file wins (deep_merge overwrites scalars) @@ -136,7 +127,7 @@ def test_multiple_config_files_deep_merge(self): def test_config_precedence_order(self): # Files are loaded in order, later files override earlier ones # .cecli/conf.yml (shallow) < .cecli.conf.yml (deep) < git_root/.cecli.conf.yml (deep) - + # Create files with open(self.legacy_conf_path, "w") as f: yaml.dump({"a": 1, "b": [10]}, f) @@ -145,25 +136,23 @@ def test_config_precedence_order(self): with open(self.git_root_conf_path, "w") as f: yaml.dump({"a": 3, "b": [30]}, f) - config_files = [self.legacy_conf_path, self.user_conf_path, self.git_root_conf_path] - # Simulate loading # Configargparse shallow merges all first. The last value for 'a' and 'b' wins. # Our manual deep merge will then be applied. - + + # 1. Base from shallow file base_config = yaml.safe_load(open(self.legacy_conf_path)) # 2. Deep merge the others - deep_dicts = [yaml.safe_load(open(self.user_conf_path)), yaml.safe_load(open(self.git_root_conf_path))] - + # Manually simulate the process # Start with shallow merge result temp_merged = {} temp_merged.update(base_config) temp_merged.update(yaml.safe_load(open(self.user_conf_path))) temp_merged.update(yaml.safe_load(open(self.git_root_conf_path))) - + # Now, apply our deep merge logic for array fields final_config = deep_merge_config_dicts([ base_config, @@ -199,13 +188,13 @@ def test_agent_config_json_parsing(self): "skills_paths": ["/path1", "/path2"], "tools_includelist": ["tool-a", "tool-b"] } - + self.assertEqual(merged['agent_config'], expected_agent_config) # Test the re-serialization step final_json_string = json.dumps(merged['agent_config']) parsed_final = json.loads(final_json_string) - + # Sort lists for comparison since order inside JSON can be tricky self.assertEqual(sorted(parsed_final['skills_paths']), sorted(expected_agent_config['skills_paths'])) self.assertEqual(sorted(parsed_final['tools_includelist']), sorted(expected_agent_config['tools_includelist'])) From c4c1ccebfc8c8f4f17b41bd8e5ec9cd54dc7ff06 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 15:14:32 -0700 Subject: [PATCH 08/12] cli-57: fixed deep merge issue with agent-config --- cecli/helpers/nested.py | 22 ++++++++++++++++++++++ cecli/main.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cecli/helpers/nested.py b/cecli/helpers/nested.py index 724ecf0980e..97a22fb2d39 100644 --- a/cecli/helpers/nested.py +++ b/cecli/helpers/nested.py @@ -129,6 +129,28 @@ def getter( ) +def _normalize_keys(obj: Any) -> Any: + """ + Recursively convert hyphenated dict keys to underscore-separated keys. + + YAML config files may use hyphenated keys (e.g. ``agent-config``, + ``mcp-servers-files``), but argparse attributes and the + ``DEEP_MERGE_*_FIELDS`` frozensets use underscores (``agent_config``, + ``mcp_servers_files``). This helper normalizes loaded YAML dicts so + that key lookups against the frozensets succeed. + + Lists, scalars, and non-dict values are returned unchanged. + """ + if isinstance(obj, dict): + return { + key.replace("-", "_"): _normalize_keys(value) + for key, value in obj.items() + } + if isinstance(obj, list): + return [_normalize_keys(item) for item in obj] + return obj + + def _deduplicate_list(merged_list: list, new_list: list) -> list: """ Append elements from new_list to merged_list, skipping duplicates. diff --git a/cecli/main.py b/cecli/main.py index 0e749a91d8b..f75a31f9e0a 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -659,7 +659,7 @@ async def main_async( with open(config_file, "r") as f: config_data = yaml.safe_load(f) if isinstance(config_data, dict): - cecli_conf_dicts.append(config_data) + cecli_conf_dicts.append(nested._normalize_keys(config_data)) except yaml.YAMLError as e: logging.warning( "Could not parse YAML file %s: %s", config_file, e From d0ff5e848f3af2a12090f96395662aee7f510912 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 16:33:16 -0700 Subject: [PATCH 09/12] fix: Remove unused import _canonical_repr from test_deep_merge.py --- tests/unit/test_deep_merge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_deep_merge.py b/tests/unit/test_deep_merge.py index 119f2bb25e4..bba0da200d1 100644 --- a/tests/unit/test_deep_merge.py +++ b/tests/unit/test_deep_merge.py @@ -2,7 +2,6 @@ import unittest from cecli.helpers.nested import ( - _canonical_repr, _deduplicate_list, deep_merge, deep_merge_config_dicts, From c4801b9172ceb3c2d2b94ff3672e2c147e462074 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 17:39:04 -0700 Subject: [PATCH 10/12] docs: Update conf.md with deep merge behavior for .cecli.conf.yml --- cecli/website/docs/config/conf.md | 129 ++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/cecli/website/docs/config/conf.md b/cecli/website/docs/config/conf.md index ad3c1a8578c..14c69820d03 100644 --- a/cecli/website/docs/config/conf.md +++ b/cecli/website/docs/config/conf.md @@ -23,6 +23,135 @@ You can also specify the `--config ` parameter, which will only load t Lists of values can be specified either as a bulleted list: +## Deep merge behavior for `.cecli.conf.yml` + +There are two types of YAML config files with different merge behaviors: + +| Config File | Merge Behavior | Array Fields | +|-------------|----------------|--------------| +| `.cecli/conf.yml` | Shallow merge (later values replace earlier) | All fields | +| `.cecli.conf.yml` | Deep merge (arrays are concatenated with deduplication) | Array fields only | + +### How deep merge works + +When you have multiple `.cecli.conf.yml` files (e.g., in your home directory and git root), array fields are **concatenated** rather than replaced. Duplicate values are automatically removed, and the order of entries is preserved (earlier config files keep their position). + +```yaml +# ~/.cecli.conf.yml +read: + - ~/.cecli/rules/global.txt + +# project/.cecli.conf.yml +read: + - ./project-rules.txt + +# Result: ["~/.cecli/rules/global.txt", "./project-rules.txt"] +``` + +### Array fields that support deep merge + +The following **top-level** array fields are deep-merged: + +- `rules` - Rules files to load +- `file` - Files to edit +- `read` - Read-only files +- `mcp-servers-files` - MCP server configuration files +- `set-env` - Environment variables +- `api-key` - API keys +- `alias` - Model aliases +- `exempt-paths` - Exempt paths +- `lint-cmd` - Lint commands + +The following **nested** array fields (inside `agent-config`) are also deep-merged: + +- `skills_paths` - Directories to search for skills +- `skills_includelist` - Skills to include (whitelist) +- `skills_excludelist` - Skills to exclude (blacklist) +- `skills_init` - Skills to load on startup +- `subagent_paths` - Directories for sub-agent definitions +- `tools_paths` - Directories for tools +- `tools_includelist` - Tools to include +- `tools_excludelist` - Tools to exclude +- `servers_includelist` - MCP servers to include +- `servers_excludelist` - MCP servers to exclude +- `allowed_commands` - Allowed shell commands + +### JSON/YAML string fields + +The following fields are stored as JSON/YAML strings but are **internally deep-merged** at the dict/list level: + +- `agent-config` - Agent configuration +- `mcp-servers` - MCP server definitions +- `hooks` - Hook configurations +- `model-providers` - Model provider configurations +- `security-config` - Security settings +- `retries` - Retry configuration +- `custom` - Custom configurations +- `tui-config` - TUI configuration + +### Examples + +#### Adding skills directories + +```yaml +# ~/.cecli.conf.yml +agent-config: '{"skills_paths": [".cecli/skills"]}' + +# project/.cecli.conf.yml +agent-config: '{"skills_paths": ["./project-skills"]}' + +# Result: skills_paths contains both ".cecli/skills" and "./project-skills" +``` + +#### Extending allowed commands + +```yaml +# ~/.cecli.conf.yml +allowed-commands: + - git + - npm + +# project/.cecli.conf.yml +allowed-commands: + - docker + - git + +# Result: ["git", "npm", "docker"] (git appears only once, order preserved) +``` + +#### Adding lint commands + +```yaml +# ~/.cecli.conf.yml +lint-cmd: + - python: flake8 + +# project/.cecli.conf.yml +lint-cmd: + - javascript: eslint + +# Result: Both lint commands are active +``` + +### Deduplication + +Entries are deduplicated based on value equality. For primitive values (strings, numbers), equality is checked with `==`. For dicts, `json.dumps` comparison is used. The first occurrence of a value is preserved, ensuring consistent ordering. + +```yaml +# config1 +skills-includelist: + - skill-a + - skill-b + +# config2 +skills-includelist: + - skill-b + - skill-c + +# Result: ["skill-a", "skill-b", "skill-c"] (skill-b not duplicated) +``` + + ``` read: - CONVENTIONS.md From 4ba13490108b6ac15c6c2534769b8d654406d014 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 17:55:00 -0700 Subject: [PATCH 11/12] fix: Normalize array fields to ensure empty lists or JSON objects --- cecli/main.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cecli/main.py b/cecli/main.py index f75a31f9e0a..c34c09a1a42 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -741,6 +741,30 @@ async def main_async( "(configargparse handles .cecli/conf.yml)" ) + # ── Normalize array fields to never be None ─────────────────────────── + # After merging, ensure every DEEP_MERGE_LIST_FIELDS attribute is at + # minimum an empty list [] and every DEEP_MERGE_JSON_FIELDS attribute + # is at minimum an empty JSON object "{}". Downstream code should never + # have to deal with None for these fields. + for key in nested.DEEP_MERGE_LIST_FIELDS: + if hasattr(args, key): + val = getattr(args, key) + if val is None or (isinstance(val, str) and val.strip() == ""): + setattr(args, key, []) + elif not isinstance(val, list): + logging.warning( + "args.%s is not a list (type: %s), coercing to empty list", + key, + type(val).__name__, + ) + setattr(args, key, []) + + for key in nested.DEEP_MERGE_JSON_FIELDS: + if hasattr(args, key): + val = getattr(args, key) + if val is None or (isinstance(val, str) and val.strip() == ""): + setattr(args, key, "{}") + if len(unknown): print("Unknown Args: ", unknown) From b1d7296d732827150784455f990e2abfbb554eb4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 19:12:13 -0700 Subject: [PATCH 12/12] fix: apply black formatting to pass pre-commit --- cecli/coders/agent_coder.py | 14 +++++-- cecli/helpers/nested.py | 5 +-- cecli/main.py | 16 +++---- tests/integration/test_config_loading.py | 53 ++++++++++++------------ tests/unit/test_deep_merge.py | 24 ++++++----- tests/unit/test_edge_cases.py | 2 +- tests/unit/test_performance.py | 2 +- tests/unit/test_shallow_merge.py | 2 +- 8 files changed, 60 insertions(+), 58 deletions(-) diff --git a/cecli/coders/agent_coder.py b/cecli/coders/agent_coder.py index 40c43cd2488..41331965417 100644 --- a/cecli/coders/agent_coder.py +++ b/cecli/coders/agent_coder.py @@ -153,9 +153,17 @@ def _get_agent_config(self): # Validate that array fields are lists, wrap scalars in lists array_fields = [ - "skills_paths", "skills_includelist", "skills_excludelist", "skills_init", - "subagent_paths", "tools_paths", "tools_includelist", "tools_excludelist", - "servers_includelist", "servers_excludelist", "allowed_commands" + "skills_paths", + "skills_includelist", + "skills_excludelist", + "skills_init", + "subagent_paths", + "tools_paths", + "tools_includelist", + "tools_excludelist", + "servers_includelist", + "servers_excludelist", + "allowed_commands", ] for field in array_fields: if field in config and not isinstance(config[field], list): diff --git a/cecli/helpers/nested.py b/cecli/helpers/nested.py index 97a22fb2d39..1153d94d478 100644 --- a/cecli/helpers/nested.py +++ b/cecli/helpers/nested.py @@ -142,10 +142,7 @@ def _normalize_keys(obj: Any) -> Any: Lists, scalars, and non-dict values are returned unchanged. """ if isinstance(obj, dict): - return { - key.replace("-", "_"): _normalize_keys(value) - for key, value in obj.items() - } + return {key.replace("-", "_"): _normalize_keys(value) for key, value in obj.items()} if isinstance(obj, list): return [_normalize_keys(item) for item in obj] return obj diff --git a/cecli/main.py b/cecli/main.py index c34c09a1a42..b7982129193 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -593,7 +593,9 @@ async def main_async( if git_root: all_config_paths.append(str(Path(git_root) / ".cecli.conf.yml")) - conf_yml_files = [p for p in all_config_paths if p.endswith("conf.yml") and not p.endswith(".cecli.conf.yml")] + conf_yml_files = [ + p for p in all_config_paths if p.endswith("conf.yml") and not p.endswith(".cecli.conf.yml") + ] cecli_conf_yml_files = [p for p in all_config_paths if p.endswith(".cecli.conf.yml")] # Only pass .cecli/conf.yml files to configargparse for shallow merge @@ -661,9 +663,7 @@ async def main_async( if isinstance(config_data, dict): cecli_conf_dicts.append(nested._normalize_keys(config_data)) except yaml.YAMLError as e: - logging.warning( - "Could not parse YAML file %s: %s", config_file, e - ) + logging.warning("Could not parse YAML file %s: %s", config_file, e) if cecli_conf_dicts: # Deep-merge .cecli.conf.yml files together @@ -685,9 +685,7 @@ async def main_async( if isinstance(existing_val, list): # Concatenate: .cecli/conf.yml values first, then # .cecli.conf.yml values (deduplicated) - setattr(args, key, nested._deduplicate_list( - list(existing_val), cecli_val - )) + setattr(args, key, nested._deduplicate_list(list(existing_val), cecli_val)) else: setattr(args, key, cecli_val) @@ -732,9 +730,7 @@ async def main_async( try: setattr(args, key, json.dumps(merged_val)) except (TypeError, ValueError) as e: - logging.warning( - "Could not serialize merged config for %s: %s", key, e - ) + logging.warning("Could not serialize merged config for %s: %s", key, e) else: logging.debug( "No .cecli.conf.yml files found — shallow-merge-only mode active " diff --git a/tests/integration/test_config_loading.py b/tests/integration/test_config_loading.py index bf8bf693309..185f005af7c 100644 --- a/tests/integration/test_config_loading.py +++ b/tests/integration/test_config_loading.py @@ -36,7 +36,7 @@ def parse_known_args(self, argv=None): all_configs = {} for f in self.default_config_files: if os.path.exists(f): - with open(f, 'r') as stream: + with open(f, "r") as stream: config = yaml.safe_load(stream) # Shallow merge for simulation all_configs.update(config) @@ -56,7 +56,7 @@ def __init__(self, d): return ArgsNamespace(self.args), [] -sys.modules['configargparse'].ArgumentParser = MockArgumentParser +sys.modules["configargparse"].ArgumentParser = MockArgumentParser from cecli.helpers.nested import ( # noqa: E402 @@ -80,6 +80,7 @@ def setUp(self): def tearDown(self): import shutil + shutil.rmtree(self.conf_dir) def test_multiple_config_files_deep_merge(self): @@ -118,7 +119,7 @@ def test_multiple_config_files_deep_merge(self): expected = { "skills_paths": ["/legacy/skills", "/user/skills", "./project/skills"], "model": "gpt-4", - "temperature": 0.7 + "temperature": 0.7, } self.assertEqual(final_config["skills_paths"], expected["skills_paths"]) self.assertEqual(final_config["model"], expected["model"]) @@ -140,7 +141,6 @@ def test_config_precedence_order(self): # Configargparse shallow merges all first. The last value for 'a' and 'b' wins. # Our manual deep merge will then be applied. - # 1. Base from shallow file base_config = yaml.safe_load(open(self.legacy_conf_path)) @@ -154,30 +154,24 @@ def test_config_precedence_order(self): temp_merged.update(yaml.safe_load(open(self.git_root_conf_path))) # Now, apply our deep merge logic for array fields - final_config = deep_merge_config_dicts([ - base_config, - yaml.safe_load(open(self.user_conf_path)), - yaml.safe_load(open(self.git_root_conf_path)) - ]) + final_config = deep_merge_config_dicts( + [ + base_config, + yaml.safe_load(open(self.user_conf_path)), + yaml.safe_load(open(self.git_root_conf_path)), + ] + ) # 'a' is a scalar, so last one wins - self.assertEqual(final_config['a'], 3) + self.assertEqual(final_config["a"], 3) # 'b' is an array, should be deep merged - self.assertEqual(final_config['b'], [10, 20, 30]) + self.assertEqual(final_config["b"], [10, 20, 30]) def test_agent_config_json_parsing(self): # This tests that a JSON string field like 'agent_config' is correctly merged - conf1 = { - "agent_config": { - "skills_paths": ["/path1"], - "tools_includelist": ["tool-a"] - } - } + conf1 = {"agent_config": {"skills_paths": ["/path1"], "tools_includelist": ["tool-a"]}} conf2 = { - "agent_config": { - "skills_paths": ["/path2"], - "tools_includelist": ["tool-b", "tool-a"] - } + "agent_config": {"skills_paths": ["/path2"], "tools_includelist": ["tool-b", "tool-a"]} } # Simulate loading from two .cecli.conf.yml files @@ -186,19 +180,24 @@ def test_agent_config_json_parsing(self): # The result is a Python dict. In main.py, this would be re-serialized to JSON. expected_agent_config = { "skills_paths": ["/path1", "/path2"], - "tools_includelist": ["tool-a", "tool-b"] + "tools_includelist": ["tool-a", "tool-b"], } - self.assertEqual(merged['agent_config'], expected_agent_config) + self.assertEqual(merged["agent_config"], expected_agent_config) # Test the re-serialization step - final_json_string = json.dumps(merged['agent_config']) + final_json_string = json.dumps(merged["agent_config"]) parsed_final = json.loads(final_json_string) # Sort lists for comparison since order inside JSON can be tricky - self.assertEqual(sorted(parsed_final['skills_paths']), sorted(expected_agent_config['skills_paths'])) - self.assertEqual(sorted(parsed_final['tools_includelist']), sorted(expected_agent_config['tools_includelist'])) + self.assertEqual( + sorted(parsed_final["skills_paths"]), sorted(expected_agent_config["skills_paths"]) + ) + self.assertEqual( + sorted(parsed_final["tools_includelist"]), + sorted(expected_agent_config["tools_includelist"]), + ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_deep_merge.py b/tests/unit/test_deep_merge.py index bba0da200d1..c2a77b49c2b 100644 --- a/tests/unit/test_deep_merge.py +++ b/tests/unit/test_deep_merge.py @@ -15,19 +15,19 @@ def test_deep_merge_simple_arrays(self): "skills_paths": [".cecli/skills"], "subagent_paths": [".cecli/subagents"], "tools_paths": [".cecli/tools"], - "allowed_commands": ["git", "npm"] + "allowed_commands": ["git", "npm"], } dict2 = { "skills_paths": ["./project-skills"], "subagent_paths": ["./custom-subagents"], "tools_paths": ["./project-tools"], - "allowed_commands": ["docker", "git"] + "allowed_commands": ["docker", "git"], } expected = { "skills_paths": [".cecli/skills", "./project-skills"], "subagent_paths": [".cecli/subagents", "./custom-subagents"], "tools_paths": [".cecli/tools", "./project-tools"], - "allowed_commands": ["git", "npm", "docker"] + "allowed_commands": ["git", "npm", "docker"], } merged = deep_merge(dict1, dict2, deep_merge_arrays=True) self.assertEqual(merged, expected) @@ -49,11 +49,15 @@ def test_deep_merge_order_preservation(self): def test_deep_merge_nested_arrays(self): dict1 = {"mcp_servers": [{"server-a": {"command": ["cmd1"]}}]} dict2 = {"mcp_servers": [{"server-b": {"command": ["cmd2"]}}]} - expected = {"mcp_servers": [{"server-a": {"command": ["cmd1"]}}, {"server-b": {"command": ["cmd2"]}}]} + expected = { + "mcp_servers": [ + {"server-a": {"command": ["cmd1"]}}, + {"server-b": {"command": ["cmd2"]}}, + ] + } merged = deep_merge(dict1, dict2, deep_merge_arrays=True) self.assertEqual(merged, expected) - def test_json_field_serialization(self): dict1 = {"mcp_servers": {"server-a": {"command": ["cmd1"]}}} dict2 = {"mcp_servers": {"server-b": {"command": ["cmd2"]}}} @@ -66,15 +70,13 @@ def test_json_field_serialization(self): def test_json_field_deep_merge(self): dict1 = {"mcp_servers": {"server-a": {"command": ["cmd1"]}}} dict2 = {"mcp_servers": {"server-b": {"command": ["cmd2"]}}} - expected = {"mcp_servers": {"server-a": {"command": ["cmd1"]}, "server-b": {"command": ["cmd2"]}}} + expected = { + "mcp_servers": {"server-a": {"command": ["cmd1"]}, "server-b": {"command": ["cmd2"]}} + } merged = deep_merge(dict1, dict2, deep_merge_arrays=True) self.assertEqual(merged, expected) - - - - class TestConfigHelpers(unittest.TestCase): def test_is_cecli_conf_file(self): self.assertTrue(is_cecli_conf_file("/path/to/.cecli.conf.yml")) @@ -95,4 +97,4 @@ def test_deduplicate_list(self): list2 = [2, 3, {"a": 1}, {"b": 2}] expected = [1, 2, {"a": 1}, 3, {"b": 2}] result = _deduplicate_list(list1, list2) - self.assertEqual(result, expected) \ No newline at end of file + self.assertEqual(result, expected) diff --git a/tests/unit/test_edge_cases.py b/tests/unit/test_edge_cases.py index 17233b21cb3..0bdd1da643d 100644 --- a/tests/unit/test_edge_cases.py +++ b/tests/unit/test_edge_cases.py @@ -47,4 +47,4 @@ def test_malformed_yaml(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_performance.py b/tests/unit/test_performance.py index eec48979f67..83518e45344 100644 --- a/tests/unit/test_performance.py +++ b/tests/unit/test_performance.py @@ -24,4 +24,4 @@ def test_deep_copy_behavior(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_shallow_merge.py b/tests/unit/test_shallow_merge.py index c4dd4747ebb..cf439002efc 100644 --- a/tests/unit/test_shallow_merge.py +++ b/tests/unit/test_shallow_merge.py @@ -13,4 +13,4 @@ def test_shallow_merge_replaces_arrays(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()