From 1639ec127d927fe422137716202b65796fcc4671 Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sat, 18 Jul 2026 08:31:05 +0200 Subject: [PATCH 1/2] feat: index nix and pkl sources --- graphify/detect.py | 2 +- graphify/extract.py | 4 ++ graphify/extractors/nix.py | 82 ++++++++++++++++++++++++++++++++++++++ graphify/extractors/pkl.py | 77 +++++++++++++++++++++++++++++++++++ tests/test_detect.py | 5 +++ tests/test_extract.py | 20 +++++++++- 6 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 graphify/extractors/nix.py create mode 100644 graphify/extractors/pkl.py diff --git a/graphify/detect.py b/graphify/detect.py index c2638a0ef..65ace9e39 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.nix', '.pkl', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index bc10f7d7d..4d614b00f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -44,8 +44,10 @@ from graphify.extractors.go import extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 from graphify.extractors.markdown import extract_markdown # noqa: F401 +from graphify.extractors.nix import extract_nix # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 +from graphify.extractors.pkl import extract_pkl # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 from graphify.extractors.sln import extract_sln # noqa: F401 @@ -3920,6 +3922,8 @@ def add_existing_edge(edge: dict) -> None: ".sh": extract_bash, ".bash": extract_bash, ".json": extract_json, + ".nix": extract_nix, + ".pkl": extract_pkl, ".tf": extract_terraform, ".tfvars": extract_terraform, ".hcl": extract_terraform, diff --git a/graphify/extractors/nix.py b/graphify/extractors/nix.py new file mode 100644 index 000000000..e3d9921ea --- /dev/null +++ b/graphify/extractors/nix.py @@ -0,0 +1,82 @@ +"""Dependency-free structural extraction for Nix expressions.""" +from __future__ import annotations + +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id + +_BINDING_RE = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_'-]*)\s*=\s*") +_FUNCTION_RE = re.compile( + r"(?m)^\s*([A-Za-z_][A-Za-z0-9_'-]*)\s*=\s*(?:\([^\n]*\)|[A-Za-z_][A-Za-z0-9_'-]*)\s*:\s*" +) +_IMPORT_RE = re.compile(r"(? dict: + try: + source = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + source_file = str(path) + stem = _file_stem(path) + file_id = _make_id(stem) + nodes = [{"id": file_id, "label": path.name, "file_type": "code", + "source_file": source_file, "source_location": "L1"}] + edges: list[dict] = [] + seen = {file_id} + bindings: dict[str, str] = {} + + def line(match: re.Match) -> int: + return source.count("\n", 0, match.start()) + 1 + + def add_node(name: str, at: int, kind: str = "binding") -> str: + node_id = _make_id(stem, name) + if node_id not in seen: + seen.add(node_id) + nodes.append({"id": node_id, "label": name, "file_type": "code", + "source_file": source_file, "source_location": f"L{at}", + "kind": kind}) + return node_id + + def add_edge(source: str, target: str, relation: str, at: int, context: str | None = None) -> None: + if source == target: + return + edge = {"source": source, "target": target, "relation": relation, + "confidence": "EXTRACTED", "source_file": source_file, + "source_location": f"L{at}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + for match in _BINDING_RE.finditer(source): + name = match.group(1) + if name in {"let", "in", "with", "inherit", "assert", "rec"}: + continue + kind = "function" if _FUNCTION_RE.match(source, match.start()) else "binding" + binding_id = add_node(name, line(match), kind) + bindings.setdefault(name, binding_id) + add_edge(file_id, binding_id, "contains", line(match)) + + for match in _IMPORT_RE.finditer(source): + target = match.group(1) + target_id = _make_id(_file_stem(path.parent / target)) + if target_id not in seen: + seen.add(target_id) + nodes.append({"id": target_id, "label": Path(target).name, "file_type": "concept", + "source_file": source_file, "source_location": f"L{line(match)}", + "kind": "relative-import"}) + add_edge(file_id, target_id, "imports", line(match), "literal-relative-path") + + for match in _INTERPOLATION_RE.finditer(source): + target = bindings.get(match.group(1)) + if target: + add_edge(file_id, target, "references", line(match), "interpolation") + for match in _INHERIT_RE.finditer(source): + for name in re.findall(r"[A-Za-z_][A-Za-z0-9_'-]*", match.group(1)): + target = bindings.get(name) + if target: + add_edge(file_id, target, "references", line(match), "inherit") + return {"nodes": nodes, "edges": edges} diff --git a/graphify/extractors/pkl.py b/graphify/extractors/pkl.py new file mode 100644 index 000000000..2b0bfe07c --- /dev/null +++ b/graphify/extractors/pkl.py @@ -0,0 +1,77 @@ +"""Dependency-free structural extraction for Pkl configuration modules.""" +from __future__ import annotations + +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id + +_DECL_RE = re.compile( + r"(?m)^\s*(?:(abstract)\s+)?(module|class|typealias|function|modulemethod)\s+([A-Za-z_][A-Za-z0-9_]*)" +) +_PROPERTY_RE = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z_][A-Za-z0-9_?.<>\[\]]*)") +_IMPORT_RE = re.compile(r"(?m)^\s*(import|amends|extends)\s+([^\n]+)") + + +def extract_pkl(path: Path) -> dict: + try: + source = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + source_file = str(path) + stem = _file_stem(path) + file_id = _make_id(stem) + nodes = [{"id": file_id, "label": path.name, "file_type": "code", + "source_file": source_file, "source_location": "L1"}] + edges: list[dict] = [] + seen = {file_id} + + def line(match: re.Match) -> int: + return source.count("\n", 0, match.start()) + 1 + + def add_node(name: str, at: int, kind: str) -> str: + node_id = _make_id(stem, name) + if node_id not in seen: + seen.add(node_id) + nodes.append({"id": node_id, "label": name, "file_type": "code", + "source_file": source_file, "source_location": f"L{at}", + "kind": kind}) + return node_id + + def add_edge(source: str, target: str, relation: str, at: int, context: str | None = None) -> None: + if source == target: + return + edge = {"source": source, "target": target, "relation": relation, + "confidence": "EXTRACTED", "source_file": source_file, + "source_location": f"L{at}", "weight": 1.0} + if context: + edge["context"] = context + edges.append(edge) + + symbols: dict[str, str] = {} + for match in _DECL_RE.finditer(source): + name, kind = match.group(3), match.group(2) + symbol_id = add_node(name, line(match), kind) + symbols.setdefault(name, symbol_id) + add_edge(file_id, symbol_id, "contains", line(match)) + for match in _PROPERTY_RE.finditer(source): + name, type_name = match.group(1), match.group(2) + if name in {"module", "class", "typealias", "function", "extends", "amends"}: + continue + property_id = add_node(name, line(match), "property") + symbols.setdefault(name, property_id) + add_edge(file_id, property_id, "contains", line(match)) + type_id = add_node(type_name, line(match), "type") + add_edge(property_id, type_id, "references", line(match), "type") + for match in _IMPORT_RE.finditer(source): + relation, expression = match.group(1), match.group(2).strip() + target = expression.strip('"') + target_id = _make_id("pkl", target) + if target_id not in seen: + seen.add(target_id) + nodes.append({"id": target_id, "label": target, "file_type": "concept", + "source_file": source_file, "source_location": f"L{line(match)}", + "kind": "module-reference"}) + add_edge(file_id, target_id, "imports" if relation == "import" else relation, + line(match), "module-reference") + return {"nodes": nodes, "edges": edges} diff --git a/tests/test_detect.py b/tests/test_detect.py index 837d8cf3d..348ee8510 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -12,6 +12,11 @@ def test_classify_python(): def test_classify_typescript(): assert classify_file(Path("bar.ts")) == FileType.CODE + +def test_classify_nix_and_pkl(): + assert classify_file(Path("flake.nix")) == FileType.CODE + assert classify_file(Path("Schema.pkl")) == FileType.CODE + def test_classify_powershell_module(): # #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap). assert classify_file(Path("Utils.psm1")) == FileType.CODE diff --git a/tests/test_extract.py b/tests/test_extract.py index 98b73edb9..b26db6fe7 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -7,11 +7,29 @@ import pytest from graphify.build import build_from_json -from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, _DISPATCH +from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, extract_nix, extract_pkl, _DISPATCH FIXTURES = Path(__file__).parent / "fixtures" +def test_extract_nix_records_bindings_and_literal_imports(tmp_path): + path = tmp_path / "default.nix" + path.write_text('''{ inputs, ... }:\nlet\n pkgs = import ./nixpkgs.nix;\n value = "${pkgs}";\nin\n{ inherit value; }\n''', encoding="utf-8") + result = extract_nix(path) + labels = {node["label"] for node in result["nodes"]} + assert {"pkgs", "value", "nixpkgs.nix"} <= labels + assert any(edge["relation"] == "imports" for edge in result["edges"]) + + +def test_extract_pkl_records_declarations_and_module_relationships(tmp_path): + path = tmp_path / "Schema.pkl" + path.write_text('''module Schema\nimport "pkl:base"\nclass Host {}\nname: String\n''', encoding="utf-8") + result = extract_pkl(path) + labels = {node["label"] for node in result["nodes"]} + assert {"Schema", "Host", "name", "String", "pkl:base"} <= labels + assert any(edge["relation"] == "imports" for edge in result["edges"]) + + def test_make_id_strips_dots_and_underscores(): assert _make_id("_auth") == "auth" assert _make_id(".httpx._client") == "httpx_client" From 2e0587b043ea83b4de902acb1a4074d1f586487e Mon Sep 17 00:00:00 2001 From: "Can H. Tartanoglu" Date: Sun, 19 Jul 2026 10:10:06 +0200 Subject: [PATCH 2/2] refactor: move format-specific extraction behind external ingress --- graphify/cli.py | 43 +++++++++++++++++--- graphify/detect.py | 2 +- graphify/external.py | 73 +++++++++++++++++++++++++++++++++ graphify/extract.py | 2 - graphify/extractors/pkl.py | 77 ----------------------------------- graphify/watch.py | 58 +++++++++++++++++++++++++- tests/test_detect.py | 3 +- tests/test_external.py | 83 ++++++++++++++++++++++++++++++++++++++ tests/test_extract.py | 11 +---- 9 files changed, 253 insertions(+), 99 deletions(-) create mode 100644 graphify/external.py delete mode 100644 graphify/extractors/pkl.py create mode 100644 tests/test_external.py diff --git a/graphify/cli.py b/graphify/cli.py index 774034220..9891267dd 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1647,12 +1647,25 @@ def dispatch_command(cmd: str) -> None: no_cluster = False args = sys.argv[2:] watch_arg: str | None = None - for a in args: + external_extractions: list[Path] = [] + i = 0 + while i < len(args): + a = args[i] if a == "--force": force = True + i += 1 continue if a == "--no-cluster": no_cluster = True + i += 1 + continue + if a == "--external-extraction" and i + 1 < len(args): + external_extractions.append(Path(args[i + 1])) + i += 2 + continue + if a.startswith("--external-extraction="): + external_extractions.append(Path(a.split("=", 1)[1])) + i += 1 continue if a.startswith("-"): print(f"error: unknown update option: {a}", file=sys.stderr) @@ -1661,6 +1674,7 @@ def dispatch_command(cmd: str) -> None: print("error: update accepts at most one path argument", file=sys.stderr) sys.exit(2) watch_arg = a + i += 1 if watch_arg is not None: watch_path = Path(watch_arg) @@ -1680,7 +1694,13 @@ def dispatch_command(cmd: str) -> None: # Interactive CLI: block on the per-repo lock rather than skip, so the # user sees their explicit `graphify update` complete instead of # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) + ok = _rebuild_code( + watch_path, + force=force, + no_cluster=no_cluster, + external_extractions=external_extractions, + block_on_lock=True, + ) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") if not ( @@ -2325,6 +2345,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print( "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " + "[--external-extraction PATH ...] " "[--no-gitignore] " "[--max-workers N] [--token-budget N] [--max-concurrency N] " "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", @@ -2347,6 +2368,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": extract_mode: str | None = None out_dir: Path | None = None cli_postgres_dsn: str | None = None + external_extractions: list[Path] = [] cli_cargo: bool = False cli_allow_partial: bool = False no_cluster = False @@ -2412,6 +2434,10 @@ def _parse_float(name: str, raw: str) -> float: out_dir = Path(args[i + 1]); i += 2 elif a.startswith("--out="): out_dir = Path(a.split("=", 1)[1]); i += 1 + elif a == "--external-extraction" and i + 1 < len(args): + external_extractions.append(Path(args[i + 1])); i += 2 + elif a.startswith("--external-extraction="): + external_extractions.append(Path(a.split("=", 1)[1])); i += 1 elif a == "--no-cluster": no_cluster = True; i += 1 elif a == "--dedup-llm": @@ -2499,6 +2525,12 @@ def _parse_float(name: str, raw: str) -> float: out_root = (out_dir.resolve() if out_dir else target) graphify_out = out_root / _GRAPHIFY_OUT graphify_out.mkdir(parents=True, exist_ok=True) + from graphify.external import load_extractions as _load_external_extractions + external_result = _load_external_extractions(external_extractions, root=target) + external_source_files = { + str((target / source).resolve()) + for source in external_result.get("source_files", []) + } # Persist corpus-shaping options so later update/watch/hook rebuilds # use the same file set as the initial extraction (#1886). from graphify.watch import ( @@ -2584,6 +2616,7 @@ def _parse_float(name: str, raw: str) -> float: # graph's own sources are reconciled against the current corpus. _seen_files = {f for _fl in files_by_type.values() for f in _fl} _seen_files.update(detection.get("unclassified", [])) + _seen_files.update(external_source_files) graph_stale_sources = _stale_graph_sources( existing_graph_path, target, _seen_files ) @@ -2996,9 +3029,9 @@ def _progress(idx: int, total: int, _result: dict) -> None: # for symbols also referenced in docs). Hyperedges only come from the # semantic side. merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), + "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])) + list(external_result.get("nodes", [])), + "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])) + list(external_result.get("edges", [])), + "hyperedges": list(sem_result.get("hyperedges", [])) + list(external_result.get("hyperedges", [])), "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), } diff --git a/graphify/detect.py b/graphify/detect.py index 65ace9e39..34112341f 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.nix', '.pkl', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.nix', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/external.py b/graphify/external.py new file mode 100644 index 000000000..6e9271459 --- /dev/null +++ b/graphify/external.py @@ -0,0 +1,73 @@ +"""Generic external extraction ingress for Graphify integrations.""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +from graphify.validate import assert_valid + +_MAX_EXTERNAL_BYTES = 32 * 1024 * 1024 +_BUCKETS = ("nodes", "edges", "hyperedges") + + +def load_extractions(paths: list[Path], *, root: Path) -> dict: + """Load and merge generic extraction envelopes from trusted integrations. + + Each envelope contains ``nodes``, ``edges``, optional ``hyperedges``, and + ``source_files``. Source paths are normalized to POSIX paths relative to + the scan root, matching Graphify's ordinary extraction output. + """ + merged = {"nodes": [], "edges": [], "hyperedges": [], "source_files": []} + seen_sources: set[str] = set() + root = root.resolve() + for raw_path in paths: + path = Path(raw_path) + if not path.is_file(): + raise ValueError(f"external extraction does not exist: {path}") + if path.stat().st_size > _MAX_EXTERNAL_BYTES: + raise ValueError(f"external extraction exceeds {_MAX_EXTERNAL_BYTES} bytes: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"external extraction must be an object: {path}") + if "hyperedges" not in payload: + payload["hyperedges"] = [] + assert_valid(payload) + + payload_sources = payload.get("source_files") + if payload_sources is None: + payload_sources = [] + for bucket in _BUCKETS: + for item in payload.get(bucket, []): + source = item.get("source_file") + if source: + payload_sources.append(source) + if not isinstance(payload_sources, list) or not all( + isinstance(source, str) and source for source in payload_sources + ): + raise ValueError(f"external extraction source_files must be a list of strings: {path}") + + for bucket in _BUCKETS: + for item in payload.get(bucket, []): + source = item.get("source_file") + if source: + item["source_file"] = _relative_source(source, root) + item.setdefault("_origin", "external") + merged[bucket].append(item) + for source in payload_sources: + normalized = _relative_source(source, root) + if normalized not in seen_sources: + seen_sources.add(normalized) + merged["source_files"].append(normalized) + + return merged + + +def _relative_source(source: str, root: Path) -> str: + path = Path(source) + if not path.is_absolute(): + path = root / path + try: + return path.resolve().relative_to(root).as_posix() + except ValueError: + return os.path.relpath(path.resolve(), root).replace(os.sep, "/") diff --git a/graphify/extract.py b/graphify/extract.py index 4d614b00f..1c89935d3 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -47,7 +47,6 @@ from graphify.extractors.nix import extract_nix # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 -from graphify.extractors.pkl import extract_pkl # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 from graphify.extractors.rust import extract_rust # noqa: F401 from graphify.extractors.sln import extract_sln # noqa: F401 @@ -3923,7 +3922,6 @@ def add_existing_edge(edge: dict) -> None: ".bash": extract_bash, ".json": extract_json, ".nix": extract_nix, - ".pkl": extract_pkl, ".tf": extract_terraform, ".tfvars": extract_terraform, ".hcl": extract_terraform, diff --git a/graphify/extractors/pkl.py b/graphify/extractors/pkl.py deleted file mode 100644 index 2b0bfe07c..000000000 --- a/graphify/extractors/pkl.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Dependency-free structural extraction for Pkl configuration modules.""" -from __future__ import annotations - -import re -from pathlib import Path - -from graphify.extractors.base import _file_stem, _make_id - -_DECL_RE = re.compile( - r"(?m)^\s*(?:(abstract)\s+)?(module|class|typealias|function|modulemethod)\s+([A-Za-z_][A-Za-z0-9_]*)" -) -_PROPERTY_RE = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z_][A-Za-z0-9_?.<>\[\]]*)") -_IMPORT_RE = re.compile(r"(?m)^\s*(import|amends|extends)\s+([^\n]+)") - - -def extract_pkl(path: Path) -> dict: - try: - source = path.read_text(encoding="utf-8", errors="replace") - except OSError as exc: - return {"nodes": [], "edges": [], "error": str(exc)} - source_file = str(path) - stem = _file_stem(path) - file_id = _make_id(stem) - nodes = [{"id": file_id, "label": path.name, "file_type": "code", - "source_file": source_file, "source_location": "L1"}] - edges: list[dict] = [] - seen = {file_id} - - def line(match: re.Match) -> int: - return source.count("\n", 0, match.start()) + 1 - - def add_node(name: str, at: int, kind: str) -> str: - node_id = _make_id(stem, name) - if node_id not in seen: - seen.add(node_id) - nodes.append({"id": node_id, "label": name, "file_type": "code", - "source_file": source_file, "source_location": f"L{at}", - "kind": kind}) - return node_id - - def add_edge(source: str, target: str, relation: str, at: int, context: str | None = None) -> None: - if source == target: - return - edge = {"source": source, "target": target, "relation": relation, - "confidence": "EXTRACTED", "source_file": source_file, - "source_location": f"L{at}", "weight": 1.0} - if context: - edge["context"] = context - edges.append(edge) - - symbols: dict[str, str] = {} - for match in _DECL_RE.finditer(source): - name, kind = match.group(3), match.group(2) - symbol_id = add_node(name, line(match), kind) - symbols.setdefault(name, symbol_id) - add_edge(file_id, symbol_id, "contains", line(match)) - for match in _PROPERTY_RE.finditer(source): - name, type_name = match.group(1), match.group(2) - if name in {"module", "class", "typealias", "function", "extends", "amends"}: - continue - property_id = add_node(name, line(match), "property") - symbols.setdefault(name, property_id) - add_edge(file_id, property_id, "contains", line(match)) - type_id = add_node(type_name, line(match), "type") - add_edge(property_id, type_id, "references", line(match), "type") - for match in _IMPORT_RE.finditer(source): - relation, expression = match.group(1), match.group(2).strip() - target = expression.strip('"') - target_id = _make_id("pkl", target) - if target_id not in seen: - seen.add(target_id) - nodes.append({"id": target_id, "label": target, "file_type": "concept", - "source_file": source_file, "source_location": f"L{line(match)}", - "kind": "module-reference"}) - add_edge(file_id, target_id, "imports" if relation == "import" else relation, - line(match), "module-reference") - return {"nodes": nodes, "edges": edges} diff --git a/graphify/watch.py b/graphify/watch.py index 37edc9cb1..5b6380a43 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -442,6 +442,8 @@ def _reconcile_existing_graph( full_rebuild: bool, deleted_paths: set[str], deleted_source_identities: set[str], + external_source_identities: set[str], + external_rebuilt_source_identities: set[str], ) -> tuple[dict, dict]: """Merge fresh extraction with preserved graph entries and evict stale sources.""" existing_graph_data: dict = {} @@ -467,9 +469,11 @@ def _reconcile_existing_graph( current_sources = { source_paths.absolute_identity(str(path), project_root) for path in code_files } + current_sources.update(external_source_identities) rebuilt_source_identities = { source_paths.absolute_identity(str(path), project_root) for path in extract_targets } + rebuilt_source_identities.update(external_rebuilt_source_identities) node_evicted_source_identities = set(deleted_source_identities) hyperedge_evicted_source_identities = set(deleted_source_identities) # Deletion evicts edges regardless of tier; re-extraction only owns a @@ -478,6 +482,42 @@ def _reconcile_existing_graph( if not full_rebuild: node_evicted_source_identities.update(rebuilt_source_identities) + # External integrations own every source they submit on each run. Keep + # that ownership separate from the AST tier so a fresh fragment + # replaces its prior contribution without evicting semantic nodes. + node_evicted_source_identities.update(external_rebuilt_source_identities) + edge_evicted_source_identities.update(external_rebuilt_source_identities) + hyperedge_evicted_source_identities.update(external_rebuilt_source_identities) + + # A source that disappeared from an external payload is deleted only + # when its file is gone. If it still exists, preserve it fail-closed so + # a temporarily incomplete integration cannot silently erase data. + external_missing_files: set[str] = set() + external_missing_nodes = 0 + for node in existing.get("nodes", []): + if node.get("_origin") != "external": + continue + identity = source_paths.identity(node.get("source_file")) + if not identity or identity in external_source_identities: + continue + if not source_paths.in_watch_root(node.get("source_file")): + continue + if Path(identity).exists(): + continue + normalized = source_paths.normalize(node.get("source_file")) + if normalized: + deleted_paths.add(normalized) + external_missing_files.add(identity) + node_evicted_source_identities.add(identity) + edge_evicted_source_identities.add(identity) + hyperedge_evicted_source_identities.add(identity) + external_missing_nodes += 1 + if external_missing_nodes: + print( + f"[graphify watch] pruned {external_missing_nodes} node(s) from " + f"{len(external_missing_files)} removed external source(s)." + ) + # Reconcile every rebuild against the current watched corpus. Hook change # lists can contain only a rename destination, so explicit paths alone # cannot identify the stale source. Keep the comparison scoped to the @@ -562,7 +602,7 @@ def _reconcile_existing_graph( and edge.get("target") in all_ids and not source_paths.is_evicted(edge, edge_evicted_source_identities) and not ( - edge.get("_origin") == "ast" + edge.get("_origin") in ("ast", "external") and source_paths.is_evicted(edge, rebuilt_source_identities) ) ] @@ -797,6 +837,7 @@ def _rebuild_code( follow_symlinks: bool = False, force: bool = False, no_cluster: bool = False, + external_extractions: list[Path] | None = None, acquire_lock: bool = True, block_on_lock: bool = False, ) -> bool: @@ -857,6 +898,7 @@ def _rebuild_code( follow_symlinks=follow_symlinks, force=force, no_cluster=no_cluster, + external_extractions=external_extractions, acquire_lock=False, ) # Late-arrival drain: another hook may have queued work while we @@ -874,6 +916,7 @@ def _rebuild_code( follow_symlinks=follow_symlinks, force=force, no_cluster=no_cluster, + external_extractions=external_extractions, acquire_lock=False, ) and ok return ok @@ -883,6 +926,7 @@ def _rebuild_code( report_root = _report_root_label(watch_path) try: from graphify.extract import extract, _get_extractor + from graphify.external import load_extractions from graphify.detect import detect from graphify.build import build_from_json, _norm_source_file as _nsf from graphify.cluster import cluster, remap_communities_to_previous, score_all @@ -901,6 +945,11 @@ def _rebuild_code( gitignore=_read_build_gitignore(out), ) code_files = [Path(f) for f in detected['files']['code']] + external_result = load_extractions(external_extractions or [], root=watch_root) + external_source_identities = { + Path(os.path.abspath(watch_root / source)).as_posix() + for source in external_result.get("source_files", []) + } # Include document files that have AST extractors (e.g. .md, .mdx, .qmd) ast_doc_files: list[Path] = [] @@ -911,7 +960,7 @@ def _rebuild_code( ast_doc_files.append(p) existing_graph = out / "graph.json" - if not code_files and not existing_graph.exists(): + if not code_files and not external_result.get("source_files") and not existing_graph.exists(): print("[graphify watch] No code files found - nothing to rebuild.") return False @@ -1042,6 +1091,9 @@ def _add_deleted_source(path: Path) -> None: "nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, } + result.setdefault("nodes", []).extend(external_result.get("nodes", [])) + result.setdefault("edges", []).extend(external_result.get("edges", [])) + result.setdefault("hyperedges", []).extend(external_result.get("hyperedges", [])) _rebase_relative_source_files(result, watch_root, project_root) # Preserve semantic nodes/edges from a previous full run. @@ -1063,6 +1115,8 @@ def _add_deleted_source(path: Path) -> None: full_rebuild=changed_paths is None, deleted_paths=deleted_paths, deleted_source_identities=deleted_source_identities, + external_source_identities=external_source_identities, + external_rebuilt_source_identities=external_source_identities, ) _relativize_source_files(result, project_root, scope=watch_root) diff --git a/tests/test_detect.py b/tests/test_detect.py index 348ee8510..3ebcd6d3f 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -13,9 +13,8 @@ def test_classify_typescript(): assert classify_file(Path("bar.ts")) == FileType.CODE -def test_classify_nix_and_pkl(): +def test_classify_nix(): assert classify_file(Path("flake.nix")) == FileType.CODE - assert classify_file(Path("Schema.pkl")) == FileType.CODE def test_classify_powershell_module(): # #1315: .psm1 modules were never indexed (CODE_EXTENSIONS gap). diff --git a/tests/test_external.py b/tests/test_external.py new file mode 100644 index 000000000..0dd513f2c --- /dev/null +++ b/tests/test_external.py @@ -0,0 +1,83 @@ +import json +from pathlib import Path + +import pytest + +from graphify.external import load_extractions +from graphify.watch import _rebuild_code + + +def _payload(source: str, suffix: str) -> dict: + source_id = f"external::{suffix}" + return { + "nodes": [ + { + "id": source_id, + "label": suffix, + "file_type": "code", + "source_file": source, + } + ], + "edges": [], + "hyperedges": [], + "source_files": [source], + } + + +def test_load_extraction_normalizes_sources_and_merges_fragments(tmp_path: Path): + first = tmp_path / "first.json" + second = tmp_path / "second.json" + first.write_text(json.dumps(_payload("config/one.cfg", "one")), encoding="utf-8") + second.write_text(json.dumps(_payload("config/two.cfg", "two")), encoding="utf-8") + + merged = load_extractions([first, second], root=tmp_path) + + assert [node["label"] for node in merged["nodes"]] == ["one", "two"] + assert merged["source_files"] == ["config/one.cfg", "config/two.cfg"] + + +def test_load_extraction_rejects_invalid_schema(tmp_path: Path): + path = tmp_path / "invalid.json" + path.write_text(json.dumps({"nodes": [], "edges": [{"source": "missing"}]}), encoding="utf-8") + + with pytest.raises(ValueError, match="missing required field"): + load_extractions([path], root=tmp_path) + + +def test_update_replaces_and_prunes_external_sources(tmp_path: Path): + source = tmp_path / "external" / "config.source" + source.parent.mkdir() + source.write_text("live", encoding="utf-8") + payload = tmp_path / "graphify-out" / "payload.json" + payload.parent.mkdir() + + payload.write_text(json.dumps(_payload("external/config.source", "first")), encoding="utf-8") + assert _rebuild_code( + tmp_path, + external_extractions=[payload], + no_cluster=True, + acquire_lock=False, + ) + graph = json.loads((tmp_path / "graphify-out/graph.json").read_text(encoding="utf-8")) + assert {node["label"] for node in graph["nodes"]} == {"first"} + + payload.write_text(json.dumps(_payload("external/config.source", "second")), encoding="utf-8") + assert _rebuild_code( + tmp_path, + external_extractions=[payload], + no_cluster=True, + acquire_lock=False, + ) + graph = json.loads((tmp_path / "graphify-out/graph.json").read_text(encoding="utf-8")) + assert {node["label"] for node in graph["nodes"]} == {"second"} + + source.unlink() + payload.write_text(json.dumps({"nodes": [], "edges": [], "hyperedges": [], "source_files": []}), encoding="utf-8") + assert _rebuild_code( + tmp_path, + external_extractions=[payload], + no_cluster=True, + acquire_lock=False, + ) + graph = json.loads((tmp_path / "graphify-out/graph.json").read_text(encoding="utf-8")) + assert graph["nodes"] == [] diff --git a/tests/test_extract.py b/tests/test_extract.py index b26db6fe7..ec05a6e00 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -7,7 +7,7 @@ import pytest from graphify.build import build_from_json -from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, extract_nix, extract_pkl, _DISPATCH +from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, extract_nix, _DISPATCH FIXTURES = Path(__file__).parent / "fixtures" @@ -21,15 +21,6 @@ def test_extract_nix_records_bindings_and_literal_imports(tmp_path): assert any(edge["relation"] == "imports" for edge in result["edges"]) -def test_extract_pkl_records_declarations_and_module_relationships(tmp_path): - path = tmp_path / "Schema.pkl" - path.write_text('''module Schema\nimport "pkl:base"\nclass Host {}\nname: String\n''', encoding="utf-8") - result = extract_pkl(path) - labels = {node["label"] for node in result["nodes"]} - assert {"Schema", "Host", "name", "String", "pkl:base"} <= labels - assert any(edge["relation"] == "imports" for edge in result["edges"]) - - def test_make_id_strips_dots_and_underscores(): assert _make_id("_auth") == "auth" assert _make_id(".httpx._client") == "httpx_client"