From 4c31235c374e21e7b983f2f8e978f0e550719902 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Sun, 11 Oct 2026 09:21:13 +0000 Subject: [PATCH] fix: write graph.json as one line of compact JSON (#3413) graph.json was pretty-printed (indent=2), so every PR that commits a rebuilt graph reports tens of thousands of changed lines. GitHub's PR line counts ignore .gitattributes (-diff, binary), so only the file's content can fix it: one line of compact JSON is +1 -1. Every writer of a project graph.json now emits separators=(",", ":") with no indent: to_json, the `update` raw/no-cluster candidate text, prune, merge-driver, merge-graphs, `extract --no-cluster` and the global graph. Readers are unaffected. `update` compares parsed data, not bytes, so an existing indented graph would have been "unchanged" forever. It is now rewritten compact once, then left alone. Co-Authored-By: Claude --- graphify/cli.py | 8 +- graphify/export.py | 2 +- graphify/global_graph.py | 2 +- graphify/paths.py | 12 +- graphify/watch.py | 17 ++- tests/test_compact_graph_json.py | 187 +++++++++++++++++++++++++++++++ 6 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 tests/test_compact_graph_json.py diff --git a/graphify/cli.py b/graphify/cli.py index e4d8d58243..4991af17ae 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -684,7 +684,7 @@ def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int from graphify.export import backup_if_protected as _backup _backup(graph_path.parent) from graphify.paths import write_json_atomic - write_json_atomic(graph_path, data, indent=2) + write_json_atomic(graph_path, data, compact=True) return n_removed @@ -2725,7 +2725,7 @@ def _load_graph(p: str): if _ts is not None and _tt is not None: _link["source"], _link["target"] = _ts, _tt from graphify.paths import write_json_atomic - write_json_atomic(_current_path, out_data, indent=2) + write_json_atomic(_current_path, out_data, compact=True) sys.exit(0) elif cmd == "merge-graphs": @@ -2916,7 +2916,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": out_data["hyperedges"] = merged.graph.get("hyperedges", []) out_path.parent.mkdir(parents=True, exist_ok=True) from graphify.paths import write_json_atomic as _wja - _wja(out_path, out_data, indent=2) + _wja(out_path, out_data, compact=True) print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") print(f"Written to: {out_path}") @@ -4618,7 +4618,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: _write_json_atomic( graph_json_path, {k: v for k, v in merged.items() if k not in _RUN_ONLY_KEYS}, - indent=2, + compact=True, ) try: # Record the scan root so a later build_merge / update runbook can diff --git a/graphify/export.py b/graphify/export.py index 7d51318d1d..3865550ef7 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -460,7 +460,7 @@ def _canonical(item: dict, lead: tuple[str, ...]) -> dict: data["built_at_commit"] = commit from graphify.paths import write_json_atomic # Atomic write: a crash/ENOSPC mid-write must not truncate a good graph.json. - write_json_atomic(output_path, data, indent=2) + write_json_atomic(output_path, data, compact=True) return True diff --git a/graphify/global_graph.py b/graphify/global_graph.py index cd1ea19244..1cf3fa970c 100644 --- a/graphify/global_graph.py +++ b/graphify/global_graph.py @@ -67,7 +67,7 @@ def _save_global_graph(G: nx.Graph) -> None: except TypeError: data = _jg.node_link_data(G) from graphify.paths import write_json_atomic - write_json_atomic(_GLOBAL_GRAPH, data, indent=2) + write_json_atomic(_GLOBAL_GRAPH, data, compact=True) def _file_hash(path: Path) -> str: diff --git a/graphify/paths.py b/graphify/paths.py index 0cf500bedb..2582ef1c10 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -188,12 +188,18 @@ def write_text_atomic_if_changed(path: "str | Path", text: str) -> bool: return True -def write_json_atomic(path: "str | Path", obj, *, indent: "int | None" = None, ensure_ascii: bool = True) -> None: +def write_json_atomic( + path: "str | Path", obj, *, indent: "int | None" = None, ensure_ascii: bool = True, compact: bool = False +) -> None: """Atomically write ``obj`` as JSON to ``path``, streaming the encode into the temp file rather than materializing the whole string first (matters for very large graphs). ``ensure_ascii`` mirrors ``json.dump`` so callers that emit raw - UTF-8 (non-ASCII labels/paths) keep byte-for-byte output. See :func:`_atomic_replace`.""" - _atomic_replace(path, lambda f: json.dump(obj, f, indent=indent, ensure_ascii=ensure_ascii)) + UTF-8 (non-ASCII labels/paths) keep byte-for-byte output. ``compact`` writes one + line with no padding after ``,``/``:`` (graph.json, #3413). See :func:`_atomic_replace`.""" + separators = (",", ":") if compact else None + _atomic_replace( + path, lambda f: json.dump(obj, f, indent=indent, ensure_ascii=ensure_ascii, separators=separators) + ) # Directory segments that, when they appear as a whole path component, mark the # whole path as a test location. Matched against path *segments* (not raw diff --git a/graphify/watch.py b/graphify/watch.py index 3509667418..80ce2d6784 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -1495,8 +1495,15 @@ def _report_for_compare(report_text: str) -> str: return re.sub(r"^- Built from commit: `[^`]+`\n?", "", report_text, flags=re.MULTILINE) +def _is_single_line(path: Path) -> bool: + """True when ``path`` holds one line (the compact graph.json layout, #3413).""" + with path.open("rb") as f: + f.readline() + return not f.readline() + + def _json_text(data: dict) -> str: - return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + return json.dumps(data, ensure_ascii=False, separators=(",", ":")) + "\n" def _stabilize_rebuild_cwd(watch_path: Path) -> bool: @@ -2127,6 +2134,10 @@ def _failed(f: str) -> bool: print(f"error: {exc}", file=sys.stderr) return False + # The equality skips below compare parsed data, not layout; an existing + # multi-line graph.json must still be rewritten compact (#3413). + legacy_layout = existing_graph.exists() and not _is_single_line(existing_graph) + _relativize_source_files(result, project_root, scope=watch_root) # Source files re-extracted this run — their symbol sets may legitimately # shrink (a removed function), so the shrink-guard should not block the @@ -2208,6 +2219,7 @@ def _failed(f: str) -> bool: ) except Exception: same_graph = False + same_graph = same_graph and not legacy_layout if not same_graph: if not _check_shrink( force, existing_graph_data, candidate_graph_data, @@ -2277,7 +2289,7 @@ def _failed(f: str) -> bool: # build_from_json defaults to directed=False otherwise. G = build_from_json(result, directed=bool((existing_graph_data or {}).get("directed", False))) candidate_topology = _topology_from_graph(G) - if existing_graph_data: + if existing_graph_data and not legacy_layout: try: same_topology = ( json.dumps(_canonical_topology_for_compare(existing_graph_data), sort_keys=True, ensure_ascii=False) @@ -2415,6 +2427,7 @@ def _failed(f: str) -> bool: ) except Exception: same_graph = False + same_graph = same_graph and not legacy_layout if report_path.exists(): old_report = report_path.read_text(encoding="utf-8") same_report = _report_for_compare(old_report) == _report_for_compare(report) diff --git a/tests/test_compact_graph_json.py b/tests/test_compact_graph_json.py new file mode 100644 index 0000000000..6673ef56cb --- /dev/null +++ b/tests/test_compact_graph_json.py @@ -0,0 +1,187 @@ +"""graph.json is one line of compact JSON (#3413). + +A pretty-printed graph.json makes every PR that commits a rebuilt graph report +tens of thousands of changed lines, and GitHub's line counts ignore +``.gitattributes``. Only the file's content can fix that, so every writer of a +project graph.json emits a single line, and ``update`` rewrites an existing +indented graph instead of treating it as unchanged. +""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import networkx as nx +import pytest + +from graphify.watch import _rebuild_code + +PYTHON = sys.executable + + +def _run(args, cwd): + return subprocess.run([PYTHON, "-m", "graphify"] + args, cwd=cwd, capture_output=True, text=True) + + +def _assert_one_line(path: Path): + text = path.read_text(encoding="utf-8") + assert len(text.splitlines()) == 1, f"{path.name} spans {len(text.splitlines())} lines" + # compact separators: no space after ',' or ':' outside string values + assert '": ' not in text and '", ' not in text and "}, " not in text + return json.loads(text) + + +def _node(nid: str, source_file: str = "a.py") -> dict: + return {"id": nid, "label": nid, "file_type": "code", "source_file": source_file} + + +def _graph_file(path: Path, node_ids, edges=(), *, indent=None) -> dict: + data = { + "directed": False, "multigraph": False, "graph": {}, + "nodes": [_node(n) for n in node_ids], + "links": [{"source": s, "target": t, "relation": "calls", "confidence": "EXTRACTED", + "source_file": "a.py"} for s, t in edges], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=indent), encoding="utf-8") + return data + + +def _corpus(tmp_path: Path) -> Path: + (tmp_path / "app.py").write_text( + "def alpha():\n return beta()\n\n\ndef beta():\n return 1\n", encoding="utf-8" + ) + return tmp_path + + +def test_to_json_writes_one_compact_line(tmp_path): + from graphify.export import to_json + + G = nx.Graph() + G.add_node("a", label="a", file_type="code", source_file="a.py", community=0) + G.add_node("b", label="b", file_type="code", source_file="a.py", community=0) + G.add_edge("a", "b", relation="calls", confidence="EXTRACTED", source_file="a.py") + out = tmp_path / "graph.json" + + assert to_json(G, {0: ["a", "b"]}, str(out), force=True) + + data = _assert_one_line(out) + assert {n["id"] for n in data["nodes"]} == {"a", "b"} + assert len(data["links"]) == 1 + + +@pytest.mark.parametrize("no_cluster", [False, True], ids=["clustered", "no-cluster"]) +def test_update_writes_one_compact_line(tmp_path, no_cluster): + corpus = _corpus(tmp_path) + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + + data = _assert_one_line(corpus / "graphify-out" / "graph.json") + labels = {n["label"] for n in data["nodes"]} + assert {"alpha()", "beta()"} <= labels + + +@pytest.mark.parametrize("no_cluster", [False, True], ids=["clustered", "no-cluster"]) +def test_update_rewrites_existing_indented_graph_compact(tmp_path, no_cluster): + """An unchanged graph still checked in with indent=2 must not stay that way + forever: the topology/graph-equality skips compare parsed data, not bytes.""" + corpus = _corpus(tmp_path) + graph = corpus / "graphify-out" / "graph.json" + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + before = _assert_one_line(graph) + + graph.write_text(json.dumps(before, indent=2) + "\n", encoding="utf-8") + assert len(graph.read_text(encoding="utf-8").splitlines()) > 1 + + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + + after = _assert_one_line(graph) + assert {n["id"] for n in after["nodes"]} == {n["id"] for n in before["nodes"]} + assert len(after["links"]) == len(before["links"]) + + +@pytest.mark.parametrize("no_cluster", [False, True], ids=["clustered", "no-cluster"]) +def test_update_leaves_compact_graph_untouched(tmp_path, no_cluster): + """The legacy rewrite must not make every update rewrite a compact graph.""" + corpus = _corpus(tmp_path) + graph = corpus / "graphify-out" / "graph.json" + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + before = graph.stat() + + assert _rebuild_code(corpus, no_cluster=no_cluster, acquire_lock=False) is True + + after = graph.stat() + assert (after.st_ino, after.st_mtime_ns) == (before.st_ino, before.st_mtime_ns), ( + "an already-compact, unchanged graph was rewritten" + ) + + +def test_merge_driver_output_is_one_line_union(tmp_path): + base, cur, oth = (tmp_path / n for n in ("base.json", "cur.json", "oth.json")) + _graph_file(base, ["a"]) + _graph_file(cur, ["a", "b"], [("a", "b")]) + _graph_file(oth, ["a", "c"], [("a", "c")]) + for p in (base, cur, oth): + p.write_text(json.dumps(json.loads(p.read_text(encoding="utf-8")), separators=(",", ":"))) + + r = _run(["merge-driver", str(base), str(cur), str(oth)], tmp_path) + + assert r.returncode == 0, r.stderr + merged = _assert_one_line(cur) + assert {n["id"] for n in merged["nodes"]} == {"a", "b", "c"} + assert {(l["source"], l["target"]) for l in merged["links"]} == {("a", "b"), ("a", "c")} + + +def test_merge_graphs_output_is_one_line(tmp_path): + a = tmp_path / "r1" / "graphify-out" / "graph.json" + b = tmp_path / "r2" / "graphify-out" / "graph.json" + _graph_file(a, ["x"]) + _graph_file(b, ["y"]) + out = tmp_path / "merged.json" + + r = _run(["merge-graphs", str(a), str(b), "--out", str(out)], tmp_path) + + assert r.returncode == 0, r.stderr + data = _assert_one_line(out) + assert {n["id"] for n in data["nodes"]} == {"r1::x", "r2::y"} + + +def test_prune_output_is_one_line(tmp_path): + from graphify.cli import _prune_graph_json_sources + + graph = tmp_path / "graph.json" + data = _graph_file(graph, ["keep", "drop"], indent=2) + data["nodes"][1]["source_file"] = "gone.py" + graph.write_text(json.dumps(data, indent=2), encoding="utf-8") + + assert _prune_graph_json_sources(graph, ["gone.py"]) == 1 + + pruned = _assert_one_line(graph) + assert [n["id"] for n in pruned["nodes"]] == ["keep"] + + +def test_extract_no_cluster_output_is_one_line(tmp_path): + (tmp_path / "corpus").mkdir() + corpus = _corpus(tmp_path / "corpus") + out = tmp_path / "scratch" + + r = _run(["extract", str(corpus), "--out", str(out), "--no-cluster", "--code-only"], tmp_path) + + assert r.returncode == 0, r.stderr + data = _assert_one_line(out / "graphify-out" / "graph.json") + assert {"alpha()", "beta()"} <= {n["label"] for n in data["nodes"]} + + +def test_global_graph_output_is_one_line(tmp_path, monkeypatch): + import graphify.global_graph as gg + + monkeypatch.setattr(gg, "_GLOBAL_DIR", tmp_path) + monkeypatch.setattr(gg, "_GLOBAL_GRAPH", tmp_path / "global-graph.json") + G = nx.Graph() + G.add_edge("r::a", "r::b", relation="calls") + + gg._save_global_graph(G) + + data = _assert_one_line(tmp_path / "global-graph.json") + assert {n["id"] for n in data["nodes"]} == {"r::a", "r::b"}