feat: make CLI and MCP output lean by default

README.md

@@ -629,7 +629,13 @@ Every MCP tool can be invoked as a local, one-shot command. CLI tools neither st
 
 Commands that mutate graph data use shared OS-backed, per-project locks. This serializes conflicting work from CLI and MCP sessions on the same project while allowing unrelated projects to proceed independently.
 
-When stderr is an interactive terminal, the CLI automatically shows lifecycle and indexing progress. Pass `--progress` to force the same feedback when stderr is redirected or the command is run non-interactively. Progress is written only to stderr; stdout remains reserved for the command result, so pipes and scripts stay machine-safe. Pass `--json` when the full MCP result envelope is needed.
+When stderr is an interactive terminal, the CLI automatically shows lifecycle and indexing progress. Pass `--progress` to force the same feedback when stderr is redirected or the command is run non-interactively. Pass `--quiet` to disable automatic terminal progress and ordinary diagnostics while retaining errors; it cannot be combined with `--progress` or outer `cli --verbose`. Routine informational logs are quiet by default; pass outer `cli --verbose` to include them. Progress and logs use stderr while stdout remains reserved for the command result. Read tools return a compact tree by default; pass a tool's `--format json` for machine-readable payload JSON, or outer `--json` for the full MCP envelope.
+
+Large compact-tree tables may start with a response-local `<section>_refs` directory and an explicit `<section>_ref_rule`. A cell such as `@0+handler.go` reconstructs to ref `0`'s prefix plus `handler.go`. References are local to that sibling `<section>` table and expansion is non-recursive: entries inside `<section>_refs` are always literal prefixes. This is limited to declared path and qualified-name columns and activates only when the exact rendered table is at least 15% and 64 bytes smaller and a conservative model-neutral token-shape proxy also improves by at least 1%. Search and trace likewise render direct and prefix-grouped tree shapes and keep the smaller complete representation, so singleton or scattered answers do not pay directory overhead. Keys are declared once per table but never cryptically abbreviated, and `--format json` keeps stable literal strings for machine consumers. Both gates are deterministic; exact token counts still depend on the caller's tokenizer.
+
+Lean responses truncate semantically, never by cutting arbitrary bytes from code or identifiers. Ranked graph rows are retained ahead of raw grep rows and diagnostic summaries; omitted rows/sections report totals, `has_more`, and a strictly advancing continuation offset or cursor. If even the first whole row cannot fit, CBM asks for a higher budget and emits no self-looping cursor. `max_output_tokens` is model-neutral sizing guidance: CBM enforces a deterministic ceiling of four UTF-8 bytes per requested token, so it is not a tokenizer-exact count. Detail flags such as `diagnostics`, `source_mode`, and `detail` opt into heavier fields. `search_code` pages ranked rows with `result_limit`/`result_offset` (`limit` remains a compatibility alias), raw rows with `raw_limit`/`raw_offset`, and directory summaries with `directory_limit`/`directory_offset`. Raw lines default to a UTF-8-safe match-centered preview; each row reports `content_start_byte`, returned/total byte counts, match byte bounds when known, and a content continuation offset. Pass `raw_content_offset` to page the original line without moving the raw-row cursor. `match_limit` and `source_max_lines` bound per-result details, with exact omission metadata. `detect_changes` pages changed files, impacted symbols, and module summaries independently; prefer its snapshot-bound `*_cursor` continuations, which reject changed commits, worktree bytes, graph generation, or semantic arguments instead of silently skipping or duplicating rows.
+
+Every response is standard UTF-8. Identifiers, paths, and raw search previews preserve POSIX byte-string identities: a preserved value containing malformed UTF-8 is emitted reversibly as `@bytes:<lowercase hex of every original byte>`. A valid preserved value that literally begins with the reserved `@bytes:` or `@utf8:` prefix is emitted as `@utf8:<original value>`, so decoding is unambiguous: strip one `@utf8:` prefix for literal UTF-8, or hex-decode one `@bytes:` prefix for original bytes. Ordinary valid UTF-8 is unchanged and pays no output-token overhead. To keep code readable, source bodies replace malformed UTF-8 with U+FFFD; use the pageable raw search preview when byte-exact source inspection is required.
 
 Use `cli <tool> --help` to see the flags generated from that tool's input schema:
 
@@ -644,7 +650,10 @@ codebase-memory-mcp cli query_graph --project my-project --query 'MATCH (f:Funct
 
 # Force human-readable progress without contaminating stdout.
 codebase-memory-mcp cli --progress index_repository --repo-path /path/to/repo
-codebase-memory-mcp cli search_graph --project my-project --label Function | jq '.results[].name'
+# Suppress automatic terminal progress and non-error diagnostics.
+codebase-memory-mcp cli --quiet list_projects --format json
+codebase-memory-mcp cli search_graph --project my-project --label Function --format json
+codebase-memory-mcp cli list_projects --format json --detail stats | jq '.projects[].name'
 ```
 
 JSON arguments can also be piped on stdin, for tools that take arguments. A tool whose input schema declares none — `list_projects` — never reads stdin, so it stays responsive when it inherits a pipe the caller never closes (the default for `child_process.spawn` and similar wrappers). Inline JSON remains accepted for backward compatibility but is deprecated in favor of flags, `--args-file`, or stdin.
@@ -664,7 +673,7 @@ JSON arguments can also be piped on stdin, for tools that take arguments. A tool
 
 | Tool | Description |
 |------|-------------|
-| `search_graph` | Structured search by label, name pattern, file pattern, degree filters. Pagination via limit/offset. |
+| `search_graph` | Structural, BM25, and semantic search. Page structural rows with `offset`/`limit` and ranked semantic rows independently with `semantic_offset`/`semantic_limit`. |
 | `trace_path` | BFS traversal — who calls a function and what it calls (alias: `trace_call_path`). Depth 1-5. |
 | `detect_changes` | Map git diff to affected symbols + blast radius with risk classification. |
 | `query_graph` | Execute Cypher-like graph queries (read-only). |
@@ -730,7 +739,7 @@ codebase-memory-mcp config reset auto_index              # reset to default
 | `CBM_CACHE_DIR` | `~/.cache/codebase-memory-mcp` | Override the database storage directory. All project indexes and config are stored here. One account can use only one canonical cache root at a time; close active CBM sessions/commands before switching it. |
 | `CBM_DIAGNOSTICS` | `false` | Set to `1` or `true` to enable the shared daemon's periodic `snapshot.json` and retained `trajectory.ndjson` below a fresh owner-private directory in the system temp directory. Exact paths are logged by `diagnostics.start`. |
 | `CBM_DOWNLOAD_URL` | *(GitHub releases)* | Override the download URL for updates. Used for testing or self-hosted deployments. |
-| `CBM_LOG_LEVEL` | `info` | Set the minimum log level. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none` — or their numeric equivalents `0`–`4` matching the internal enum. Thin-frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. |
+| `CBM_LOG_LEVEL` | role-aware | Set the minimum log level. Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and its supervised index workers default to `info` so lifecycle and liveness records remain available. Accepted values (case-insensitive): `debug`, `info`, `warn`, `error`, `none` — or their numeric equivalents `0`–`4`. A physical worker retains INFO liveness records even under a stricter override because its private log drives the supervisor's no-progress timeout. Frontend messages go to that session's stderr; detached daemon events go to `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. Stdout is reserved for MCP JSON-RPC. |
 | `CBM_WORKERS` | *(detected)* | Override the parallel-indexing worker count returned by `cbm_default_worker_count`. Useful inside containers where `sysconf(_SC_NPROCESSORS_ONLN)` reports host CPUs rather than the cgroup's effective quota. Range 1–256; invalid values are ignored with a warning. |
 | `CBM_MEM_BUDGET_MB` | *(detected)* | Override the in-memory graph budget with an explicit cap in MiB, taking precedence over the `ram_fraction × total_RAM` default. Useful on bare-metal hosts without a cgroup limit, or to pin a budget *below* the cgroup limit so headroom is left for sibling processes. Must be a positive integer; it is clamped to detected total RAM (logged as `mem.budget.clamped`), and non-numeric or non-positive values are ignored with a warning (`mem.budget.env.invalid`). |
 | `CBM_DUMP_VERIFY_MIN_RATIO` | `0.5` | After indexing, compare persisted SQLite node count to the in-memory dump count. When persisted nodes fall below this fraction of committed nodes (and committed > 50), `index_repository` returns `status:"degraded"` instead of silent `indexed`. Range 0–1; set `0` to disable. Invalid values are ignored with a warning. |

docs/CONFIGURATION.md

@@ -149,7 +149,7 @@ These environment variables affect runtime behavior:
 | `CBM_CACHE_DIR` | `~/.cache/codebase-memory-mcp` | Override the cache directory used for indexes, `_config.db`, and UI `config.json`. |
 | `CBM_DIAGNOSTICS` | `false` | Enable periodic `snapshot.json` and retained `trajectory.ndjson` below a fresh owner-private directory in the system temp directory. The daemon records the randomized paths in the `diagnostics.start` discovery record (a single JSON line) in `${CBM_CACHE_DIR}/logs/cbm-daemon.log`; that one record is emitted even when `CBM_LOG_LEVEL` suppresses ordinary logging, so the paths always remain discoverable. |
 | `CBM_DOWNLOAD_URL` | GitHub releases | Override the update download URL. |
-| `CBM_LOG_LEVEL` | `info` | Set the log level to `debug`, `info`, `warn`, `error`, or `none` (or `0`-`4`). Thin-frontend messages use that session's stderr; detached daemon events use `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. |
+| `CBM_LOG_LEVEL` | role-aware | Set the log level to `debug`, `info`, `warn`, `error`, or `none` (or `0`-`4`). Thin MCP/CLI/hook frontends default to `warn`; the detached daemon and supervised index workers default to `info`. Physical workers retain INFO liveness records because their private logs drive the supervisor's no-progress timeout. Frontend messages use that session's stderr; detached daemon events use `${CBM_CACHE_DIR}/logs/cbm-daemon.log`. |
 | `CBM_RUNTIME_DIR` | `%LOCALAPPDATA%` (Windows), `/private/tmp` (macOS), `/tmp` (other) | Parent directory for the daemon/CLI rendezvous directory, which CBM creates inside it as `cbm-daemon-<uid>` (`cbm-daemon-<key>` on Windows). Set it when the default ancestry cannot pass the private-directory check — see below. `CBM_CACHE_DIR` does **not** move the rendezvous. |
 | `CBM_WORKERS` | auto-detected | Override the indexing worker count. |
 

graph-ui/src/components/NodeDetailPanel.test.tsx

@@ -68,6 +68,12 @@ describe("NodeDetailPanel code preview + deep-link", () => {
     /* …but was NOT parsed into a real <script> element, and did not execute. */
     expect(container.querySelector("script")).toBeNull();
     expect((window as unknown as { __pwned?: boolean }).__pwned).toBeUndefined();
+    expect(callToolMock).toHaveBeenCalledWith("get_code_snippet", {
+      qualified_name: "app::render",
+      project: "demo",
+      format: "json",
+      source_mode: "full",
+    });
   });
 
   it("builds an https GitHub deep-link with URL-encoded path segments", () => {

graph-ui/src/components/NodeDetailPanel.tsx

@@ -76,6 +76,8 @@ export function NodeDetailPanel({
       const res = await callTool<SnippetResult>("get_code_snippet", {
         qualified_name: node.qualified_name,
         project,
+        format: "json",
+        source_mode: "full",
       });
       setCode(res.source ?? "(source not available)");
     } catch (e) {

graph-ui/src/components/StatsTab.test.tsx

@@ -12,7 +12,7 @@ function mockProjectsFetch(extra?: (url: string, init?: RequestInit) => Response
     if (overridden) return overridden;
     if (url === "/rpc") {
       return new Response(JSON.stringify({
-        result: { content: [{ text: JSON.stringify({ projects: [] }) }] },
+        result: { content: [{ text: JSON.stringify({ projects: [], has_more: false }) }] },
       }), { status: 200, headers: { "Content-Type": "application/json" } });
     }
     if (url.startsWith("/api/ui-config")) {
@@ -221,8 +221,11 @@ describe("StatsTab index modal", () => {
                 root_path: "/repo",
                 indexed_at: "2026-01-01T00:00:00Z",
               }],
+              has_more: false,
             }
-          : { node_labels: [], edge_types: [], total_nodes: 0, total_edges: 0 };
+          : {
+              node_labels: [], edge_types: [], total_nodes: 0, total_edges: 0, has_more: false,
+            };
         return new Response(JSON.stringify({
           result: { content: [{ text: JSON.stringify(result) }] },
         }), { status: 200, headers: { "Content-Type": "application/json" } });

graph-ui/src/hooks/useProjects.test.tsx

@@ -0,0 +1,86 @@
+/* @vitest-environment jsdom */
+import { renderHook, waitFor } from "@testing-library/react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { useProjects } from "./useProjects";
+
+const callToolMock = vi.fn();
+vi.mock("../api/rpc", () => ({
+  callTool: (...args: unknown[]) => callToolMock(...args),
+}));
+
+describe("useProjects machine-readable pagination", () => {
+  beforeEach(() => callToolMock.mockReset());
+
+  it("requests JSON and merges every project and schema page", async () => {
+    callToolMock.mockImplementation(async (name: string, args: Record<string, unknown>) => {
+      if (name === "list_projects") {
+        if (args.offset === 0) {
+          return {
+            projects: [{ name: "alpha", root_path: "/alpha", indexed_at: "now" }],
+            has_more: true,
+            next_offset: 1,
+          };
+        }
+        return {
+          projects: [{ name: "beta", root_path: "/beta", indexed_at: "now" }],
+          has_more: false,
+        };
+      }
+      if (name === "get_graph_schema" && args.project === "alpha") {
+        if (args.offset === 0) {
+          return {
+            node_labels: [{ label: "Function", count: 3 }],
+            edge_types: [],
+            total_nodes: 3,
+            total_edges: 2,
+            has_more: true,
+            next_offset: 1,
+          };
+        }
+        return {
+          node_labels: [],
+          edge_types: [{ type: "CALLS", count: 2 }],
+          total_nodes: 3,
+          total_edges: 2,
+          has_more: false,
+        };
+      }
+      return {
+        node_labels: [{ label: "Class", count: 1 }],
+        edge_types: [],
+        total_nodes: 1,
+        total_edges: 0,
+        has_more: false,
+      };
+    });
+
+    const { result } = renderHook(() => useProjects());
+    await waitFor(() => expect(result.current.loading).toBe(false));
+
+    expect(result.current.projects).toHaveLength(2);
+    expect(result.current.projects[0].schema?.node_labels).toEqual([
+      { label: "Function", count: 3 },
+    ]);
+    expect(result.current.projects[0].schema?.edge_types).toEqual([
+      { type: "CALLS", count: 2 },
+    ]);
+    expect(callToolMock).toHaveBeenCalledWith("list_projects", {
+      format: "json",
+      detail: "stats",
+      limit: 500,
+      offset: 0,
+    });
+    expect(callToolMock).toHaveBeenCalledWith("list_projects", {
+      format: "json",
+      detail: "stats",
+      limit: 500,
+      offset: 1,
+    });
+    expect(callToolMock).toHaveBeenCalledWith("get_graph_schema", {
+      project: "alpha",
+      format: "json",
+      limit: 500,
+      offset: 1,
+    });
+  });
+});

graph-ui/src/hooks/useProjects.ts

@@ -7,6 +7,70 @@ interface ProjectInfo {
   schema: SchemaInfo | null;
 }
 
+interface ProjectPage {
+  projects?: Project[];
+  has_more?: boolean;
+  next_offset?: number;
+}
+
+interface SchemaPage extends SchemaInfo {
+  has_more?: boolean;
+  next_offset?: number;
+}
+
+const PAGE_LIMIT = 500;
+
+function nextPageOffset(page: { has_more?: boolean; next_offset?: number }, offset: number) {
+  if (typeof page.has_more !== "boolean") {
+    throw new Error("Invalid pagination response");
+  }
+  if (!page.has_more) return null;
+  if (!Number.isInteger(page.next_offset) || page.next_offset! <= offset) {
+    throw new Error("Invalid pagination response");
+  }
+  return page.next_offset!;
+}
+
+async function fetchAllProjects(): Promise<Project[]> {
+  const projects: Project[] = [];
+  let offset = 0;
+  for (;;) {
+    const page = await callTool<ProjectPage>("list_projects", {
+      format: "json",
+      detail: "stats",
+      limit: PAGE_LIMIT,
+      offset,
+    });
+    projects.push(...(page.projects ?? []));
+    const next = nextPageOffset(page, offset);
+    if (next === null) return projects;
+    offset = next;
+  }
+}
+
+async function fetchFullSchema(project: string): Promise<SchemaInfo> {
+  const nodeLabels: SchemaInfo["node_labels"] = [];
+  const edgeTypes: SchemaInfo["edge_types"] = [];
+  let firstPage: SchemaPage | null = null;
+  let offset = 0;
+  for (;;) {
+    const page = await callTool<SchemaPage>("get_graph_schema", {
+      project,
+      format: "json",
+      limit: PAGE_LIMIT,
+      offset,
+    });
+    firstPage ??= page;
+    nodeLabels.push(...(page.node_labels ?? []));
+    edgeTypes.push(...(page.edge_types ?? []));
+    const next = nextPageOffset(page, offset);
+    if (next === null) {
+      return { ...firstPage, node_labels: nodeLabels, edge_types: edgeTypes };
+    }
+    offset = next;
+  }
+}
+
 interface UseProjectsResult {
   projects: ProjectInfo[];
   loading: boolean;
@@ -23,16 +87,13 @@ export function useProjects(): UseProjectsResult {
     setLoading(true);
     setError(null);
     try {
-      const result = await callTool<{ projects: Project[] }>("list_projects");
-      const list = result.projects ?? [];
+      const list = await fetchAllProjects();
 
       /* Fetch schema for each project */
       const infos: ProjectInfo[] = await Promise.all(
         list.map(async (p) => {
           try {
-            const schema = await callTool<SchemaInfo>("get_graph_schema", {
-              project: p.name,
-            });
+            const schema = await fetchFullSchema(p.name);
             return { project: p, schema };
           } catch {
             return { project: p, schema: null };