Views call graph: compact store (diet part 1)

.alfonso/release-notes/next/views-store-compact-payloads.md

@@ -0,0 +1 @@
+- With views on, each call-graph database is about 45% smaller (1.97 GB to 1.05 GB for AFT's own repository) and builds about 10% faster, with identical answers.

benchmarks/aft-search/slice-descriptors/land-views-store-compact-payloads.json

@@ -0,0 +1,8 @@
+{
+  "slice_class": "non_ranking",
+  "targeted_mechanism": "none",
+  "kind": "harness",
+  "fixtures": [
+    "harness-goldens"
+  ]
+}

crates/aft/Cargo.toml

@@ -203,6 +203,10 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["ring",
 tokio-util = { version = "0.7", features = ["io"] }
 futures-util = "0.3"
 async-compression = { version = "0.4", features = ["tokio", "gzip", "zlib", "brotli"] }
+# Deflate for the call-graph view's per-file JSON payloads. Already in the
+# build through async-compression, image and ureq; listed here to call it
+# synchronously.
+flate2 = "1.1"
 rcgen = "0.13"
 # Shared CortexKit-neutral path-canonicalization primitive (ProjectRootId).
 # Lives in github.com/cortexkit/commons (product-neutral, used without subc);

crates/aft/src/callgraph_store/mod.rs

@@ -13406,7 +13406,26 @@ pub(crate) fn initialize_schema(conn: &Connection) -> Result<()> {
         CREATE TABLE IF NOT EXISTS staging_ref_context (
             ref_id        TEXT PRIMARY KEY,
             caller_symbol TEXT
-        );",
+        );
+
+        -- Node ids hold 64 bits of a digest of file, kind and scoped name
+        -- (`node_id`). Two different symbols with the same id fail the write
+        -- instead of one silently replacing the other.
+        CREATE TRIGGER IF NOT EXISTS nodes_id_collision_on_insert BEFORE INSERT ON nodes
+        WHEN EXISTS (
+            SELECT 1 FROM nodes WHERE id = NEW.id AND (
+                file_path IS NOT NEW.file_path
+                OR scoped_name IS NOT NEW.scoped_name
+                OR kind IS NOT NEW.kind
+            )
+        )
+        BEGIN SELECT RAISE(ABORT, 'call-graph node id collision'); END;
+        CREATE TRIGGER IF NOT EXISTS nodes_id_collision_on_update
+        BEFORE UPDATE OF file_path, scoped_name, kind ON nodes
+        WHEN OLD.file_path IS NOT NEW.file_path
+            OR OLD.scoped_name IS NOT NEW.scoped_name
+            OR OLD.kind IS NOT NEW.kind
+        BEGIN SELECT RAISE(ABORT, 'call-graph node id collision'); END;",
     )?;
     // CREATE TABLE IF NOT EXISTS leaves older tables untouched. Upgrade the
     // column before creating indexes that reference it; readiness still forces
@@ -13684,7 +13703,10 @@ fn ensure_database_ready(conn: &Connection) -> Result<()> {
 ///   `super()`, TypeScript object and factory members, and Rust trait and
 ///   associated calls resolve to their real targets; Go, Python and TypeScript
 ///   no longer guess callers by method name alone.
-const BUILD_OUTPUT_VERSION: &str = "v23-resolver-precision";
+/// - v24: node ids carry 64 bits of their digest instead of 256, views stores
+///   spell them `vp`/`vt` plus the digest, and two indexes no statement used
+///   (edges by target node, dispatch hints by method name) are gone.
+const BUILD_OUTPUT_VERSION: &str = "v24-short-node-ids";
 
 const H_HEADER_LANGUAGE_META_KEY: &str = "h_header_language_revision";
 
@@ -13694,7 +13716,7 @@ mod h_header_invalidation_tests {
 
     #[test]
     fn h_header_legacy_migration_is_targeted_atomic_and_once() {
-        assert_eq!(BUILD_OUTPUT_VERSION, "v23-resolver-precision");
+        assert_eq!(BUILD_OUTPUT_VERSION, "v24-short-node-ids");
         let temp = tempfile::tempdir().unwrap();
         std::fs::create_dir_all(temp.path().join("root")).unwrap();
         // Parse counters key files by the canonical path the store reads them
@@ -14206,13 +14228,11 @@ const SECONDARY_INDEXES: &[(&str, &str)] = &[
         "file_dependencies(dep_file)",
     ),
     ("idx_edges_source_kind", "edges(source_node, kind)"),
-    ("idx_edges_target_kind", "edges(target_node, kind)"),
     (
         "idx_edges_target_file_symbol",
         "edges(target_file, target_symbol, kind)",
     ),
     ("idx_edges_ref_id", "edges(ref_id, kind)"),
-    ("idx_dispatch_hints_method", "dispatch_hints(method_name)"),
     ("idx_dispatch_hints_file", "dispatch_hints(file)"),
     (
         "idx_backend_file_state_file",
@@ -14224,10 +14244,20 @@ const SECONDARY_INDEXES: &[(&str, &str)] = &[
     ),
 ];
 
+/// Indexes older builds created that no statement uses any more: no query
+/// filters edges by target node or dispatch hints by method name. Dropping
+/// them keeps a store opened by this build from carrying their pages.
+const RETIRED_SECONDARY_INDEXES: &[&str] = &["idx_edges_target_kind", "idx_dispatch_hints_method"];
+
 fn create_secondary_indexes_sql() -> String {
-    SECONDARY_INDEXES
+    RETIRED_SECONDARY_INDEXES
         .iter()
-        .map(|(name, target)| format!("CREATE INDEX IF NOT EXISTS {name} ON {target};"))
+        .map(|name| format!("DROP INDEX IF EXISTS {name};"))
+        .chain(
+            SECONDARY_INDEXES
+                .iter()
+                .map(|(name, target)| format!("CREATE INDEX IF NOT EXISTS {name} ON {target};")),
+        )
         .collect::<Vec<_>>()
         .join("\n")
 }
@@ -15993,21 +16023,42 @@ fn selection_range(source: &str, scoped_name: &str, name: &str, fallback: &Range
     }
 }
 
+/// A node's id: its class prefix and the first 64 bits of a BLAKE3 digest of
+/// its file, kind and scoped name, as 16 hex digits.
+///
+/// Node ids are repeated in every ref, edge and index that names a caller or a
+/// target, so their length is a large share of the store. 64 bits keep them
+/// unique in practice; the store's insert and update triggers refuse two
+/// different symbols with the same id (`initialize_schema`) rather than merging
+/// them. Hex digits of a prefix sort as the full digest does, so ordering by id
+/// is unchanged.
 fn node_id(rel_path: &str, kind: &str, scoped_name: &str) -> String {
     if scoped_name == TOP_LEVEL_SYMBOL {
-        return format!("top:{}", hash_to_hex(blake3::hash(rel_path.as_bytes())));
+        return format!("top:{}", short_hash_hex(blake3::hash(rel_path.as_bytes())));
     }
     // Length-delimited fields avoid aliasing paths or names containing colons.
     let mut hash = blake3::Hasher::new();
     for field in [rel_path, kind, scoped_name] {
         hash.update(&(field.len() as u64).to_le_bytes());
         hash.update(field.as_bytes());
     }
-    format!("pos:symbol:{}", hash_to_hex(hash.finalize()))
+    format!("pos:symbol:{}", short_hash_hex(hash.finalize()))
 }
 
+/// A views store's node id: `vp` plus 16 hex digits for a symbol, `vt` plus
+/// 16 hex digits for a file's top level.
+///
+/// The digests are `node_id`'s. Views stores never hold `node_id`'s longer
+/// spellings, so the short class letters only have to keep the two classes
+/// apart and in the same order: symbol ids still sort before top-level ids, and
+/// ids of one class still sort by digest.
 pub(crate) fn view_node_id(rel_path: &str, kind: &str, scoped_name: &str) -> String {
-    format!("view:symbol:{}", node_id(rel_path, kind, scoped_name))
+    let id = node_id(rel_path, kind, scoped_name);
+    match (id.strip_prefix("pos:symbol:"), id.strip_prefix("top:")) {
+        (Some(digest), _) => format!("vp{digest}"),
+        (_, Some(digest)) => format!("vt{digest}"),
+        _ => unreachable!("node_id returns a symbol or top-level id"),
+    }
 }
 
 fn build_call_refs(
@@ -26182,6 +26233,11 @@ fn hash_to_hex(hash: blake3::Hash) -> String {
     hash.to_hex().to_string()
 }
 
+/// The first 64 bits of a digest as 16 hex digits.
+fn short_hash_hex(hash: blake3::Hash) -> String {
+    hash.to_hex()[..16].to_string()
+}
+
 fn hash_from_hex(value: &str) -> Option<blake3::Hash> {
     let bytes = hex_to_bytes(value)?;
     Some(blake3::Hash::from_bytes(bytes))

crates/aft/src/callgraph_store/position_identity_tests.rs

@@ -189,6 +189,66 @@ fn s4_two_scoped_names_share_a_range_without_identity_collision() {
     assert_ne!(nodes[0].id, nodes[1].id);
 }
 
+#[test]
+fn node_ids_are_short_digests_that_sort_like_the_full_digest() {
+    let id = node_id("a.rs", "function", "f");
+    let hex = id.strip_prefix("pos:symbol:").unwrap();
+    assert_eq!(hex.len(), 16);
+    assert!(hex.bytes().all(|byte| byte.is_ascii_hexdigit()));
+    let top = node_id("a.rs", "module", TOP_LEVEL_SYMBOL);
+    assert_eq!(top.strip_prefix("top:").unwrap().len(), 16);
+    // Ordering by the stored id must match ordering by the full digest it was
+    // cut from, since replies order nodes by id.
+    let mut ids = (0..200)
+        .map(|n| {
+            let mut hash = blake3::Hasher::new();
+            hash.update(n.to_string().as_bytes());
+            let full = hash.finalize();
+            (short_hash_hex(full), hash_to_hex(full))
+        })
+        .collect::<Vec<_>>();
+    ids.sort_by(|left, right| left.0.cmp(&right.0));
+    assert!(ids.windows(2).all(|pair| pair[0].1 < pair[1].1));
+}
+
+#[test]
+fn two_symbols_with_one_node_id_fail_the_write_instead_of_merging() {
+    let conn = Connection::open_in_memory().unwrap();
+    initialize_schema(&conn).unwrap();
+    let insert = |scoped_name: &str| {
+        conn.execute(
+            "INSERT INTO nodes(id, file_path, name, scoped_name, kind, start_line, start_col,
+                 end_line, end_col, range_ordinal, signature, exported, is_default_export,
+                 is_type_like, is_callgraph_entry_point, provenance, is_test)
+             VALUES('pos:symbol:0123456789abcdef', 'a.rs', ?1, ?1, 'function', 1, 0, 1, 1, 0,
+                 NULL, 0, 0, 0, 0, 'treesitter', 0)
+             ON CONFLICT(id) DO UPDATE SET name=excluded.name, scoped_name=excluded.scoped_name",
+            [scoped_name],
+        )
+    };
+    insert("f").unwrap();
+    // The same symbol again is an ordinary upsert.
+    insert("f").unwrap();
+    let error = insert("g").unwrap_err().to_string();
+    assert!(error.contains("call-graph node id collision"), "{error}");
+    let error = conn
+        .execute(
+            "INSERT OR REPLACE INTO nodes(id, file_path, name, scoped_name, kind, start_line,
+                 start_col, end_line, end_col, range_ordinal, signature, exported,
+                 is_default_export, is_type_like, is_callgraph_entry_point, provenance, is_test)
+             VALUES('pos:symbol:0123456789abcdef', 'b.rs', 'f', 'f', 'function', 9, 0, 9, 1, 0,
+                 NULL, 0, 0, 0, 0, 'treesitter', 0)",
+            [],
+        )
+        .unwrap_err()
+        .to_string();
+    assert!(error.contains("call-graph node id collision"), "{error}");
+    let scoped: String = conn
+        .query_row("SELECT scoped_name FROM nodes", [], |row| row.get(0))
+        .unwrap();
+    assert_eq!(scoped, "f");
+}
+
 #[test]
 fn s4_node_identity_uses_path_kind_and_scoped_name_only() {
     assert_ne!(
@@ -307,7 +367,7 @@ pub(crate) fn assert_current_graph(conn: &Connection) {
         ("edges", "target_node"),
         ("dispatch_hints", "caller_node"),
     ] {
-        let old: usize = conn.query_row(&format!("SELECT count(*) FROM {table} WHERE {column} IS NOT NULL AND {column} NOT LIKE 'top:%' AND {column} NOT LIKE 'pos:symbol:%' AND {column} NOT LIKE 'view:symbol:%'"), [], |r| r.get(0)).unwrap();
+        let old: usize = conn.query_row(&format!("SELECT count(*) FROM {table} WHERE {column} IS NOT NULL AND {column} NOT LIKE 'top:%' AND {column} NOT LIKE 'pos:symbol:%' AND {column} NOT LIKE 'vp%' AND {column} NOT LIKE 'vt%'"), [], |r| r.get(0)).unwrap();
         assert_eq!(old, 0, "old identity in {table}.{column}");
     }
 }
@@ -409,7 +469,7 @@ pub(crate) fn with_pre_s4_project<T>(root: &Path, run: impl FnOnce() -> T) -> T
 
 pub(crate) fn seed_old_dispatch_hint(conn: &Connection) {
     conn.execute("INSERT INTO dispatch_hints SELECT 'pre-s4-hint', 'unique', id, file_path, start_line, 0, 0, 'treesitter' FROM nodes ORDER BY id LIMIT 1", []).unwrap();
-    assert!(conn.query_row("SELECT count(*) FROM dispatch_hints WHERE caller_node NOT LIKE 'pos:symbol:%' AND caller_node NOT LIKE 'view:symbol:%'", [], |r| r.get::<_,usize>(0)).unwrap()>0);
+    assert!(conn.query_row("SELECT count(*) FROM dispatch_hints WHERE caller_node NOT LIKE 'pos:symbol:%' AND caller_node NOT LIKE 'vp%' AND caller_node NOT LIKE 'vt%'", [], |r| r.get::<_,usize>(0)).unwrap()>0);
 }
 
 #[test]

crates/aft/src/views/materialization.rs

@@ -131,6 +131,7 @@ pub(crate) fn apply_manifest_diff_profiled(
 mod module_digest;
 #[cfg(test)]
 pub(crate) mod parity;
+pub(crate) mod payload;
 pub(crate) mod profile;
 mod resolution_facts;
 mod stable_rows;
@@ -139,7 +140,7 @@ mod stable_rows;
 // per-reference skip optimization when reference ordinals collide, or Go
 // bindings whose package file listings were not recorded (an added sibling
 // `.go` file would never reselect them).
-pub(crate) const MATERIALIZATION_VERSION: &str = "12";
+pub(crate) const MATERIALIZATION_VERSION: &str = "13";
 
 const BASE_FINGERPRINT_MISMATCH: &str =
     "derived manifest fingerprint mismatch; cold materialization required";
@@ -432,11 +433,11 @@ fn materialize_reading(
     transaction.execute_batch(
         "CREATE TABLE IF NOT EXISTS view_bindings (
              file_path TEXT PRIMARY KEY,
-             payload TEXT NOT NULL
+             payload BLOB NOT NULL
          );
          CREATE TABLE IF NOT EXISTS view_file_surfaces (
              file_path TEXT PRIMARY KEY,
-             payload TEXT NOT NULL
+             payload BLOB NOT NULL
          )",
     )?;
     transaction.execute_batch(module_digest::CREATE_TABLES)?;
@@ -947,7 +948,11 @@ fn materialize_reading(
                 .surface_json()
                 .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?
             {
-                surface_rows.push((path.clone(), payload, existing_surface_paths.contains(path)));
+                surface_rows.push((
+                    path.clone(),
+                    payload::pack(&payload),
+                    existing_surface_paths.contains(path),
+                ));
             }
         }
         let old = cached.get(path).filter(|_| {
@@ -985,7 +990,7 @@ fn materialize_reading(
         let replaced = old.is_some() || existing_binding_paths.contains(path);
         let payload = serde_json::to_string(binding)
             .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
-        binding_inserts.push((path.clone(), payload, replaced));
+        binding_inserts.push((path.clone(), payload::pack(&payload), replaced));
     }
     let surface_updates = surface_rows
         .iter()
@@ -1755,14 +1760,15 @@ fn load_bindings(
     let mut statement = connection.prepare("SELECT file_path, payload FROM view_bindings")?;
     let rows = statement
         .query_map([], |row| {
-            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
+            Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))
         })?
         .collect::<std::result::Result<Vec<_>, _>>()?;
     let mut bindings = rows
         .into_par_iter()
         .map(|(path, payload)| {
-            let binding: join::ViewBindingDependencies = serde_json::from_str(&payload)
-                .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
+            let binding: join::ViewBindingDependencies =
+                serde_json::from_str(&payload::unpack(&payload)?)
+                    .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
             Ok((path, binding))
         })
         .collect::<Result<BTreeMap<_, _>>>()?;
@@ -1780,13 +1786,14 @@ fn load_surfaces(
     let mut statement = connection.prepare("SELECT file_path, payload FROM view_file_surfaces")?;
     let rows = statement
         .query_map([], |row| {
-            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
+            Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))
         })?
         .collect::<std::result::Result<Vec<_>, _>>()?;
     rows.into_par_iter()
         .map(|(path, surface)| {
-            let binding = join::ViewBindingDependencies::from_surface_json(&surface)
-                .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
+            let binding =
+                join::ViewBindingDependencies::from_surface_json(&payload::unpack(&surface)?)
+                    .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
             Ok((path, binding))
         })
         .collect()
@@ -1826,14 +1833,15 @@ fn load_bindings_for_selection(
         )?;
         let rows = statement
             .query_map([], |row| {
-                Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
+                Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))
             })?
             .collect::<std::result::Result<Vec<_>, _>>()?;
         let selected = rows
             .into_par_iter()
             .map(|(path, payload)| {
-                let binding: join::ViewBindingDependencies = serde_json::from_str(&payload)
-                    .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
+                let binding: join::ViewBindingDependencies =
+                    serde_json::from_str(&payload::unpack(&payload)?)
+                        .map_err(|error| CallGraphStoreError::Unavailable(error.to_string()))?;
                 Ok((path, binding))
             })
             .collect::<Result<Vec<_>>>()?;