From ea66a43a29f369d306894fd245af0e7a322aa779 Mon Sep 17 00:00:00 2001 From: doubleinfinity <6169958+doubleinfinity@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:49:22 +1100 Subject: [PATCH] fix: allocate upper list slots on a load only for lists the dump fills A load gives a wide list to an upper list at or below the level of its node only if the dump fills that list. A wide list has a descriptor and m + 1 target slots. An empty list of this type is one empty word, as an empty list above the level of its node is. Thus a level in the layer table uses one word for each layer and no slots. The in-degree pass of the load promotes an empty word to a wide list when an entry at that layer refers to its node. The inbound counter of a list is on its wide descriptor. Insertion opens a wide list for each layer of a new node up to its level. Insertion promotes an empty word when it first counts into the word. The reverse update puts an entry into a list only after this count. Thus an empty word at or below the level of its node never holds an entry. The storage that a load makes follows the node count, m and the entries in the dump. A dump that keeps no list empty at or below a level loads to the same graph as before. Search and insertion on a loaded graph give the same results as before. --- crates/core/src/graph/dump.rs | 516 +++++++++++++++++++- crates/core/src/graph/mutable/mod.rs | 64 ++- crates/core/src/graph/structure.rs | 9 +- crates/hnsw/src/collection/persist_tests.rs | 234 ++++++++- 4 files changed, 806 insertions(+), 17 deletions(-) diff --git a/crates/core/src/graph/dump.rs b/crates/core/src/graph/dump.rs index b613413..5bf7321 100644 --- a/crates/core/src/graph/dump.rs +++ b/crates/core/src/graph/dump.rs @@ -1628,7 +1628,7 @@ mod tests { use super::*; use crate::distance::CosineDist; use crate::storage::FsDir; - use std::collections::BTreeMap; + use std::collections::{BTreeMap, BTreeSet}; /// Build a small graph the tests can round trip. fn sample_graph( @@ -2875,4 +2875,518 @@ mod tests { assert_eq!((built.m(), built.dim()), (256, 1)); round_trips_at_its_counter(&built, &built_store); } + + /// What the graph a load builds from `parsed` holds, from the dump's + /// content alone. + struct LoadedShape { + /// Nodes, each with its per node arrays and its layer zero slots. + nodes: usize, + /// The largest origin id, which the id-to-node array reaches. + largest_id: usize, + /// Upper words, being each node's span, the higher of its level and + /// the highest layer its lists fill, and the run the in-degree pass + /// appends whenever an entry names a node above its span. + words: usize, + /// Wide lists, being every list above layer zero that the dump fills + /// at or below its owner's level or fills with two or more entries + /// above it, and every node an entry names above layer zero, each node + /// and layer once. + wide: usize, + /// The wide lists the graph would hold were every list at or below + /// its owner's level opened wide whatever it holds. + wide_if_every_owned_list_were: usize, + } + + fn loaded_shape(parsed: &ParsedDump) -> LoadedShape { + let mut first = Vec::with_capacity(parsed.points_by_layer.len()); + let mut points: Vec<&LoadedPoint> = Vec::new(); + let mut levels: Vec = Vec::new(); + for (layer, layer_points) in parsed.points_by_layer.iter().enumerate() { + first.push(points.len()); + for point in layer_points { + points.push(point); + levels.push(layer); + } + } + let node = |target: PointId| first[target.0 as usize] + target.1 as usize; + let mut spans: Vec = points + .iter() + .zip(&levels) + .map(|(point, &level)| { + point + .neighbours + .iter() + .rposition(|list| !list.is_empty()) + .map_or(level, |highest| highest.max(level)) + }) + .collect(); + let mut words: usize = spans.iter().sum(); + let mut wide = BTreeSet::new(); + for (owner, point) in points.iter().enumerate() { + for (layer, list) in point.neighbours.iter().enumerate().skip(1) { + if !list.is_empty() && (layer <= levels[owner] || list.len() >= 2) { + wide.insert((owner, layer)); + } + } + } + // The in-degree pass, in its order: every entry of every list up to + // the span its owner holds when the pass reaches it. + for owner in 0..points.len() { + for layer in 0..=spans[owner] { + for edge in points[owner].neighbours.get(layer).into_iter().flatten() { + let target = node(edge.target); + if layer > spans[target] { + spans[target] = layer; + words += layer; + } + if layer > 0 { + wide.insert((target, layer)); + } + } + } + } + let mut owned = wide.clone(); + for (owner, &level) in levels.iter().enumerate() { + for layer in 1..=level { + owned.insert((owner, layer)); + } + } + LoadedShape { + nodes: points.len(), + largest_id: points + .iter() + .map(|point| point.origin_id) + .max() + .unwrap_or(0), + words, + wide: wide.len(), + wide_if_every_owned_list_were: owned.len(), + } + } + + /// The bytes the graph a load builds from `parsed` asks the allocator + /// for. Each node holds its origin id, level, layer zero length and + /// counter, first word and span, and `2m + 1` layer zero slots, the + /// id-to-node array four bytes an id up to the largest, each upper word + /// four bytes, and each wide list a ten byte descriptor and `m + 1` slots. + fn loaded_bytes(parsed: &ParsedDump) -> usize { + let shape = loaded_shape(parsed); + let m = parsed.m; + std::mem::size_of::>() + + shape.nodes * (8 * m + 24) + + 4 * (shape.largest_id + 1) + + 4 * shape.words + + (4 * m + 14) * shape.wide + } + + /// One node inserted alone at the top layer loads to the graph its save + /// wrote, holding a word a layer and no wide list. + /// + /// Insertion opens the node's lists at every layer up to its level wide, + /// because install site 2 fills them wholesale, and with no other node + /// nothing fills them, so the save records no list for it. A load opens + /// each as an empty word. The graph holds the built one's lists, its + /// storage is what the dump's content gives, it is the same graph read at + /// the largest counter, and it writes the same bytes again. + #[test] + fn a_node_alone_at_the_top_layer_loads_to_the_graph_its_save_wrote() { + let top = NB_LAYER_MAX as usize - 1; + let scale = super::super::levels::LevelGenerator::default_scale(16); + let (mut built, mut built_store) = + MutableGraph::new(4, 16, 64, scale, 1, CosineDist {}).unwrap(); + let vector = [0.1f32, 0.2, 0.3, 0.4]; + let planned = built.plan_insertion(&built_store, &vector, top); + built.install_insertion(&mut built_store, &vector, 1, planned); + assert_eq!(built.level(0) as usize, top); + assert_eq!(built.nb_wide_lists(), top); + + let dir = tempfile::tempdir().unwrap(); + write_dump( + &built.dump_view(&built_store), + GraphKind::Cosine, + &FsDir::new(dir.path()), + DUMP_FILENAME, + ) + .unwrap(); + let expected = Expected { + max_origin_id: 1, + ..expected_for(&built, 4, 1) + }; + let ((read, read_store), (reference, reference_store)) = read_both(dir.path(), &expected); + assert_eq!(read.arrays(), reference.arrays()); + assert_eq!(store_bytes(&read_store), store_bytes(&reference_store)); + assert_eq!(topology(&built, &built_store), topology(&read, &read_store)); + assert_eq!(read.entry_level() as usize, top); + assert_eq!((read.nb_upper_lists(), read.nb_wide_lists()), (top, 0)); + let parsed = parse_dump::(&FsDir::new(dir.path()), DUMP_FILENAME, &expected).unwrap(); + assert_eq!(loaded_shape(&parsed).wide_if_every_owned_list_were, top); + assert_eq!(read.memory_bytes(), loaded_bytes(&parsed)); + + let largest = Expected { + max_origin_id: u32::MAX as usize, + ..expected + }; + let (at_largest, _) = read_dump::( + &FsDir::new(dir.path()), + DUMP_FILENAME, + &largest, + CosineDist {}, + ) + .unwrap(); + assert_eq!(at_largest.arrays(), read.arrays()); + + let again = tempfile::tempdir().unwrap(); + write_dump( + &read.dump_view(&read_store), + GraphKind::Cosine, + &FsDir::new(again.path()), + DUMP_FILENAME, + ) + .unwrap(); + assert_eq!( + std::fs::read(dir.path().join(DUMP_FILENAME)).unwrap(), + std::fs::read(again.path().join(DUMP_FILENAME)).unwrap() + ); + } + + /// Lists at or below their owners' levels that the dump leaves empty load + /// as words, and as wide lists where an entry names their owner. + /// + /// The deep graph a small `m` builds holds such lists, at layers no node + /// had reached when their owner arrived. Insertion opened them wide. A + /// load opens each as an empty word and the in-degree pass promotes the + /// ones an entry names, since the counter lives on the wide descriptor, + /// so the load holds one wide list fewer for each empty list nothing + /// names. Its storage is what the dump's content gives, it holds the + /// written graph's topology, and it writes the same bytes again. + #[test] + fn empty_lists_at_or_below_a_level_load_as_words_unless_an_entry_names_them() { + let (built, built_store) = sample_graph(300, 4, 2); + let (mut named, mut unnamed) = (0usize, 0usize); + for layer in 1..NB_LAYER_MAX as usize { + let counted = built.counted_in_degree(layer); + for node in 0..built.nb_points() as u32 { + if layer <= built.level(node) as usize + && built.neighbours_at(node, layer).is_empty() + { + if counted[node as usize] > 0 { + named += 1; + } else { + unnamed += 1; + } + } + } + } + assert!( + named > 0 && unnamed > 0, + "{} empty lists an entry names and {} nothing names", + named, + unnamed + ); + + let dir = tempfile::tempdir().unwrap(); + write_dump( + &built.dump_view(&built_store), + GraphKind::Cosine, + &FsDir::new(dir.path()), + DUMP_FILENAME, + ) + .unwrap(); + let expected = expected_for(&built, 4, 300); + let ((read, read_store), (reference, _)) = read_both(dir.path(), &expected); + assert_eq!(read.arrays(), reference.arrays()); + assert_eq!(topology(&built, &built_store), topology(&read, &read_store)); + let parsed = parse_dump::(&FsDir::new(dir.path()), DUMP_FILENAME, &expected).unwrap(); + let shape = loaded_shape(&parsed); + assert_eq!(read.nb_wide_lists(), shape.wide); + assert_eq!(shape.wide_if_every_owned_list_were - shape.wide, unnamed); + assert_eq!(read.nb_upper_lists(), shape.words); + assert_eq!(read.memory_bytes(), loaded_bytes(&parsed)); + + let again = tempfile::tempdir().unwrap(); + write_dump( + &read.dump_view(&read_store), + GraphKind::Cosine, + &FsDir::new(again.path()), + DUMP_FILENAME, + ) + .unwrap(); + assert_eq!( + std::fs::read(dir.path().join(DUMP_FILENAME)).unwrap(), + std::fs::read(again.path().join(DUMP_FILENAME)).unwrap() + ); + } + + /// A node the dump files at the top layer with no entry above layer zero + /// costs a word a layer and no wide list, at the largest `m` and the + /// smallest dimension, whatever counter the dump is read at. + /// + /// Three nodes are added at the top layer of a dump a save wrote, under + /// fresh ids, each a copy of a point's vector carrying one layer zero + /// entry at the distance a save records. Their lists above layer zero are + /// empty and nothing names them there, so the load gives them fifteen + /// words each and no wide list, where opening every list at or below a + /// level wide gives them fifteen wide lists each. The graph's storage is + /// what the dump's content gives, the same at the tightest counter and at + /// the largest, and a save of it writes the forged file again. + #[test] + fn a_node_filed_high_with_no_upper_entry_costs_a_word_a_layer() { + let top = NB_LAYER_MAX as usize - 1; + let (built, built_store) = sample_graph(300, 1, 256); + assert_eq!((built.m(), built.dim()), (256, 1)); + let expected = expected_for(&built, 1, 300); + let parsed = parse_written(&built, &built_store, &expected); + let plain = written(&parsed); + let (base, _) = read_dump::( + &FsDir::new(plain.path()), + DUMP_FILENAME, + &expected, + CosineDist {}, + ) + .unwrap(); + + let mut forged = parsed.clone(); + let donor = forged.points_by_layer[0][0].clone(); + for k in 0..3usize { + let target = &forged.points_by_layer[0][k + 1]; + let mut point = donor.clone(); + point.origin_id = 300 + k; + point.neighbours = vec![vec![LoadedEdge { + target: PointId(0, k as i32 + 1), + distance: CosineDist {}.eval(&donor.data, &target.data), + }]]; + forged.points_by_layer[top].push(point); + } + let dir = written(&forged); + let tight = Expected { + max_origin_id: 302, + ..expected + }; + let ((read, read_store), (reference, _)) = read_both(dir.path(), &tight); + assert_eq!(read.arrays(), reference.arrays()); + assert_eq!(read.nb_points(), 303); + assert_eq!(read.nb_wide_lists(), base.nb_wide_lists()); + assert_eq!(read.nb_upper_lists(), base.nb_upper_lists() + 3 * top); + let reparsed = parse_dump::(&FsDir::new(dir.path()), DUMP_FILENAME, &tight).unwrap(); + let shape = loaded_shape(&reparsed); + assert_eq!(shape.wide_if_every_owned_list_were - shape.wide, 3 * top); + assert_eq!(read.memory_bytes(), loaded_bytes(&reparsed)); + + let largest = Expected { + max_origin_id: u32::MAX as usize, + ..tight + }; + let (at_largest, _) = read_dump::( + &FsDir::new(dir.path()), + DUMP_FILENAME, + &largest, + CosineDist {}, + ) + .unwrap(); + assert_eq!(at_largest.arrays(), read.arrays()); + + let again = tempfile::tempdir().unwrap(); + write_dump( + &read.dump_view(&read_store), + GraphKind::Cosine, + &FsDir::new(again.path()), + DUMP_FILENAME, + ) + .unwrap(); + assert_eq!( + std::fs::read(dir.path().join(DUMP_FILENAME)).unwrap(), + std::fs::read(again.path().join(DUMP_FILENAME)).unwrap() + ); + } + + /// A loaded graph holding empty lists at or below its levels, changed by + /// insertion, gives the dumps and the pages the graph that wrote it gives + /// after the same insertions. + /// + /// Insertion does not depend on the form a list takes, and a load builds + /// the lists and the inbound counts of the graph that wrote the dump, so + /// the graph that never left memory is what the loaded one has to match. + /// Both take the same insertions at chosen levels, since a loaded graph + /// draws from a fresh stream, reaching every layer the graph holds and two + /// above it, so the lists the load held as words are counted into and + /// filled. Before the insertions and after them, every inbound counter of + /// each graph is the count of the lists naming its node, and the two hold + /// the same counts. After each insertion the two dumps are the same bytes. + /// Then the two answer the same pages, unfiltered and with ids filtered out + /// as a removal does, they evicted the same entries, a graph rebuilt from + /// the live points of each is the same graph, and each written and read + /// back is the same graph answering the same pages. + #[test] + fn a_loaded_graph_changed_after_the_load_matches_the_graph_that_wrote_it() { + type Graph = MutableGraph; + type Store = super::super::store::VectorStore; + fn dump_of(graph: &Graph, store: &Store) -> Vec { + let dir = tempfile::tempdir().unwrap(); + write_dump( + &graph.dump_view(store), + GraphKind::Cosine, + &FsDir::new(dir.path()), + DUMP_FILENAME, + ) + .unwrap(); + std::fs::read(dir.path().join(DUMP_FILENAME)).unwrap() + } + fn spread(state: &mut u64) -> Vec { + (0..4) + .map(|_| { + *state ^= *state << 13; + *state ^= *state >> 7; + *state ^= *state << 17; + (*state >> 40) as f32 / 16_777_216.0 - 0.5 + }) + .collect() + } + // The lists naming each node at each layer, by origin id and layer, + // counted from the lists themselves. + fn counted(graph: &Graph) -> BTreeMap<(usize, usize), u32> { + let mut out = BTreeMap::new(); + for layer in 0..NB_LAYER_MAX as usize { + for (node, &count) in graph.counted_in_degree(layer).iter().enumerate() { + if count > 0 { + out.insert((graph.origin_id_of(node as u32), layer), count); + } + } + } + out + } + // Every id that is not three more than a multiple of seven, as a + // removal leaves the live set. + let live = |id: &usize| id % 7 != 3; + let pages = |graph: &Graph, store: &Store| -> Vec> { + let mut state = 0xbb67_ae85_84ca_a73bu64; + let mut out = Vec::new(); + for _ in 0..24 { + let query = spread(&mut state); + for page in [ + graph.search(store, &query, 10, 32, None::<&fn(&usize) -> bool>), + graph.search(store, &query, 10, 32, Some(&live)), + ] { + out.push( + page.iter() + .map(|hit| (hit.internal_id, hit.distance.to_bits())) + .collect(), + ); + } + } + out + }; + + let (mut built, mut built_store) = sample_graph(300, 4, 2); + let dir = tempfile::tempdir().unwrap(); + write_dump( + &built.dump_view(&built_store), + GraphKind::Cosine, + &FsDir::new(dir.path()), + DUMP_FILENAME, + ) + .unwrap(); + let (mut read, mut read_store): (Graph, Store) = read_dump( + &FsDir::new(dir.path()), + DUMP_FILENAME, + &expected_for(&built, 4, 300), + CosineDist {}, + ) + .unwrap(); + // The lists the load holds as words at or below their owners' + // levels, by origin id and layer. + let mut words: Vec<(usize, usize)> = Vec::new(); + for layer in 1..NB_LAYER_MAX as usize { + let counted = read.counted_in_degree(layer); + for node in 0..read.nb_points() as u32 { + if layer <= read.level(node) as usize + && read.neighbours_at(node, layer).is_empty() + && counted[node as usize] == 0 + { + words.push((read.origin_id_of(node), layer)); + } + } + } + assert!(!words.is_empty(), "the load holds no list as a word"); + assert_eq!(read.in_degrees(), counted(&read)); + assert_eq!(read.in_degrees(), built.in_degrees()); + let (built_guard, read_guard) = (built.guard_stats(), read.guard_stats()); + + let top = built.entry_level() as usize; + let mut state = 0x6a09_e667_f3bc_c909u64; + for k in 0..48usize { + let vector = spread(&mut state); + let level = (k * 7) % (top + 3); + for (graph, store) in [(&mut built, &mut built_store), (&mut read, &mut read_store)] { + let planned = graph.plan_insertion(store, &vector, level); + graph.install_insertion(store, &vector, 300 + k, planned); + } + assert_eq!( + dump_of(&built, &built_store), + dump_of(&read, &read_store), + "after insertion {} at level {}", + k, + level + ); + } + let filled = words + .iter() + .filter(|&&(id, layer)| { + let node = read.node_of(id).expect("every origin id has a node"); + !read.neighbours_at(node, layer).is_empty() + }) + .count(); + assert!( + filled > 0, + "no insertion reached a list the load held as a word" + ); + assert_eq!(built.in_degrees(), counted(&built)); + assert_eq!(read.in_degrees(), counted(&read)); + assert_eq!(read.in_degrees(), built.in_degrees()); + + let evicted = |now: (u64, u64, u64), then: (u64, u64, u64)| { + (now.0 - then.0, now.1 - then.1, now.2 - then.2) + }; + let built_evictions = evicted(built.guard_stats(), built_guard); + assert_eq!(built_evictions, evicted(read.guard_stats(), read_guard)); + assert!(built_evictions.0 > 0, "no insertion overflowed a list"); + let built_pages = pages(&built, &built_store); + assert_eq!(built_pages, pages(&read, &read_store)); + + let rebuilt = |graph: &Graph, store: &Store| { + let scale = super::super::levels::LevelGenerator::default_scale(2); + let mut levels = + super::super::levels::LevelGenerator::new(scale, NB_LAYER_MAX as usize); + let (mut fresh, mut fresh_store) = + MutableGraph::new(4, 2, 64, scale, 1, CosineDist {}).unwrap(); + for id in (0..348usize).filter(live) { + let node = graph.node_of(id).expect("every inserted id has a node"); + fresh.insert(&mut fresh_store, store.get(node), id, &mut levels); + } + dump_of(&fresh, &fresh_store) + }; + assert_eq!(rebuilt(&built, &built_store), rebuilt(&read, &read_store)); + + let reread = |graph: &Graph, store: &Store| { + let dir = tempfile::tempdir().unwrap(); + write_dump( + &graph.dump_view(store), + GraphKind::Cosine, + &FsDir::new(dir.path()), + DUMP_FILENAME, + ) + .unwrap(); + read_dump::( + &FsDir::new(dir.path()), + DUMP_FILENAME, + &expected_for(graph, 4, 348), + CosineDist {}, + ) + .unwrap() + }; + let (built_again, built_again_store) = reread(&built, &built_store); + let (read_again, read_again_store) = reread(&read, &read_store); + assert_eq!(built_again.arrays(), read_again.arrays()); + assert_eq!(pages(&built_again, &built_again_store), built_pages); + assert_eq!(pages(&read_again, &read_again_store), built_pages); + } } diff --git a/crates/core/src/graph/mutable/mod.rs b/crates/core/src/graph/mutable/mod.rs index f6c5e4c..dfc9b54 100644 --- a/crates/core/src/graph/mutable/mod.rs +++ b/crates/core/src/graph/mutable/mod.rs @@ -94,6 +94,17 @@ //! no traversal reaches a node above its level except through the entry point //! chain, so it stays one word for the life of the graph. //! +//! A loaded node's list at or below its level that the dump leaves empty is +//! one empty word too. Install site 2 writes only the new node's lists, so +//! nothing fills such a list wholesale after a load, and it is promoted the +//! first time its owner is named at that layer, which is before anything is +//! pushed into it, exactly as a list above the level is. So a list at or below +//! its owner's level is wide unless it holds no entry and no list names its +//! owner there, and the storage a load builds follows the entries the dump +//! holds rather than the levels it declares: a declared level costs a word a +//! layer, and a wide list exists only where the dump fills a list or an entry +//! names its owner. +//! //! Keeping these lists is not a choice. The vendored //! descent increments the pivot's inbound counter at the layer it files the edge //! at, and the overflow pop guard reads that counter, so a build that skips them @@ -637,7 +648,10 @@ where /// /// The span is the highest layer the point carries an entry at, and at /// least its own level, so a node carrying entries above its level owns - /// lists there, as a built graph's nodes do. A list above its owner's + /// lists there, as a built graph's nodes do. A list at or below its + /// owner's level that the point fills is wide, as insertion opens it, and + /// one the point leaves empty is an empty word, so the level the dump files + /// a node at costs a word a layer and no slot. A list above its owner's /// level holding one entry is one word, and one holding more is promoted, /// which is the state a built graph holds an old entry point's list in. A /// list longer than the cap its layer allows is refused, since no build @@ -656,7 +670,9 @@ where span = span.max(list_layer); } } - self.open_node(span, layer); + self.open_node(span, |list_layer| { + list_layer <= layer && lists.get(list_layer).is_some_and(|list| !list.is_empty()) + }); for (list_layer, list) in lists.iter().enumerate() { if list.is_empty() { @@ -719,8 +735,9 @@ where // `memory_bytes` be checked against the counts rather than believed. The // upper regions are the ones whose size is not known before the walk, // because a span is a property of the file, and the counters' pass - // above promotes the lists the entry point chain named, so the trim - // comes after it. + // above promotes every word an entry names, being the entry point + // chain's lists above their level and any empty list at or below its + // owner's level, so the trim comes after it. self.node_of.shrink_to_fit(); self.upper_word.shrink_to_fit(); self.wide_at.shrink_to_fit(); @@ -1231,10 +1248,15 @@ where } /// Open one node's upper lists, giving it a word at every layer from one up - /// to `span`. A list at or below `level` is opened wide, because install - /// site 2 fills it wholesale; one above `level` is an empty word, which is - /// what 99.95 percent of them ever need. - fn open_node(&mut self, span: usize, level: usize) { + /// to `span`. A layer `wide` names is opened wide and every other is an + /// empty word, which is all nearly every list above a node's level ever + /// needs. + /// + /// Insertion opens wide every list at or below the new node's level, + /// because install site 2 fills it wholesale. A load opens wide only the + /// ones the dump fills, since nothing fills the rest wholesale afterwards + /// and a word is promoted the first time its owner is named. + fn open_node(&mut self, span: usize, wide: impl Fn(usize) -> bool) { if span == 0 { self.upper_first.push(NO_UPPER); self.upper_span.push(0); @@ -1243,7 +1265,7 @@ where self.upper_first.push(self.upper_word.len() as u32); self.upper_span.push(span as u8); for layer in 1..=span { - let word = if layer <= level { + let word = if wide(layer) { WORD_WIDE | self.open_wide() } else { WORD_EMPTY @@ -1312,7 +1334,9 @@ where /// /// This is what a second entry needs and what an inbound counter needs, /// since both live on the wide descriptor. It fires for the old entry - /// points' lists above their level and for nothing else on a build. + /// points' lists above their level and for nothing else on a build, and + /// on a load also for an empty list at or below its owner's level that an + /// entry names. fn promote(&mut self, list: usize) -> usize { let word = self.upper_word[list]; assert!( @@ -1659,7 +1683,7 @@ where for &(layer, _) in above { span = span.max(layer as usize); } - self.open_node(span, level); + self.open_node(span, |layer| layer <= level); for &(layer, target) in above { self.push_edge(node, layer as usize, target); @@ -1732,6 +1756,24 @@ where counts } + /// Every inbound counter the graph holds that is above zero, keyed by the + /// origin id of the node it counts into and the layer, so a test can hold + /// two graphs that number their nodes differently to the same counts, and + /// either to what [`Self::counted_in_degree`] counts from the lists. + #[cfg(test)] + pub(super) fn in_degrees(&self) -> std::collections::BTreeMap<(usize, usize), u32> { + let mut out = std::collections::BTreeMap::new(); + for node in 0..self.origin_ids.len() as u32 { + for layer in 0..=self.span(node) { + let count = self.in_degree(node, layer); + if count > 0 { + out.insert((self.origin_ids[node as usize], layer), count); + } + } + } + out + } + /// Return every buffer's spare capacity to the allocator. /// /// A graph built by insertion grows its arenas geometrically from whatever diff --git a/crates/core/src/graph/structure.rs b/crates/core/src/graph/structure.rs index 670f84c..f225c36 100644 --- a/crates/core/src/graph/structure.rs +++ b/crates/core/src/graph/structure.rs @@ -342,9 +342,12 @@ fn the_mutable_memory_figure_is_exact() { wide * (M + 1), "every wide list holds m + 1 slots and nothing else holds a slot" ); - // Every list at or below its owner's level is wide, every list above it - // is one word until the entry point chain promotes it, and a descent - // residue list is one word holding its one edge. + // A loaded list at or below its owner's level is wide unless it holds no + // entry and no entry names its owner there, every list above it is one + // word until the entry point chain promotes it, and a descent residue list + // is one word holding its one edge. This fixture holds no empty list at or + // below a level, so every list at or below its owner's level is wide here, + // which the count against the levels below relies on. let (empty, single, wide_words) = mutable.word_census(); assert_eq!(empty + single + wide_words, lists); assert!( diff --git a/crates/hnsw/src/collection/persist_tests.rs b/crates/hnsw/src/collection/persist_tests.rs index 6752400..c298d67 100644 --- a/crates/hnsw/src/collection/persist_tests.rs +++ b/crates/hnsw/src/collection/persist_tests.rs @@ -12,8 +12,8 @@ use std::sync::Arc; use serde_json::{json, Value}; use zeusdb_vector_core::{ - compile_filter, Error, FsDir, IdfScope, SparseVector, VectorGraph, VectorIndex, DUMP_FILENAME, - NB_LAYER_MAX, + compile_filter, Error, FsDir, IdfScope, Operation, SparseVector, VectorGraph, VectorIndex, + DUMP_FILENAME, NB_LAYER_MAX, }; use zeusdb_vector_sparse::{SparseConfig, Weighting}; use zeusdb_vector_text::{SimpleTokenizer, Tokenizer, TokenizerConfig}; @@ -1091,3 +1091,233 @@ fn a_directory_at_the_largest_m_and_the_smallest_dimension_reads_its_dump_and_wr dense_page(&collection, &[0.42]) ); } + +/// Lists at or below their owners' levels that a graph dump leaves empty, the +/// trailing ones the writer trims included, read off the file. +fn empty_lists_at_or_below_levels(dump: &[u8]) -> usize { + let word = |at: usize| u64::from_le_bytes(dump[at..at + 8].try_into().unwrap()) as usize; + let count = |at: usize| u32::from_le_bytes(dump[at..at + 4].try_into().unwrap()) as usize; + let layers = NB_LAYER_MAX as usize; + let mut at = 96 + 4 * layers + 8 * word(40); + let mut empty = 0; + for level in 0..layers { + for _ in 0..count(96 + 4 * level) { + let lists = dump[at] as usize; + at += 1; + for layer in 0..lists { + let entries = count(at); + at += 4 + 9 * entries; + if (1..=level).contains(&layer) && entries == 0 { + empty += 1; + } + } + empty += (lists.max(1)..=level).count(); + } + } + empty +} + +/// The largest origin id a graph dump names, read off the file. +fn largest_origin_id(dump: &[u8]) -> u64 { + let nodes = u64::from_le_bytes(dump[40..48].try_into().unwrap()) as usize; + let first = 96 + 4 * NB_LAYER_MAX as usize; + (0..nodes) + .map(|node| { + let at = first + 8 * node; + u64::from_le_bytes(dump[at..at + 8].try_into().unwrap()) + }) + .max() + .unwrap_or(0) +} + +/// Four values set by one record's number, the same every run. +fn spread(i: u32) -> Vec { + let x = i as f32; + vec![ + (x * 0.37).sin(), + (x * 0.11).cos(), + (x * 0.23).sin(), + (x * 0.07).cos(), + ] +} + +/// A collection's dense pages over `queries`, and the graph dump a save of it +/// writes. +fn answers(collection: &Collection, queries: &[Vec]) -> (Vec, Vec) { + let pages = queries + .iter() + .map(|query| dense_page(collection, query)) + .collect(); + let dir = TempDir::new(); + let path = dir.path().join("answers.zdb"); + collection.save(path.to_str().unwrap()).unwrap(); + (pages, std::fs::read(path.join(DUMP_FILENAME)).unwrap()) +} + +/// A collection whose saved graph holds empty lists at or below its levels, +/// read back from its dump and then changed, answers and writes what the +/// collection that saved it answers and writes after the same changes. +/// +/// At `m` 2 the levels drawn over three hundred records put a node more than +/// one layer above every earlier one, so its lists at the layers between stay +/// empty, and the load holds each as a word. Both collections then take the +/// same inserts at recorded levels through `apply`, reaching every layer the +/// graph holds and two above it, a removal, an overwrite and a compaction. +/// After each step the two answer the same pages and a save of each writes +/// the same dump, and so do the two after a save and a load of each. +#[test] +fn a_collection_changed_after_its_load_matches_the_collection_that_saved_it() { + let declaration = Declaration::validate(4, "l2", 2, 64, 400, vec![]).unwrap(); + let built = Collection::build(declaration, None); + let records: Vec = (0..300u32) + .map(|i| ParsedRecord { + id: format!("r{i}"), + vector: spread(i), + sparse: None, + metadata: HashMap::new(), + }) + .collect(); + assert_eq!(built.add_records(records, vec![], false).total_errors, 0); + let dir = TempDir::new(); + let path = dir.path().join("deep.zdb"); + built.save(path.to_str().unwrap()).unwrap(); + assert!( + empty_lists_at_or_below_levels(&std::fs::read(path.join(DUMP_FILENAME)).unwrap()) > 0, + "the saved graph holds no empty list at or below a level" + ); + let (loaded, recovery) = + Collection::recover(path.to_str().unwrap(), None, Durability::default()).unwrap(); + assert!( + !recovery.graph_rebuilt, + "the graph was rebuilt rather than read from its dump" + ); + + let queries: Vec> = (1_000..1_012u32).map(spread).collect(); + let same = |label: &str| { + let (built_pages, built_dump) = answers(&built, &queries); + let (loaded_pages, loaded_dump) = answers(&loaded, &queries); + assert_eq!(loaded_pages, built_pages, "{}", label); + assert!(loaded_dump == built_dump, "{}: the two dumps differ", label); + }; + let apply = |operation: Operation| { + built.apply(operation.clone()).unwrap(); + loaded.apply(operation).unwrap(); + }; + same("after the load"); + + let mut issued = built.id_counter(); + assert_eq!(loaded.id_counter(), issued); + for k in 0..40u32 { + issued += 1; + apply(Operation::Insert { + id: format!("n{k}"), + internal_id: issued as u64, + level: (k * 5 % 12) as u8, + vector: spread(2_000 + k), + metadata: serde_json::Map::new(), + sparse: None, + }); + } + same("after inserts at recorded levels"); + + let removed: Vec = (0..300u32) + .step_by(9) + .map(|i| format!("r{i}")) + .chain(["n3".to_string(), "n17".to_string()]) + .collect(); + apply(Operation::Remove { ids: removed }); + apply(Operation::Remove { + ids: vec!["r1".to_string()], + }); + issued += 1; + apply(Operation::Insert { + id: "r1".to_string(), + internal_id: issued as u64, + level: 3, + vector: spread(3_000), + metadata: serde_json::Map::new(), + sparse: None, + }); + same("after a removal and an overwrite"); + + apply(Operation::Compact); + same("after a compaction"); + + let built_again = reads_its_dump_and_writes_it_back(&built, "built.zdb"); + let loaded_again = reads_its_dump_and_writes_it_back(&loaded, "loaded.zdb"); + let (built_pages, built_dump) = answers(&built_again, &queries); + let (loaded_pages, loaded_dump) = answers(&loaded_again, &queries); + assert_eq!(loaded_pages, built_pages); + assert!( + loaded_dump == built_dump, + "after a save and a load: the two dumps differ" + ); + assert_eq!(ids_of(&loaded_again), ids_of(&built_again)); +} + +/// A directory whose counter is above every id its files name reads its +/// dump, holds what was saved, issues the counter's next id, and writes the +/// same dump again. +/// +/// Removing the newest records and compacting drops their stranded nodes and +/// keeps the counter, so `config.json` counts ids that neither the id map nor +/// the graph dump names. Nothing a directory holds bounds the counter +/// tighter, and the load reads the dump as it reads one at its counter. +#[test] +fn a_directory_whose_counter_is_above_its_largest_id_reads_its_dump_and_writes_it_back() { + let collection = Collection::build(base(), None); + let records: Vec = (0..300u32) + .map(|i| { + record( + &format!("r{i}"), + &[(i % 17) as f32 * 0.3, (i % 11) as f32 * 0.7], + None, + if i % 2 == 0 { "a" } else { "b" }, + ) + }) + .collect(); + assert_eq!( + collection.add_records(records, vec![], false).total_errors, + 0 + ); + for i in 250..300 { + assert!(collection.remove_point(format!("r{i}")).unwrap()); + } + collection.compact().unwrap(); + assert_eq!(collection.id_counter(), 300); + + let dir = TempDir::new(); + let path = dir.path().join("counted.zdb"); + collection.save(path.to_str().unwrap()).unwrap(); + assert_eq!(config(&path)["id_counter"], json!(300)); + assert_eq!( + largest_origin_id(&std::fs::read(path.join(DUMP_FILENAME)).unwrap()), + 250 + ); + assert_eq!( + ids_of(&collection) + .iter() + .map(|(_, internal)| *internal) + .max(), + Some(250) + ); + + let loaded = reads_its_dump_and_writes_it_back(&collection, "counted.zdb"); + assert_eq!(loaded.id_counter(), 300); + assert_eq!( + loaded.dense().index.read().unwrap().graph().nb_points(), + 250 + ); + assert_eq!(ids_of(&loaded), ids_of(&collection)); + assert_eq!( + dense_page(&loaded, &[1.5, 3.0]), + dense_page(&collection, &[1.5, 3.0]) + ); + assert_eq!( + loaded + .add_records(vec![record("late", &[0.4, 0.8], None, "a")], vec![], false) + .total_errors, + 0 + ); + assert!(ids_of(&loaded).contains(&("late".to_string(), 301))); +}