From dd30d54df833f1c876353633fbe40095d7625edf Mon Sep 17 00:00:00 2001 From: ansh Date: Wed, 21 Oct 2026 14:34:57 +0530 Subject: [PATCH 01/12] feat: add enums and structs to enable comprehensive conditions representation --- crates/defs/src/types.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/defs/src/types.rs b/crates/defs/src/types.rs index 1507653..4e1da4c 100644 --- a/crates/defs/src/types.rs +++ b/crates/defs/src/types.rs @@ -31,7 +31,7 @@ pub enum ContentType { #[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] pub struct Payload { pub content_type: ContentType, - pub content: String, + pub content: String, //two choices here either I change this payload to have the filtering metadata or change the point itself I am favoring changing this } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] @@ -48,6 +48,11 @@ pub struct Point { pub payload: Option, } +// An Enum having the operations we can do with the attributes which we can filter i.e. grt(>) grteq(>=) eq(==) in(this attribute exists for the results) etc. +// An Enum of conditions AND OR NOT maybe XOR +// Maybe even an enum for the order in which we return the results(as an option field) +// I would need a Fiter Struct which will give a structure to the filtering requests and make them kinda modular i.e. a parameter, a value, operation enum, a possible condition like AND OR, a link to another condition possibly in a tree form to enable nesting or we could simply use a predefined heiarchy which simplifies structure but limits the choices + /// Struct which will be stored in the vector index #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] pub struct IndexedVector { @@ -86,7 +91,7 @@ pub struct SearchQueryInput { pub similarity: Similarity, pub limit: usize, #[serde(default)] - pub ef: Option, + pub ef: Option, // adding 2 terms in this Struct, filter and method of filteration } #[derive(Clone, Serialize, Deserialize, Debug)] From 1944208d945078de9c032231cb98697259245bde Mon Sep 17 00:00:00 2001 From: ansh Date: Wed, 21 Oct 2026 19:52:43 +0530 Subject: [PATCH 02/12] feat: add enums and structs to enable comprehensive conditions representation --- Cargo.lock | 1 + crates/defs/Cargo.toml | 1 + crates/defs/src/types.rs | 32 ++++++++++++++++++++++++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3471623..6c089ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,6 +490,7 @@ version = "0.1.0" dependencies = [ "axum", "serde", + "serde_json", "snafu", "uuid", ] diff --git a/crates/defs/Cargo.toml b/crates/defs/Cargo.toml index 81d8dd9..c388b57 100644 --- a/crates/defs/Cargo.toml +++ b/crates/defs/Cargo.toml @@ -9,5 +9,6 @@ license.workspace = true [dependencies] axum.workspace = true serde.workspace = true +serde_json.workspace = true snafu.workspace = true uuid.workspace = true diff --git a/crates/defs/src/types.rs b/crates/defs/src/types.rs index 4e1da4c..0f16e1d 100644 --- a/crates/defs/src/types.rs +++ b/crates/defs/src/types.rs @@ -48,10 +48,34 @@ pub struct Point { pub payload: Option, } -// An Enum having the operations we can do with the attributes which we can filter i.e. grt(>) grteq(>=) eq(==) in(this attribute exists for the results) etc. -// An Enum of conditions AND OR NOT maybe XOR -// Maybe even an enum for the order in which we return the results(as an option field) -// I would need a Fiter Struct which will give a structure to the filtering requests and make them kinda modular i.e. a parameter, a value, operation enum, a possible condition like AND OR, a link to another condition possibly in a tree form to enable nesting or we could simply use a predefined heiarchy which simplifies structure but limits the choices +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FilterOp { + Eq, + Neq, + Lt, + Leq, + Gt, + Gteq, + In, + Exists, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FieldCondition { + pub key: String, + pub op: FilterOp, + #[serde(default)] + pub value: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Filter { + Condition(FieldCondition), + And(Vec), + Or(Vec), + Not(Box), + Xor(Vec), +} /// Struct which will be stored in the vector index #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] From 0b75bbb622f3fe8b2c9e31b395536b0f7dc9de5a Mon Sep 17 00:00:00 2001 From: ansh Date: Fri, 23 Oct 2026 21:13:25 +0530 Subject: [PATCH 03/12] feat: add the filter processing function to assess whether a payload passes a filter --- crates/defs/src/types.rs | 83 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/crates/defs/src/types.rs b/crates/defs/src/types.rs index 0f16e1d..46ca935 100644 --- a/crates/defs/src/types.rs +++ b/crates/defs/src/types.rs @@ -31,7 +31,9 @@ pub enum ContentType { #[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)] pub struct Payload { pub content_type: ContentType, - pub content: String, //two choices here either I change this payload to have the filtering metadata or change the point itself I am favoring changing this + pub content: String, + #[serde(default)] + pub metadata: Option, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] @@ -58,6 +60,7 @@ pub enum FilterOp { Gteq, In, Exists, + NotExists, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -68,6 +71,13 @@ pub struct FieldCondition { pub value: Option, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum FilterMode { + #[default] + Post, + Inline, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Filter { Condition(FieldCondition), @@ -116,6 +126,10 @@ pub struct SearchQueryInput { pub limit: usize, #[serde(default)] pub ef: Option, // adding 2 terms in this Struct, filter and method of filteration + #[serde(default)] + pub filter: Option, + #[serde(default)] + pub filter_mode: FilterMode, } #[derive(Clone, Serialize, Deserialize, Debug)] @@ -167,6 +181,32 @@ impl<'q> Eq for DistanceOrderedVector<'q> {} // Context(ContextQuery), // } +impl Filter { + pub fn matches(&self, metadata: &serde_json::Value) -> bool { + match self { + Filter::Condition(c) => c.matches(metadata), + Filter::And(fs) => fs.iter().all(|f| f.matches(metadata)), + Filter::Or(fs) => fs.iter().any(|f| f.matches(metadata)), + Filter::Not(f) => !f.matches(metadata), + Filter::Xor(fs) => fs.iter().filter(|f| f.matches(metadata)).count() % 2 == 1, + } + } +} + +impl FieldCondition { + fn matches(&self, metadata: &serde_json::Value) -> bool { + let found = resolve_path(metadata, &self.key); + match self.op { + FilterOp::Exists => found.is_some(), + FilterOp::NotExists => found.is_none(), + _ => match (found, self.value.as_ref()) { + (Some(lhs), Some(rhs)) => compare_op(&self.op, lhs, rhs), + _ => false, + }, + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Default)] pub struct OrdF32(pub f32); @@ -198,3 +238,44 @@ impl From for OrdF32 { Self(x) } } + +fn resolve_path<'a>(doc: &'a serde_json::Value, key: &str) -> Option<&'a serde_json::Value> { + key.split('.').try_fold(doc, |cur, seg| cur.get(seg)) +} + +fn value_eq(a: &serde_json::Value, b: &serde_json::Value) -> bool { + match (a, b) { + (serde_json::Value::Number(x), serde_json::Value::Number(y)) => { + matches!((x.as_f64(), y.as_f64()), (Some(x), Some(y)) if x == y) + } + _ => a == b, + } +} + +fn ordered_cmp(a: &serde_json::Value, b: &serde_json::Value) -> Option { + match (a, b) { + (serde_json::Value::Number(x), serde_json::Value::Number(y)) => { + x.as_f64()?.partial_cmp(&y.as_f64()?) + } + (serde_json::Value::String(x), serde_json::Value::String(y)) => Some(x.cmp(y)), + (serde_json::Value::Bool(x), serde_json::Value::Bool(y)) => Some(x.cmp(y)), + _ => None, + } +} + +fn compare_op(op: &FilterOp, a: &serde_json::Value, b: &serde_json::Value) -> bool { + match op { + FilterOp::Eq => value_eq(a, b), + FilterOp::Neq => !value_eq(a, b), + FilterOp::Lt | FilterOp::Leq | FilterOp::Gt | FilterOp::Gteq => match ordered_cmp(a, b) { + Some(Ordering::Less) => matches!(op, FilterOp::Lt | FilterOp::Leq), + Some(Ordering::Equal) => matches!(op, FilterOp::Lt | FilterOp::Gteq), + Some(Ordering::Greater) => matches!(op, FilterOp::Gt | FilterOp::Gteq), + None => false, + }, + FilterOp::In => { + matches!(b, serde_json::Value::Array(xs) if xs.iter().any(|i| value_eq(i, a))) + } + FilterOp::Exists | FilterOp::NotExists => true, + } +} From ca1c0d447fa0d4d262f19b08de8e80c32c8ea253 Mon Sep 17 00:00:00 2001 From: ansh Date: Fri, 23 Oct 2026 21:15:35 +0530 Subject: [PATCH 04/12] feat: add connection between the search apis to the filtering functions --- crates/api/src/lib.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 741efb2..a79be4f 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -146,6 +146,8 @@ impl VectorDb { let vectors = index.search_with_ef(query.vector, query.similarity, query.limit, query.ef)?; + let vectors = self.filter_ids(vectors, query.filter.as_ref())?; + //pass the Uuid vector to filtering function made which returns the actual vector which should be returned Ok(vectors) } @@ -156,6 +158,7 @@ impl VectorDb { for query in queries { let found = index.search_with_ef(query.vector, query.similarity, query.limit, query.ef)?; + let found = self.filter_ids(found, query.filter.as_ref())?; results.push(found); } @@ -191,6 +194,31 @@ impl VectorDb { Ok(inserted) } + + fn payload_matches(&self, id: PointId, filter: &Filter) -> Result { + Ok(self + .storage + .get_payload(id)? + .and_then(|p| p.metadata) + .is_some_and(|m| filter.matches(&m))) + } + + fn filter_ids( + &self, + ids: impl IntoIterator, + filter: Option<&Filter>, + ) -> Result> { + let Some(f) = filter else { + return Ok(ids.into_iter().collect()); + }; + let mut out = Vec::new(); + for id in ids { + if self.payload_matches(id, f)? { + out.push(id); + } + } + Ok(out) + } } impl SnapshottableDb for VectorDb { From 849effd55ee3a86109e6927b4665715645327778 Mon Sep 17 00:00:00 2001 From: ansh Date: Wed, 28 Oct 2026 19:09:37 +0530 Subject: [PATCH 05/12] refactor: change metadata from serde::Value to a bmap --- crates/defs/src/types.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/defs/src/types.rs b/crates/defs/src/types.rs index f124e16..31e4b89 100644 --- a/crates/defs/src/types.rs +++ b/crates/defs/src/types.rs @@ -11,6 +11,8 @@ pub type ElementByte = u8; pub type Dimension = usize; +pub type Metadata = serde_json::Map; + // Dense Vector and Vector are considered same // Sparse vector implementation not supported yet. Refer lib/sparse/src/common/sparse_vector.rs pub type DenseVector = Vec; @@ -33,7 +35,7 @@ pub struct Payload { pub content_type: ContentType, pub content: String, #[serde(default)] - pub metadata: Option, + pub metadata: Metadata, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] @@ -183,7 +185,7 @@ impl<'q> Eq for DistanceOrderedVector<'q> {} // } impl Filter { - pub fn matches(&self, metadata: &serde_json::Value) -> bool { + pub fn matches(&self, metadata: &Metadata) -> bool { match self { Filter::Condition(c) => c.matches(metadata), Filter::And(fs) => fs.iter().all(|f| f.matches(metadata)), @@ -195,7 +197,7 @@ impl Filter { } impl FieldCondition { - fn matches(&self, metadata: &serde_json::Value) -> bool { + fn matches(&self, metadata: &Metadata) -> bool { let found = resolve_path(metadata, &self.key); match self.op { FilterOp::Exists => found.is_some(), @@ -240,8 +242,14 @@ impl From for OrdF32 { } } -fn resolve_path<'a>(doc: &'a serde_json::Value, key: &str) -> Option<&'a serde_json::Value> { - key.split('.').try_fold(doc, |cur, seg| cur.get(seg)) +fn resolve_path<'a>(metadata: &'a Metadata, key: &str) -> Option<&'a serde_json::Value> { + let mut segments = key.split('.'); + let first = segments.next()?; + let mut cur = metadata.get(first)?; + for seg in segments { + cur = cur.get(seg)?; + } + Some(cur) } fn value_eq(a: &serde_json::Value, b: &serde_json::Value) -> bool { From 68923b7897fe966d92703514822aad034133966d Mon Sep 17 00:00:00 2001 From: ansh Date: Wed, 28 Oct 2026 19:12:03 +0530 Subject: [PATCH 06/12] refactor: change existing function to reflect metadata changes --- crates/api/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 99cb9e0..47bc3db 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -198,21 +198,20 @@ impl VectorDb { Ok(self .storage .get_payload(id)? - .and_then(|p| p.metadata) - .is_some_and(|m| filter.matches(&m))) + .is_some_and(|p| filter.matches(&p.metadata))) } fn filter_ids( &self, - ids: impl IntoIterator, + ids: impl IntoIterator, filter: Option<&Filter>, - ) -> Result> { + ) -> Result> { let Some(f) = filter else { return Ok(ids.into_iter().collect()); }; let mut out = Vec::new(); for id in ids { - if self.payload_matches(id, f)? { + if self.payload_matches(id.0, f)? { out.push(id); } } From 4951571ed51e2ae9b0727e6b2b330439c54151ec Mon Sep 17 00:00:00 2001 From: ansh Date: Wed, 28 Oct 2026 19:49:59 +0530 Subject: [PATCH 07/12] feat: add condition checking for filteration mode --- crates/api/src/lib.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 47bc3db..4ca8a60 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,7 +1,10 @@ -use defs::{DbError, Dimension, IndexedVector, SearchQueryInput, Similarity, SnapshottableDb}; +use defs::{ + DbError, Dimension, FilterMode, IndexedVector, SearchQueryInput, Similarity, SnapshottableDb, +}; use defs::{DenseVector, Payload, Point, PointId, PointInput}; use index::hnsw::{HnswConfig, HnswIndex}; use index::kd_tree::{KDTree, KDTreeConfig}; +use std::iter::FilterMap; use std::path::{Path, PathBuf}; use tempfile::tempdir; // use std::sync::atomic::{AtomicU64, Ordering}; @@ -145,8 +148,9 @@ impl VectorDb { let vectors = index.search_with_ef(query.vector, query.similarity, query.limit, query.ef)?; - let vectors = self.filter_ids(vectors, query.filter.as_ref())?; - //pass the Uuid vector to filtering function made which returns the actual vector which should be returned + if query.filter_mode == FilterMode::Post { + let vectors = self.filter_ids(vectors, query.filter.as_ref())?; + } Ok(vectors) } @@ -157,7 +161,9 @@ impl VectorDb { for query in queries { let found = index.search_with_ef(query.vector, query.similarity, query.limit, query.ef)?; - let found = self.filter_ids(found, query.filter.as_ref())?; + if query.filter_mode == FilterMode::Post { + let found = self.filter_ids(found, query.filter.as_ref())?; + } results.push(found); } From 9013124a5b7063a51cc6f2e695ac23d62b75b82b Mon Sep 17 00:00:00 2001 From: ansh Date: Thu, 29 Oct 2026 15:03:56 +0530 Subject: [PATCH 08/12] fix: update test to reflect present changes to the DB --- Cargo.lock | 2 ++ crates/api/Cargo.toml | 2 ++ crates/api/src/tests.rs | 46 ++++++++++++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c089ad..508b5ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,6 +53,8 @@ version = "0.1.0" dependencies = [ "defs", "index", + "serde", + "serde_json", "snafu", "snapshot", "storage", diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 7c198a3..08756c8 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -9,6 +9,8 @@ license.workspace = true [dependencies] defs.workspace = true index.workspace = true +serde.workspace = true +serde_json.workspace = true snafu.workspace = true snapshot.workspace = true storage.workspace = true diff --git a/crates/api/src/tests.rs b/crates/api/src/tests.rs index f99d77b..94b1891 100644 --- a/crates/api/src/tests.rs +++ b/crates/api/src/tests.rs @@ -2,10 +2,10 @@ use super::*; // TODO: Add more exhaustive tests -use std::sync::Mutex; - -use defs::ContentType; +use defs::{ContentType, FilterMode::Post, Metadata}; +use serde_json::{Value, json}; use snapshot::{engine::SnapshotEngine, registry::local::LocalRegistry}; +use std::sync::Mutex; use tempfile::{TempDir, tempdir}; // Helper function to create a test database @@ -27,13 +27,18 @@ fn create_test_db_with_storage(storage_type: StorageType) -> (VectorDb, TempDir) (init_api(config).unwrap(), temp_dir) } -fn test_payload(content: &str) -> Payload { +fn test_payload(content: &str, metadata: Metadata) -> Payload { Payload { content_type: ContentType::Text, content: content.to_string(), + metadata: metadata, } } +fn meta(v: serde_json::Value) -> Metadata { + v.as_object().unwrap().clone() +} + #[test] fn test_insert_and_get() { let (db, _temp_dir) = create_test_db(); @@ -41,6 +46,7 @@ fn test_insert_and_get() { let payload = Payload { content_type: ContentType::Text, content: "Test content".to_string(), + metadata: meta(json!({"age": 23})), }; // Test insert @@ -63,7 +69,7 @@ fn test_insert_and_get() { fn test_insert_and_get_with_in_memory_storage() { let (db, _temp_dir) = create_test_db_with_storage(StorageType::InMemory); let vector = vec![1.0, 2.0, 3.0]; - let payload = test_payload("Test content"); + let payload = test_payload("Test content", meta(json!({"age":12,"height":121}))); let id = db.insert(vector.clone(), payload.clone()).unwrap(); let point = db.get(id).unwrap().unwrap(); @@ -81,6 +87,7 @@ fn test_dimension_mismatch() { let payload = defs::Payload { content_type: ContentType::Text, content: "tester".to_string(), + metadata: meta(json!({"age": 23})), }; let res1 = db.insert(v1, payload.clone()); @@ -105,6 +112,7 @@ fn test_delete() { let payload = Payload { content_type: ContentType::Text, content: "Test content".to_string(), + metadata: meta(json!({"age":23})), }; // Insert a point @@ -137,6 +145,7 @@ fn test_search() { let payload = Payload { content_type: ContentType::Text, content: format!("Test content {vector:?}"), + metadata: meta(json!({"age":23})), }; let id = db.insert(vector, payload).unwrap(); ids.push(id); @@ -150,6 +159,8 @@ fn test_search() { similarity: Similarity::Cosine, limit: 1, ef: None, + filter: None, + filter_mode: Post, }) .unwrap(); @@ -173,6 +184,7 @@ fn test_search_limit() { Payload { content_type: ContentType::Text, content: format!("Test content {i}"), + metadata: meta(json!({"age":23})), }, ) .unwrap(); @@ -187,6 +199,8 @@ fn test_search_limit() { similarity: Similarity::Euclidean, limit: 3, ef: None, + filter: None, + filter_mode: Post, }) .unwrap(); @@ -203,6 +217,8 @@ fn test_search_zero_limit() { similarity: Similarity::Cosine, limit: 0, ef: None, + filter: None, + filter_mode: Post, }); assert!(result.is_err()); @@ -228,6 +244,8 @@ fn test_empty_database() { similarity: Similarity::Cosine, limit: 10, ef: None, + filter: None, + filter_mode: Post, }) .unwrap(); assert_eq!(results.len(), 0); @@ -247,6 +265,7 @@ fn test_list_vectors() { Payload { content_type: ContentType::Text, content: format!("Test content {i}"), + metadata: meta(json!({"age":23})), }, ) .unwrap(); @@ -277,6 +296,7 @@ fn test_build_index() { Payload { content_type: ContentType::Text, content: format!("Test content {i}"), + metadata: meta(json!({"age":23})), }, ) .unwrap(); @@ -301,6 +321,7 @@ fn test_create_and_load_snapshot() { Payload { content_type: ContentType::Text, content: "test".to_string(), + metadata: meta(json!({"age":23})), }, ) .unwrap(); @@ -311,6 +332,7 @@ fn test_create_and_load_snapshot() { Payload { content_type: ContentType::Text, content: "test".to_string(), + metadata: meta(json!({"age":23})), }, ) .unwrap(); @@ -325,6 +347,7 @@ fn test_create_and_load_snapshot() { Payload { content_type: ContentType::Text, content: "test".to_string(), + metadata: meta(json!({"age":23})), }, ) .unwrap(); @@ -354,13 +377,19 @@ fn test_create_and_load_snapshot_with_in_memory_storage() { let v2 = vec![3.0, 4.0, 5.0]; let v3 = vec![6.0, 7.0, 8.0]; - let id1 = old_db.insert(v1.clone(), test_payload("one")).unwrap(); - let id2 = old_db.insert(v2.clone(), test_payload("two")).unwrap(); + let id1 = old_db + .insert(v1.clone(), test_payload("one", meta(json!({"age": 12})))) + .unwrap(); + let id2 = old_db + .insert(v2.clone(), test_payload("two", meta(json!({"age": 32})))) + .unwrap(); let temp_snapshot_dir = tempdir().unwrap(); let snapshot_path = old_db.create_snapshot(temp_snapshot_dir.path()).unwrap(); - let id3 = old_db.insert(v3, test_payload("three")).unwrap(); + let id3 = old_db + .insert(v3, test_payload("three", meta(json!({"age": 22})))) + .unwrap(); let reload_config = DbRestoreConfig { data_path: temp_dir.path().to_path_buf(), @@ -405,6 +434,7 @@ fn test_snapshot_engine() { Payload { content_type: ContentType::Text, content: format!("{}", i), + metadata: meta(json!({"age":23})), }, ) .unwrap(); From 96bd909c226112ef69d4f13e3567ea79d2e1d0c4 Mon Sep 17 00:00:00 2001 From: ansh Date: Fri, 30 Oct 2026 17:31:29 +0530 Subject: [PATCH 09/12] fix: change the result so that it is no longer shadowed while post-filtering --- crates/api/src/lib.rs | 8 ++++---- crates/defs/src/types.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 4ca8a60..5ff22a5 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -145,11 +145,11 @@ impl VectorDb { // Use vector index to find similar vectors let index = self.index.read().map_err(|_| ApiError::LockError)?; - let vectors = + let mut vectors = index.search_with_ef(query.vector, query.similarity, query.limit, query.ef)?; if query.filter_mode == FilterMode::Post { - let vectors = self.filter_ids(vectors, query.filter.as_ref())?; + vectors = self.filter_ids(vectors, query.filter.as_ref())?; } Ok(vectors) } @@ -159,10 +159,10 @@ impl VectorDb { let index = self.index.read().unwrap(); for query in queries { - let found = + let mut found = index.search_with_ef(query.vector, query.similarity, query.limit, query.ef)?; if query.filter_mode == FilterMode::Post { - let found = self.filter_ids(found, query.filter.as_ref())?; + found = self.filter_ids(found, query.filter.as_ref())?; } results.push(found); } diff --git a/crates/defs/src/types.rs b/crates/defs/src/types.rs index 31e4b89..947e3a7 100644 --- a/crates/defs/src/types.rs +++ b/crates/defs/src/types.rs @@ -278,7 +278,7 @@ fn compare_op(op: &FilterOp, a: &serde_json::Value, b: &serde_json::Value) -> bo FilterOp::Neq => !value_eq(a, b), FilterOp::Lt | FilterOp::Leq | FilterOp::Gt | FilterOp::Gteq => match ordered_cmp(a, b) { Some(Ordering::Less) => matches!(op, FilterOp::Lt | FilterOp::Leq), - Some(Ordering::Equal) => matches!(op, FilterOp::Lt | FilterOp::Gteq), + Some(Ordering::Equal) => matches!(op, FilterOp::Leq | FilterOp::Gteq), Some(Ordering::Greater) => matches!(op, FilterOp::Gt | FilterOp::Gteq), None => false, }, From 2b8893e38e4e4d23a216186c68e7046479bbdc1e Mon Sep 17 00:00:00 2001 From: ansh Date: Fri, 30 Oct 2026 17:37:04 +0530 Subject: [PATCH 10/12] test: update the test to reflect present struct changes --- Cargo.lock | 1 + crates/storage/Cargo.toml | 1 + crates/storage/src/in_memory/tests.rs | 74 ++++++++++++++++++++++----- crates/storage/src/rocks_db/tests.rs | 6 +++ 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 508b5ae..238f72d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2452,6 +2452,7 @@ dependencies = [ "flate2", "rocksdb", "serde", + "serde_json", "snafu", "tar", "tempfile", diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index 9fe9322..16d0183 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -12,6 +12,7 @@ defs.workspace = true flate2 = "1.1.5" rocksdb.workspace = true serde.workspace = true +serde_json.workspace = true snafu.workspace = true tar = "0.4.44" tempfile.workspace = true diff --git a/crates/storage/src/in_memory/tests.rs b/crates/storage/src/in_memory/tests.rs index 9d8510e..37f17ed 100644 --- a/crates/storage/src/in_memory/tests.rs +++ b/crates/storage/src/in_memory/tests.rs @@ -1,6 +1,7 @@ use super::*; -use defs::ContentType; +use defs::{ContentType, Metadata}; +use serde_json::Value; use std::io::{Read, Write}; use tempfile::{TempDir, tempdir}; use uuid::Uuid; @@ -9,19 +10,24 @@ fn create_test_storage() -> MemoryStorage { MemoryStorage::new() } -fn test_payload(content: &str) -> Payload { +fn test_payload(content: &str, metadata: Metadata) -> Payload { Payload { content_type: ContentType::Text, content: content.to_string(), + metadata: meta(json!({"age": 12})), } } +pub fn meta(v: serde_json::Value) -> Metadata { + v.as_object().unwrap().clone() +} + #[test] fn test_insert_and_get_vector() { let storage = create_test_storage(); let id = Uuid::new_v4(); let vector = Some(vec![0.1, 0.2, 0.3]); - let payload = Some(test_payload("Test")); + let payload = Some(test_payload("Test", meta(json!({"age": 12})))); storage.insert_point(id, vector.clone(), payload).unwrap(); @@ -32,7 +38,7 @@ fn test_insert_and_get_vector() { fn test_insert_and_get_payload() { let storage = create_test_storage(); let id = Uuid::new_v4(); - let payload = Some(test_payload("Test")); + let payload = Some(test_payload("Test", meta(json!({"age": 12})))); storage.insert_point(id, None, payload.clone()).unwrap(); @@ -47,7 +53,11 @@ fn test_contains_and_delete_point() { assert!(!storage.contains_point(id).unwrap()); storage - .insert_point(id, Some(vec![0.4, 0.5, 0.6]), Some(test_payload("Test"))) + .insert_point( + id, + Some(vec![0.4, 0.5, 0.6]), + Some(test_payload("Test", meta(json!({"age": 12})))), + ) .unwrap(); assert!(storage.contains_point(id).unwrap()); @@ -68,16 +78,41 @@ fn test_list_vectors_respects_offset_limit_and_skips_payload_only_points() { ]; storage - .insert_point(ids[0], Some(vec![1.0, 1.1]), Some(test_payload("one"))) + .insert_point( + ids[0], + Some(vec![1.0, 1.1]), + Some(test_payload("one", meta(json!({"age": 12, "height": 123})))), + ) .unwrap(); storage - .insert_point(ids[1], None, Some(test_payload("payload-only"))) + .insert_point( + ids[1], + None, + Some(test_payload( + "payload-only", + meta(json!({"age": 21, "height": 139})), + )), + ) .unwrap(); storage - .insert_point(ids[2], Some(vec![3.0, 3.1]), Some(test_payload("three"))) + .insert_point( + ids[2], + Some(vec![3.0, 3.1]), + Some(test_payload( + "three", + meta(json!({"age": 49, "height": 154})), + )), + ) .unwrap(); storage - .insert_point(ids[3], Some(vec![4.0, 4.1]), Some(test_payload("four"))) + .insert_point( + ids[3], + Some(vec![4.0, 4.1]), + Some(test_payload( + "four", + meta(json!({"age": 10, "height": 190})), + )), + ) .unwrap(); let (first_page, next_offset) = storage.list_vectors(Uuid::nil(), 2).unwrap().unwrap(); @@ -110,7 +145,10 @@ fn test_create_and_restore_checkpoint() { .insert_point( id_before_checkpoint, Some(vec![0.1, 0.2, 0.3]), - Some(test_payload("before")), + Some(test_payload( + "before", + meta(json!({"age": 10, "height": 190})), + )), ) .unwrap(); let checkpoint = storage.checkpoint_at(temp_dir.path()).unwrap(); @@ -119,7 +157,10 @@ fn test_create_and_restore_checkpoint() { .insert_point( id_after_checkpoint, Some(vec![0.4, 0.5, 0.6]), - Some(test_payload("after")), + Some(test_payload( + "after", + meta(json!({"age": 10, "height": 190})), + )), ) .unwrap(); @@ -129,7 +170,10 @@ fn test_create_and_restore_checkpoint() { assert!(!storage.contains_point(id_after_checkpoint).unwrap()); assert_eq!( storage.get_payload(id_before_checkpoint).unwrap(), - Some(test_payload("before")) + Some(test_payload( + "before", + meta(json!({"age": 10, "height": 190})) + )) ); } @@ -140,7 +184,11 @@ fn test_checkpoint_writes_header() { let id = Uuid::new_v4(); storage - .insert_point(id, Some(vec![0.1, 0.2, 0.3]), Some(test_payload("point"))) + .insert_point( + id, + Some(vec![0.1, 0.2, 0.3]), + Some(test_payload("point", meta(json!({"age": 12})))), + ) .unwrap(); let checkpoint = storage.checkpoint_at(temp_dir.path()).unwrap(); diff --git a/crates/storage/src/rocks_db/tests.rs b/crates/storage/src/rocks_db/tests.rs index 4673f2f..15da821 100644 --- a/crates/storage/src/rocks_db/tests.rs +++ b/crates/storage/src/rocks_db/tests.rs @@ -26,6 +26,7 @@ fn test_insert_and_get_vector() { let payload = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), + metadata: meta(json!({"age": 12})), }); assert!(db.insert_point(id, vector.clone(), payload).is_ok()); @@ -40,6 +41,7 @@ fn test_insert_and_get_payload() { let payload = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), + metadata: meta(json! {"age": 12}), }); let vector = None; @@ -49,6 +51,7 @@ fn test_insert_and_get_payload() { let expected = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), + metadata: meta(json!({"age": 12})), }); assert_eq!(result, expected); } @@ -60,6 +63,7 @@ fn test_contains_point() { let payload = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), + metadata: meta(json!({"age": 12})), }); assert!(!db.contains_point(id).unwrap()); @@ -77,6 +81,7 @@ fn test_delete_point() { let payload = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), + metadata: meta(json!({"age": 12})), }); let vector = vec![0.7, 0.8, 0.9]; @@ -138,6 +143,7 @@ fn test_create_and_load_checkpoint() { let payload = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), + metadata: meta(json!({"age": 12})), }); assert!( From 52b71dfe34a8fb7d894a4c11c05a3b88090b7309 Mon Sep 17 00:00:00 2001 From: ansh Date: Fri, 30 Oct 2026 17:46:46 +0530 Subject: [PATCH 11/12] add: add the json mico to the file scope --- crates/storage/src/in_memory/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/src/in_memory/tests.rs b/crates/storage/src/in_memory/tests.rs index 37f17ed..5b3786b 100644 --- a/crates/storage/src/in_memory/tests.rs +++ b/crates/storage/src/in_memory/tests.rs @@ -1,7 +1,7 @@ use super::*; use defs::{ContentType, Metadata}; -use serde_json::Value; +use serde_json::{Value, json}; use std::io::{Read, Write}; use tempfile::{TempDir, tempdir}; use uuid::Uuid; From 09affdf51af2dc88566768935c17e01fbe806387 Mon Sep 17 00:00:00 2001 From: ansh Date: Fri, 30 Oct 2026 18:33:53 +0530 Subject: [PATCH 12/12] fix: add correct packages --- crates/storage/src/in_memory/tests.rs | 2 +- crates/storage/src/rocks_db/tests.rs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/storage/src/in_memory/tests.rs b/crates/storage/src/in_memory/tests.rs index 5b3786b..4555a28 100644 --- a/crates/storage/src/in_memory/tests.rs +++ b/crates/storage/src/in_memory/tests.rs @@ -18,7 +18,7 @@ fn test_payload(content: &str, metadata: Metadata) -> Payload { } } -pub fn meta(v: serde_json::Value) -> Metadata { +fn meta(v: serde_json::Value) -> Metadata { v.as_object().unwrap().clone() } diff --git a/crates/storage/src/rocks_db/tests.rs b/crates/storage/src/rocks_db/tests.rs index 15da821..a62718c 100644 --- a/crates/storage/src/rocks_db/tests.rs +++ b/crates/storage/src/rocks_db/tests.rs @@ -1,6 +1,7 @@ use super::*; use defs::ContentType; +use serde_json::{Value, json}; use uuid::Uuid; use tempfile::{TempDir, tempdir}; @@ -12,6 +13,10 @@ fn create_test_db() -> (RocksDbStorage, TempDir) { (db, temp_dir) } +fn meta(v: serde_json::Value) -> Metadata { + v.as_object().unwrap().clone() +} + #[test] fn test_new_rocksdb_storage() { let (db, temp_dir) = create_test_db(); @@ -41,7 +46,7 @@ fn test_insert_and_get_payload() { let payload = Some(Payload { content_type: ContentType::Text, content: "Test".to_string(), - metadata: meta(json! {"age": 12}), + metadata: meta(json!({"age": 12})), }); let vector = None;