From d8cfbf0a2fc943cab3e7367f669f3daac75daff9 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Mon, 6 Jul 2026 13:10:03 +0000 Subject: [PATCH 1/3] feat(core): add Milvus semantic vector backend Signed-off-by: Cheney Zhang --- README.md | 2 +- docs/semantic-search.md | 63 ++ pyproject.toml | 5 + src/basic_memory/config.py | 53 +- .../repository/milvus_search_repository.py | 711 ++++++++++++++++++ .../repository/project_repository.py | 45 +- .../repository/search_repository.py | 43 +- .../repository/search_repository_base.py | 37 + src/basic_memory/services/project_service.py | 18 +- src/basic_memory/services/search_service.py | 45 +- .../test_milvus_search_repository.py | 397 ++++++++++ .../test_project_service_embedding_status.py | 85 ++- tests/test_config.py | 23 + uv.lock | 267 ++++++- 14 files changed, 1725 insertions(+), 69 deletions(-) create mode 100644 src/basic_memory/repository/milvus_search_repository.py create mode 100644 tests/repository/test_milvus_search_repository.py diff --git a/README.md b/README.md index 8374c13d2..153339d88 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,7 @@ Try a prompt: and Homebrew installs; `bm update` triggers a manual check. - **Semantic vector search.** Find notes by meaning, not just keywords. Hybrid full-text + vector ranking with FastEmbed embeddings, on SQLite or - Postgres. + Postgres, with optional Milvus vector storage. - **Schema system.** Infer, validate, and diff the structure of your knowledge base with `schema_infer`, `schema_validate`, `schema_diff`. - **Per-project cloud routing.** Route individual projects through the cloud diff --git a/docs/semantic-search.md b/docs/semantic-search.md index fb54f4d37..181d83be2 100644 --- a/docs/semantic-search.md +++ b/docs/semantic-search.md @@ -111,6 +111,10 @@ All settings are fields on `BasicMemoryConfig` and can be set via environment va | `semantic_embedding_document_prefix` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_DOCUMENT_PREFIX` | Unset | Optional literal text prefix prepended to indexed document chunks before embedding. | | `semantic_embedding_query_prefix` | `BASIC_MEMORY_SEMANTIC_EMBEDDING_QUERY_PREFIX` | Unset | Optional literal text prefix prepended to search queries before embedding. | | `semantic_vector_k` | `BASIC_MEMORY_SEMANTIC_VECTOR_K` | `100` | Candidate count for vector nearest-neighbour retrieval. Higher values improve recall at the cost of latency. | +| `semantic_vector_backend` | `BASIC_MEMORY_SEMANTIC_VECTOR_BACKEND` | `"database"` | Vector storage backend. `"database"` uses sqlite-vec or pgvector; `"milvus"` stores vectors in Milvus while keeping metadata in SQLite/Postgres. | +| `milvus_uri` | `BASIC_MEMORY_MILVUS_URI` or `MILVUS_URI` | `/milvus.db` | Milvus URI when `semantic_vector_backend="milvus"`. Use a local `.db` path for Milvus Lite, `http://localhost:19530` for Milvus server, or a Zilliz Cloud endpoint. | +| `milvus_token` | `BASIC_MEMORY_MILVUS_TOKEN` or `MILVUS_TOKEN` | unset | Optional Milvus/Zilliz Cloud token. | +| `milvus_collection_name` | `BASIC_MEMORY_MILVUS_COLLECTION_NAME` | `"basic_memory_vectors"` | Milvus collection used for semantic chunks. | ## Embedding Providers @@ -282,6 +286,47 @@ rebuild embeddings: bm reindex --embeddings ``` +## Milvus Vector Backend + +By default, Basic Memory stores vectors in the active database: sqlite-vec for +SQLite and pgvector for Postgres. Advanced users can store semantic vectors in +Milvus while keeping full-text search, entity data, and chunk metadata in the +existing SQL backend. + +Install the optional Milvus extra: + +```bash +pip install "basic-memory[milvus]" +``` + +Enable the backend: + +```bash +export BASIC_MEMORY_SEMANTIC_SEARCH_ENABLED=true +export BASIC_MEMORY_SEMANTIC_VECTOR_BACKEND=milvus +``` + +The default `milvus_uri` points to a Milvus Lite database under Basic Memory's +config directory, so no server is required. To target another Milvus deployment: + +```bash +# Milvus server +export MILVUS_URI=http://localhost:19530 + +# Zilliz Cloud +export MILVUS_URI=https:// +export MILVUS_TOKEN= +``` + +Milvus uses the same embedding provider settings as the database-backed vector +stores. After changing `semantic_vector_backend`, `milvus_uri`, +`milvus_collection_name`, model, dimensions, or provider-specific embedding +settings, rebuild embeddings: + +```bash +bm reindex --embeddings --full +``` + ## Search Modes ### `text` (default) @@ -412,3 +457,21 @@ The sqlite-vec extension is loaded per-connection. Vector tables are created laz - **Index**: HNSW index on the embedding column for fast approximate nearest-neighbour queries The Alembic migration creates the dimension-independent chunks table. The embeddings table and HNSW index are deferred to runtime because they depend on the configured vector dimensions. + +### Milvus (optional) + +- **Vector storage**: Milvus via `pymilvus.MilvusClient`, using `AUTOINDEX` and + COSINE scoring +- **Default local mode**: Milvus Lite at `/milvus.db` +- **Server/cloud mode**: set `MILVUS_URI` and optionally `MILVUS_TOKEN` +- **SQL metadata**: `search_vector_chunks` stores chunk text and invalidation + metadata; `search_vector_embeddings` is a lightweight marker table used for + embedding coverage and incremental reindex checks +- **Collection schema**: Basic Memory creates a collection with `id`, + `project_id`, `chunk_id`, `entity_id`, `chunk_key`, `chunk_text`, and + `embedding` fields + +If an existing Milvus collection has a different embedding dimension or is +missing required fields, Basic Memory fails fast. Use a new +`milvus_collection_name` or recreate the collection, then run +`bm reindex --embeddings --full`. diff --git a/pyproject.toml b/pyproject.toml index 8fc9250f0..7bafa7c8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,11 @@ Documentation = "https://github.com/basicmachines-co/basic-memory#readme" basic-memory = "basic_memory.cli.main:app" bm = "basic_memory.cli.main:app" +[project.optional-dependencies] +milvus = [ + "pymilvus[milvus-lite]>=2.6.0,<4.0.0", +] + [build-system] requires = ["hatchling", "uv-dynamic-versioning>=0.7.0"] build-backend = "hatchling.build" diff --git a/src/basic_memory/config.py b/src/basic_memory/config.py index 83de2df99..ffd7f08b3 100644 --- a/src/basic_memory/config.py +++ b/src/basic_memory/config.py @@ -27,6 +27,10 @@ WATCH_STATUS_JSON = "watch-status.json" CONFIG_DIR_MODE = 0o700 CONFIG_FILE_MODE = 0o600 +UNPREFIXED_ENV_ALIASES: dict[str, tuple[str, ...]] = { + "milvus_uri": ("MILVUS_URI",), + "milvus_token": ("MILVUS_TOKEN",), +} Environment = Literal["test", "dev", "user"] @@ -57,6 +61,13 @@ class DatabaseBackend(str, Enum): POSTGRES = "postgres" +class SemanticVectorBackend(str, Enum): + """Supported semantic vector storage backends.""" + + DATABASE = "database" + MILVUS = "milvus" + + def _default_semantic_search_enabled() -> bool: """Enable semantic search by default when required local semantic dependencies exist.""" required_modules = ("fastembed", "sqlite_vec") @@ -106,6 +117,13 @@ def default_fastembed_cache_dir() -> str: return str(resolve_data_dir() / "fastembed_cache") +def default_milvus_uri() -> str: + """Return the default Milvus Lite URI for the semantic Milvus backend.""" + if env_override := os.getenv("MILVUS_URI"): + return env_override + return str(resolve_data_dir() / "milvus.db") + + @dataclass class ProjectConfig: """Configuration for a specific basic-memory project.""" @@ -375,6 +393,34 @@ def __init__(self, **data: Any) -> None: ... "Valid values: text, vector, hybrid. " "When unset, defaults to 'hybrid' if semantic search is enabled, otherwise 'text'.", ) + semantic_vector_backend: SemanticVectorBackend = Field( + default=SemanticVectorBackend.DATABASE, + description=( + "Semantic vector storage backend. 'database' uses sqlite-vec or pgvector " + "with the active database backend; 'milvus' stores vectors in Milvus while " + "keeping search metadata in the active database." + ), + ) + milvus_uri: str = Field( + default_factory=default_milvus_uri, + description=( + "Milvus URI for semantic_vector_backend='milvus'. Defaults to a Milvus Lite " + "database under the Basic Memory config directory. Can also be set with " + "MILVUS_URI." + ), + ) + milvus_token: str | None = Field( + default_factory=lambda: os.getenv("MILVUS_TOKEN"), + description=( + "Optional Milvus/Zilliz Cloud token for semantic_vector_backend='milvus'. " + "Can also be set with MILVUS_TOKEN." + ), + ) + milvus_collection_name: str = Field( + default="basic_memory_vectors", + description="Milvus collection name for semantic vector chunks.", + min_length=1, + ) # Database connection pool configuration (Postgres only) db_pool_size: int = Field( @@ -1070,8 +1116,11 @@ def load_config(self) -> BasicMemoryConfig: merged_data = file_data.copy() for field_name in BasicMemoryConfig.model_fields.keys(): - env_var_name = f"BASIC_MEMORY_{field_name.upper()}" - if env_var_name in os.environ: + env_var_names = ( + f"BASIC_MEMORY_{field_name.upper()}", + *UNPREFIXED_ENV_ALIASES.get(field_name, ()), + ) + if any(env_var_name in os.environ for env_var_name in env_var_names): # Trigger: the file and environment both configure this field. # Why: BaseSettings only gives environment values precedence when the # field is absent from constructor data. Removing the file value lets diff --git a/src/basic_memory/repository/milvus_search_repository.py b/src/basic_memory/repository/milvus_search_repository.py new file mode 100644 index 000000000..61eb8fab5 --- /dev/null +++ b/src/basic_memory/repository/milvus_search_repository.py @@ -0,0 +1,711 @@ +"""Milvus-backed semantic vector storage for search repositories.""" + +import asyncio +import importlib +import re +from typing import Any, Callable + +from loguru import logger +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from basic_memory import db +from basic_memory.config import BasicMemoryConfig, ConfigManager, SemanticVectorBackend +from basic_memory.repository.embedding_provider import EmbeddingProvider +from basic_memory.repository.postgres_search_repository import PostgresSearchRepository +from basic_memory.repository.search_repository_base import ( + SearchRepositoryBase, +) +from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError +from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository + +type MilvusClientFactory = Callable[..., Any] + +_COLLECTION_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_MILVUS_ID_MAX_LENGTH = 128 +_MILVUS_TEXT_MAX_LENGTH = 65535 + + +def _load_milvus_symbols() -> tuple[MilvusClientFactory, Any]: + """Load pymilvus lazily so the Milvus extra remains optional.""" + try: + pymilvus = importlib.import_module("pymilvus") + except ImportError as exc: + raise SemanticDependenciesMissingError( + "Milvus vector backend requires pymilvus. Install with " + "`pip install 'basic-memory[milvus]'` or disable Milvus by setting " + "semantic_vector_backend='database'." + ) from exc + return pymilvus.MilvusClient, pymilvus.DataType + + +async def delete_milvus_project_vectors(config: BasicMemoryConfig, project_id: int) -> None: + """Delete one project's vectors from Milvus without a SearchRepository instance.""" + if config.semantic_vector_backend != SemanticVectorBackend.MILVUS: + return + + client_factory, _data_type = _load_milvus_symbols() + client_kwargs: dict[str, str] = {"uri": config.milvus_uri} + if config.milvus_token: + client_kwargs["token"] = config.milvus_token + client = client_factory(**client_kwargs) + collection_name = config.milvus_collection_name + + has_collection = await asyncio.to_thread( + client.has_collection, + collection_name=collection_name, + ) + if not has_collection: + return + + await asyncio.to_thread( + client.delete, + collection_name=collection_name, + filter=f"project_id == {int(project_id)}", + ) + + +class MilvusVectorSearchMixin(SearchRepositoryBase): + """Mixin that stores vector embeddings in Milvus and metadata in SQL.""" + + _milvus_client_factory: MilvusClientFactory | None + _milvus_data_type: Any | None + _milvus_client: Any | None + _milvus_uri: str + _milvus_token: str | None + _milvus_collection_name: str + + def _configure_milvus_vector_backend( + self, + app_config: BasicMemoryConfig, + *, + milvus_client_factory: MilvusClientFactory | None = None, + milvus_data_type: Any | None = None, + ) -> None: + """Resolve Milvus runtime settings after the SQL repository initializes.""" + if not _COLLECTION_NAME_PATTERN.match(app_config.milvus_collection_name): + raise ValueError( + "Milvus collection names must start with a letter or underscore and " + "contain only letters, numbers, and underscores." + ) + + self._milvus_uri = app_config.milvus_uri + self._milvus_token = app_config.milvus_token + self._milvus_collection_name = app_config.milvus_collection_name + self._milvus_client_factory = milvus_client_factory + self._milvus_data_type = milvus_data_type + self._milvus_client = None + + def _get_milvus_symbols(self) -> tuple[MilvusClientFactory, Any]: + if self._milvus_client_factory is not None and self._milvus_data_type is not None: + return self._milvus_client_factory, self._milvus_data_type + + client_factory, data_type = _load_milvus_symbols() + self._milvus_client_factory = client_factory + self._milvus_data_type = data_type + return client_factory, data_type + + def _get_milvus_client(self) -> Any: + if self._milvus_client is not None: + return self._milvus_client + + client_factory, _data_type = self._get_milvus_symbols() + client_kwargs: dict[str, str] = {"uri": self._milvus_uri} + if self._milvus_token: + client_kwargs["token"] = self._milvus_token + self._milvus_client = client_factory(**client_kwargs) + return self._milvus_client + + async def _milvus_call(self, method_name: str, *args: Any, **kwargs: Any) -> Any: + """Run synchronous MilvusClient calls off the event loop.""" + client = self._get_milvus_client() + method = getattr(client, method_name) + return await asyncio.to_thread(method, *args, **kwargs) + + async def _prepare_vector_session(self, session: AsyncSession) -> None: + """Milvus does not need per-SQL-connection extension setup.""" + + async def _ensure_vector_tables(self) -> None: + """Create SQL marker tables and ensure the Milvus collection exists.""" + self._assert_semantic_available() + if self._vector_tables_initialized: + return + + logger.debug( + "Ensuring Milvus vector backend is ready " + f"(collection={self._milvus_collection_name}, dimensions={self._vector_dimensions})" + ) + async with db.scoped_session(self.session_maker) as session: + await self._ensure_sql_vector_metadata_tables(session) + await session.commit() + + await self._ensure_milvus_collection() + self._vector_tables_initialized = True + + async def _ensure_sql_vector_metadata_tables(self, session: AsyncSession) -> None: + """Create derived SQL metadata used by the shared vector sync pipeline.""" + dialect_name = self._dialect_name(session) + expected_chunk_columns = { + "id", + "entity_id", + "project_id", + "chunk_key", + "chunk_text", + "source_hash", + "entity_fingerprint", + "embedding_model", + "updated_at", + } + chunk_columns = await self._table_columns( + session, table_name="search_vector_chunks", dialect_name=dialect_name + ) + chunk_schema_mismatch = bool(chunk_columns) and not expected_chunk_columns.issubset( + chunk_columns + ) + if chunk_schema_mismatch: + logger.warning("search_vector_chunks schema mismatch, recreating vector metadata") + await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings")) + await session.execute(text("DROP TABLE IF EXISTS search_vector_chunks")) + + embedding_columns = await self._table_columns( + session, table_name="search_vector_embeddings", dialect_name=dialect_name + ) + marker_columns = {"chunk_id", "project_id", "embedding_dims", "updated_at"} + marker_mismatch = bool(embedding_columns) and not marker_columns.issubset(embedding_columns) + if marker_mismatch: + # Trigger: switching from sqlite-vec/pgvector to Milvus leaves a vector + # storage table under the marker table name. + # Why: the Milvus backend stores real vectors externally and only needs + # SQL rows to track which chunks have been written. + # Outcome: discard derived vector state; reindex recreates marker rows. + logger.warning("search_vector_embeddings is not a Milvus marker table, recreating it") + await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings")) + + if dialect_name == "postgresql": + await session.execute( + text( + """ + CREATE TABLE IF NOT EXISTS search_vector_chunks ( + id BIGSERIAL PRIMARY KEY, + entity_id INTEGER NOT NULL, + project_id INTEGER NOT NULL, + chunk_key TEXT NOT NULL, + chunk_text TEXT NOT NULL, + source_hash TEXT NOT NULL, + entity_fingerprint TEXT NOT NULL, + embedding_model TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (project_id, entity_id, chunk_key) + ) + """ + ) + ) + await session.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity + ON search_vector_chunks (project_id, entity_id) + """ + ) + ) + await session.execute( + text( + """ + CREATE TABLE IF NOT EXISTS search_vector_embeddings ( + chunk_id BIGINT PRIMARY KEY + REFERENCES search_vector_chunks(id) ON DELETE CASCADE, + project_id INTEGER NOT NULL, + embedding_dims INTEGER NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + ) + await session.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_search_vector_embeddings_project_dims + ON search_vector_embeddings (project_id, embedding_dims) + """ + ) + ) + return + + await session.execute( + text( + """ + CREATE TABLE IF NOT EXISTS search_vector_chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL, + project_id INTEGER NOT NULL, + chunk_key TEXT NOT NULL, + chunk_text TEXT NOT NULL, + source_hash TEXT NOT NULL, + entity_fingerprint TEXT NOT NULL, + embedding_model TEXT NOT NULL, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (project_id, entity_id, chunk_key) + ) + """ + ) + ) + await session.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_search_vector_chunks_project_entity + ON search_vector_chunks (project_id, entity_id) + """ + ) + ) + await session.execute( + text( + """ + CREATE TABLE IF NOT EXISTS search_vector_embeddings ( + chunk_id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL, + embedding_dims INTEGER NOT NULL, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + ) + await session.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_search_vector_embeddings_project_dims + ON search_vector_embeddings (project_id, embedding_dims) + """ + ) + ) + + @staticmethod + def _dialect_name(session: AsyncSession) -> str: + if session.bind is None: + return "sqlite" + return str(session.bind.dialect.name) + + async def _table_columns( + self, + session: AsyncSession, + *, + table_name: str, + dialect_name: str, + ) -> set[str]: + if dialect_name == "postgresql": + result = await session.execute( + text( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = :table_name + """ + ), + {"table_name": table_name}, + ) + return {str(row[0]) for row in result.fetchall()} + + result = await session.execute(text(f"PRAGMA table_info({table_name})")) + return {str(row[1]) for row in result.fetchall()} + + async def _ensure_milvus_collection(self) -> None: + has_collection = await self._milvus_call( + "has_collection", + collection_name=self._milvus_collection_name, + ) + if has_collection: + await self._validate_milvus_collection() + return + + _client_factory, data_type = self._get_milvus_symbols() + schema = await self._milvus_call( + "create_schema", + auto_id=False, + enable_dynamic_field=False, + ) + schema.add_field( + field_name="id", + datatype=data_type.VARCHAR, + is_primary=True, + max_length=_MILVUS_ID_MAX_LENGTH, + ) + schema.add_field(field_name="project_id", datatype=data_type.INT64) + schema.add_field(field_name="chunk_id", datatype=data_type.INT64) + schema.add_field(field_name="entity_id", datatype=data_type.INT64) + schema.add_field( + field_name="chunk_key", + datatype=data_type.VARCHAR, + max_length=512, + ) + schema.add_field( + field_name="chunk_text", + datatype=data_type.VARCHAR, + max_length=_MILVUS_TEXT_MAX_LENGTH, + ) + schema.add_field( + field_name="embedding", + datatype=data_type.FLOAT_VECTOR, + dim=self._vector_dimensions, + ) + + index_params = await self._milvus_call("prepare_index_params") + index_params.add_index( + field_name="embedding", + index_type="AUTOINDEX", + metric_type="COSINE", + ) + await self._milvus_call( + "create_collection", + collection_name=self._milvus_collection_name, + schema=schema, + index_params=index_params, + ) + + async def _validate_milvus_collection(self) -> None: + description = await self._milvus_call( + "describe_collection", + collection_name=self._milvus_collection_name, + ) + fields = description.get("fields", []) if isinstance(description, dict) else [] + fields_by_name = {str(field.get("name")): field for field in fields if "name" in field} + required_fields = { + "id", + "project_id", + "chunk_id", + "entity_id", + "chunk_key", + "chunk_text", + "embedding", + } + missing_fields = sorted(required_fields - set(fields_by_name)) + if missing_fields: + raise ValueError( + f"Milvus collection '{self._milvus_collection_name}' is missing required " + f"fields: {', '.join(missing_fields)}" + ) + + vector_field = fields_by_name["embedding"] + params = vector_field.get("params", {}) + actual_dim = params.get("dim") + if actual_dim is None: + return + if int(actual_dim) != self._vector_dimensions: + raise ValueError( + f"Milvus collection '{self._milvus_collection_name}' has embedding " + f"dimension {actual_dim}, but Basic Memory is configured for " + f"{self._vector_dimensions}. Use a new milvus_collection_name or recreate " + "the collection before reindexing." + ) + + def _milvus_id(self, chunk_id: int) -> str: + return f"{self.project_id}:{int(chunk_id)}" + + async def _run_vector_query( + self, + session: AsyncSession, + query_embedding: list[float], + candidate_limit: int, + ) -> list[dict]: + if not query_embedding: + return [] + + raw_results = await self._milvus_call( + "search", + collection_name=self._milvus_collection_name, + data=[query_embedding], + anns_field="embedding", + limit=candidate_limit, + filter=f"project_id == {int(self.project_id)}", + output_fields=["chunk_id", "entity_id", "chunk_key", "chunk_text"], + ) + return self._milvus_search_rows(raw_results) + + @staticmethod + def _milvus_search_rows(raw_results: Any) -> list[dict]: + hits = raw_results[0] if raw_results and isinstance(raw_results[0], list) else raw_results + rows: list[dict] = [] + for hit in hits or []: + if not isinstance(hit, dict): + continue + entity = hit.get("entity") + if not isinstance(entity, dict): + entity = hit + rows.append( + { + "entity_id": int(entity["entity_id"]), + "chunk_key": str(entity["chunk_key"]), + "chunk_text": str(entity.get("chunk_text", "")), + "best_distance": float(hit.get("distance", hit.get("score", 0.0))), + } + ) + return rows + + async def _write_embeddings( + self, + session: AsyncSession, + jobs: list[tuple[int, str]], + embeddings: list[list[float]], + ) -> None: + if not jobs: + return + + chunk_ids = [row_id for row_id, _ in jobs] + chunk_rows = await self._fetch_chunk_rows(session, chunk_ids) + records = [] + marker_rows = [] + for (chunk_id, chunk_text), embedding in zip(jobs, embeddings, strict=True): + chunk_row = chunk_rows.get(chunk_id) + if chunk_row is None: + raise RuntimeError(f"Vector chunk row {chunk_id} disappeared before write.") + records.append( + { + "id": self._milvus_id(chunk_id), + "project_id": int(self.project_id), + "chunk_id": int(chunk_id), + "entity_id": int(chunk_row["entity_id"]), + "chunk_key": str(chunk_row["chunk_key"]), + "chunk_text": str(chunk_text)[:_MILVUS_TEXT_MAX_LENGTH], + "embedding": embedding, + } + ) + marker_rows.append( + { + "chunk_id": int(chunk_id), + "project_id": int(self.project_id), + "embedding_dims": len(embedding), + } + ) + + await self._milvus_call( + "upsert", + collection_name=self._milvus_collection_name, + data=records, + ) + await self._replace_marker_rows(session, marker_rows) + + async def _fetch_chunk_rows( + self, + session: AsyncSession, + chunk_ids: list[int], + ) -> dict[int, dict[str, Any]]: + placeholders = ", ".join(f":chunk_id_{index}" for index in range(len(chunk_ids))) + params: dict[str, object] = { + "project_id": self.project_id, + **{f"chunk_id_{index}": chunk_id for index, chunk_id in enumerate(chunk_ids)}, + } + result = await session.execute( + text( + "SELECT id, entity_id, chunk_key, chunk_text " + "FROM search_vector_chunks " + f"WHERE project_id = :project_id AND id IN ({placeholders})" + ), + params, + ) + return {int(row["id"]): dict(row) for row in result.mappings().all()} + + async def _replace_marker_rows( + self, + session: AsyncSession, + marker_rows: list[dict[str, int]], + ) -> None: + chunk_ids = [row["chunk_id"] for row in marker_rows] + await self._delete_marker_rows(session, chunk_ids) + timestamp_expr = self._timestamp_now_expr() + await session.execute( + text( + "INSERT INTO search_vector_embeddings " + "(chunk_id, project_id, embedding_dims, updated_at) " + f"VALUES (:chunk_id, :project_id, :embedding_dims, {timestamp_expr})" + ), + marker_rows, + ) + + async def _delete_marker_rows(self, session: AsyncSession, chunk_ids: list[int]) -> None: + if not chunk_ids: + return + placeholders = ", ".join(f":chunk_id_{index}" for index in range(len(chunk_ids))) + params = {f"chunk_id_{index}": chunk_id for index, chunk_id in enumerate(chunk_ids)} + await session.execute( + text(f"DELETE FROM search_vector_embeddings WHERE chunk_id IN ({placeholders})"), + params, + ) + + async def _delete_milvus_ids(self, chunk_ids: list[int]) -> None: + if not chunk_ids: + return + await self._milvus_call( + "delete", + collection_name=self._milvus_collection_name, + ids=[self._milvus_id(chunk_id) for chunk_id in chunk_ids], + ) + + async def _delete_milvus_filter(self, filter_expr: str) -> None: + await self._milvus_call( + "delete", + collection_name=self._milvus_collection_name, + filter=filter_expr, + ) + + async def _delete_entity_chunks( + self, + session: AsyncSession, + entity_id: int, + ) -> None: + await self._delete_milvus_filter( + f"project_id == {int(self.project_id)} and entity_id == {int(entity_id)}" + ) + await session.execute( + text( + "DELETE FROM search_vector_embeddings " + "WHERE chunk_id IN (" + "SELECT id FROM search_vector_chunks " + "WHERE project_id = :project_id AND entity_id = :entity_id" + ")" + ), + {"project_id": self.project_id, "entity_id": entity_id}, + ) + await session.execute( + text( + "DELETE FROM search_vector_chunks " + "WHERE project_id = :project_id AND entity_id = :entity_id" + ), + {"project_id": self.project_id, "entity_id": entity_id}, + ) + + async def _delete_stale_chunks( + self, + session: AsyncSession, + stale_ids: list[int], + entity_id: int, + ) -> None: + await self._delete_milvus_ids(stale_ids) + await self._delete_marker_rows(session, stale_ids) + stale_placeholders = ", ".join(f":stale_id_{idx}" for idx in range(len(stale_ids))) + stale_params = { + "project_id": self.project_id, + "entity_id": entity_id, + **{f"stale_id_{idx}": row_id for idx, row_id in enumerate(stale_ids)}, + } + await session.execute( + text( + "DELETE FROM search_vector_chunks " + f"WHERE id IN ({stale_placeholders}) " + "AND project_id = :project_id AND entity_id = :entity_id" + ), + stale_params, + ) + + async def delete_project_vector_rows(self) -> None: + """Delete this project's SQL vector metadata and Milvus records.""" + await self._ensure_vector_tables() + await self._delete_milvus_filter(f"project_id == {int(self.project_id)}") + async with db.scoped_session(self.session_maker) as session: + await session.execute( + text("DELETE FROM search_vector_embeddings WHERE project_id = :project_id"), + {"project_id": self.project_id}, + ) + await session.execute( + text("DELETE FROM search_vector_chunks WHERE project_id = :project_id"), + {"project_id": self.project_id}, + ) + await session.commit() + + async def delete_stale_vector_rows(self) -> None: + """Delete stale SQL vector metadata and matching Milvus records.""" + await self._ensure_vector_tables() + async with db.scoped_session(self.session_maker) as session: + stale_result = await session.execute( + text( + "SELECT id FROM search_vector_chunks " + "WHERE project_id = :project_id " + "AND entity_id NOT IN (" + "SELECT id FROM entity WHERE project_id = :project_id" + ")" + ), + {"project_id": self.project_id}, + ) + stale_ids = [int(row[0]) for row in stale_result.fetchall()] + await self._delete_milvus_ids(stale_ids) + await self._delete_marker_rows(session, stale_ids) + await session.execute( + text( + "DELETE FROM search_vector_chunks " + "WHERE project_id = :project_id " + "AND entity_id NOT IN (" + "SELECT id FROM entity WHERE project_id = :project_id" + ")" + ), + {"project_id": self.project_id}, + ) + await session.commit() + + async def drop_vector_tables(self) -> None: + """Clear Milvus project records and drop derived SQL vector metadata.""" + await self._ensure_vector_tables() + await self._delete_milvus_filter(f"project_id == {int(self.project_id)}") + async with db.scoped_session(self.session_maker) as session: + await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings")) + await session.execute(text("DROP TABLE IF EXISTS search_vector_chunks")) + await session.execute(text("DROP TABLE IF EXISTS search_vector_index")) + await session.commit() + self._vector_tables_initialized = False + + def _distance_to_similarity(self, distance: float) -> float: + """Milvus COSINE search returns higher scores for better matches.""" + return max(0.0, min(1.0, distance)) + + +class SQLiteMilvusSearchRepository(MilvusVectorSearchMixin, SQLiteSearchRepository): + """SQLite FTS repository with Milvus-backed semantic vectors.""" + + def __init__( + self, + session_maker, + project_id: int, + app_config: BasicMemoryConfig | None = None, + embedding_provider: EmbeddingProvider | None = None, + milvus_client_factory: MilvusClientFactory | None = None, + milvus_data_type: Any | None = None, + ): + super().__init__( + session_maker, + project_id=project_id, + app_config=app_config, + embedding_provider=embedding_provider, + ) + self._configure_milvus_vector_backend( + app_config or ConfigManager().config, + milvus_client_factory=milvus_client_factory, + milvus_data_type=milvus_data_type, + ) + + +class PostgresMilvusSearchRepository(MilvusVectorSearchMixin, PostgresSearchRepository): + """Postgres FTS repository with Milvus-backed semantic vectors.""" + + def __init__( + self, + session_maker, + project_id: int, + app_config: BasicMemoryConfig | None = None, + embedding_provider: EmbeddingProvider | None = None, + milvus_client_factory: MilvusClientFactory | None = None, + milvus_data_type: Any | None = None, + ): + super().__init__( + session_maker, + project_id=project_id, + app_config=app_config, + embedding_provider=embedding_provider, + ) + self._configure_milvus_vector_backend( + app_config or ConfigManager().config, + milvus_client_factory=milvus_client_factory, + milvus_data_type=milvus_data_type, + ) + + +__all__ = [ + "PostgresMilvusSearchRepository", + "SQLiteMilvusSearchRepository", + "delete_milvus_project_vectors", +] diff --git a/src/basic_memory/repository/project_repository.py b/src/basic_memory/repository/project_repository.py index 6971c3c5b..00af97c30 100644 --- a/src/basic_memory/repository/project_repository.py +++ b/src/basic_memory/repository/project_repository.py @@ -227,6 +227,26 @@ async def delete(self, session: AsyncSession, entity_id: int) -> bool: {"project_id": entity_id}, ) + from basic_memory.config import ConfigManager, SemanticVectorBackend + + config = ConfigManager().config + uses_milvus = config.semantic_vector_backend == SemanticVectorBackend.MILVUS + if uses_milvus: + from basic_memory.repository.milvus_search_repository import ( + delete_milvus_project_vectors, + ) + from basic_memory.repository.semantic_errors import ( + SemanticDependenciesMissingError, + ) + + try: + await delete_milvus_project_vectors(config, entity_id) + except SemanticDependenciesMissingError as exc: + logger.warning( + "Skipping Milvus project vector purge because pymilvus is unavailable: {}", + exc, + ) + # search_vector_chunks: no FK to project on either backend, so both # backends need this. SQLite must purge vec0 embeddings first # (rowid pseudocolumn — Postgres uses chunk_id and would 500 here); @@ -234,19 +254,32 @@ async def delete(self, session: AsyncSession, entity_id: int) -> bool: # we delete the chunk rows below. if "search_vector_chunks" in existing_tables: if is_sqlite and "search_vector_embeddings" in existing_tables: - # Extension loading is per-connection. We must load vec0 on - # *this* session before the DELETE; otherwise a different - # pooled connection might have written embeddings that we'd - # silently leave behind. - if await _load_sqlite_vec_on_session(session): + if uses_milvus: + # Milvus stores SQLite marker rows in a regular SQL table, + # not a sqlite-vec virtual table, so deleting them must not + # depend on loading the sqlite-vec extension. await session.execute( text( - "DELETE FROM search_vector_embeddings WHERE rowid IN (" + "DELETE FROM search_vector_embeddings WHERE chunk_id IN (" "SELECT id FROM search_vector_chunks " "WHERE project_id = :project_id)" ), {"project_id": entity_id}, ) + else: + # Extension loading is per-connection. We must load vec0 on + # *this* session before the DELETE; otherwise a different + # pooled connection might have written embeddings that we'd + # silently leave behind. + if await _load_sqlite_vec_on_session(session): + await session.execute( + text( + "DELETE FROM search_vector_embeddings WHERE rowid IN (" + "SELECT id FROM search_vector_chunks " + "WHERE project_id = :project_id)" + ), + {"project_id": entity_id}, + ) await session.execute( text("DELETE FROM search_vector_chunks WHERE project_id = :project_id"), {"project_id": entity_id}, diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index 47395a509..8cf550d2e 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -12,8 +12,17 @@ from sqlalchemy import Result from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from basic_memory.config import BasicMemoryConfig, ConfigManager, DatabaseBackend +from basic_memory.config import ( + BasicMemoryConfig, + ConfigManager, + DatabaseBackend, + SemanticVectorBackend, +) from basic_memory.repository.embedding_provider_factory import create_embedding_provider +from basic_memory.repository.milvus_search_repository import ( + PostgresMilvusSearchRepository, + SQLiteMilvusSearchRepository, +) from basic_memory.repository.postgres_search_repository import PostgresSearchRepository from basic_memory.repository.search_index_row import SearchIndexRow from basic_memory.repository.search_repository_base import VectorSyncBatchResult @@ -96,6 +105,18 @@ async def delete_entity_vector_rows(self, entity_id: int) -> None: """Delete semantic vector chunks and embeddings for one entity.""" ... + async def delete_project_vector_rows(self) -> None: + """Delete semantic vector chunks and embeddings for this repository project.""" + ... + + async def delete_stale_vector_rows(self) -> None: + """Delete semantic vector rows whose source entities no longer exist.""" + ... + + async def drop_vector_tables(self) -> None: + """Drop or clear backend-specific vector storage.""" + ... + async def sync_entity_vectors_batch( self, entity_ids: list[int], @@ -143,21 +164,27 @@ def create_search_repository( if config.semantic_search_enabled: embedding_provider = create_embedding_provider(config) + use_milvus = config.semantic_vector_backend == SemanticVectorBackend.MILVUS + if database_backend == DatabaseBackend.POSTGRES: # pragma: no cover - return PostgresSearchRepository( # pragma: no cover - session_maker, - project_id=project_id, - app_config=app_config, - embedding_provider=embedding_provider, + repository_class = ( + PostgresMilvusSearchRepository if use_milvus else PostgresSearchRepository ) - else: - return SQLiteSearchRepository( + return repository_class( # pragma: no cover session_maker, project_id=project_id, app_config=app_config, embedding_provider=embedding_provider, ) + repository_class = SQLiteMilvusSearchRepository if use_milvus else SQLiteSearchRepository + return repository_class( + session_maker, + project_id=project_id, + app_config=app_config, + embedding_provider=embedding_provider, + ) + __all__ = [ "SearchRepository", diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index aa5f094b1..341b93247 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -590,6 +590,43 @@ async def delete_entity_vector_rows(self, entity_id: int) -> None: await self._delete_entity_chunks(session, entity_id) await session.commit() + async def delete_project_vector_rows(self) -> None: + """Delete this project's derived vector rows using backend cascade semantics.""" + await self._ensure_vector_tables() + + async with db.scoped_session(self.session_maker) as session: + await self._prepare_vector_session(session) + await session.execute( + text("DELETE FROM search_vector_chunks WHERE project_id = :project_id"), + {"project_id": self.project_id}, + ) + await session.commit() + + async def delete_stale_vector_rows(self) -> None: + """Delete vector rows whose source entities no longer exist.""" + await self._ensure_vector_tables() + + async with db.scoped_session(self.session_maker) as session: + await self._prepare_vector_session(session) + await session.execute( + text( + "DELETE FROM search_vector_chunks " + "WHERE project_id = :project_id " + "AND entity_id NOT IN (SELECT id FROM entity WHERE project_id = :project_id)" + ), + {"project_id": self.project_id}, + ) + await session.commit() + + async def drop_vector_tables(self) -> None: + """Drop SQL vector tables for backends whose vectors live in the database.""" + async with db.scoped_session(self.session_maker) as session: + await session.execute(text("DROP TABLE IF EXISTS search_vector_embeddings")) + await session.execute(text("DROP TABLE IF EXISTS search_vector_chunks")) + await session.execute(text("DROP TABLE IF EXISTS search_vector_index")) + await session.commit() + self._vector_tables_initialized = False + # ------------------------------------------------------------------ # Shared semantic search: guard, text processing, chunking # ------------------------------------------------------------------ diff --git a/src/basic_memory/services/project_service.py b/src/basic_memory/services/project_service.py index f112c92e5..4495f41bf 100644 --- a/src/basic_memory/services/project_service.py +++ b/src/basic_memory/services/project_service.py @@ -26,6 +26,7 @@ ) from basic_memory.config import ( DatabaseBackend, + SemanticVectorBackend, WATCH_STATUS_JSON, ConfigManager, ProjectEntry, @@ -1040,6 +1041,7 @@ async def get_embedding_status(self, project_id: int) -> EmbeddingStatus: query_prefix_set = bool(config.semantic_embedding_query_prefix) is_postgres = config.database_backend == DatabaseBackend.POSTGRES + uses_milvus = config.semantic_vector_backend == SemanticVectorBackend.MILVUS # --- Check vector table existence --- # Both search_vector_chunks and search_vector_embeddings must exist @@ -1127,7 +1129,7 @@ async def get_embedding_status(self, project_id: int) -> EmbeddingStatus: total_entities_with_chunks = entities_with_chunks_result.scalar() or 0 # Embeddings count — join pattern differs between SQLite and Postgres - if is_postgres: + if is_postgres or uses_milvus: embeddings_sql = text( "SELECT COUNT(*) FROM search_vector_chunks c " "JOIN search_vector_embeddings e ON e.chunk_id = c.id " @@ -1143,10 +1145,10 @@ async def get_embedding_status(self, project_id: int) -> EmbeddingStatus: # The embeddings/orphan JOINs read search_vector_embeddings, a vec0 # virtual table. On SQLite that table is only visible on a connection # that loaded sqlite-vec, so route these through scalar_vec_query which - # loads the extension first. Postgres has no per-connection extension - # and uses the bare pooled session. + # loads the extension first. Postgres and Milvus marker tables have no + # per-connection extension and use the bare pooled session. async def _vec_scalar(vec_sql) -> int: - if is_postgres: + if is_postgres or uses_milvus: result = await self.repository.execute_query( session, vec_sql, {"project_id": project_id} ) @@ -1167,7 +1169,7 @@ async def _vec_scalar(vec_sql) -> int: total_embeddings = await _vec_scalar(embeddings_sql) # Orphaned chunks (chunks without embeddings — indicates interrupted indexing) - if is_postgres: + if is_postgres or uses_milvus: orphan_sql = text( "SELECT COUNT(*) FROM search_vector_chunks c " "LEFT JOIN search_vector_embeddings e ON e.chunk_id = c.id " @@ -1186,7 +1188,11 @@ async def _vec_scalar(vec_sql) -> int: # Why: project info should degrade gracefully instead of crashing on stats queries. # Outcome: report vector tables as unavailable and point the user to install the # missing dependency before rebuilding embeddings. - if is_postgres or "no such module: vec0" not in str(exc).lower(): + if ( + is_postgres + or uses_milvus + or "no such module: vec0" not in str(exc).lower() + ): raise return EmbeddingStatus( diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index e8efabce1..496da960c 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -96,23 +96,10 @@ async def init_search_index(self): async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None: """Reindex all content from database.""" - from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository - logger.info("Starting full reindex") # Clear and recreate search index await self.repository.execute_query(text("DROP TABLE IF EXISTS search_index"), params={}) - if isinstance(self.repository, SQLiteSearchRepository): - await self.repository.drop_vector_tables() - else: - await self.repository.execute_query( - text("DROP TABLE IF EXISTS search_vector_embeddings"), params={} - ) - await self.repository.execute_query( - text("DROP TABLE IF EXISTS search_vector_chunks"), params={} - ) - await self.repository.execute_query( - text("DROP TABLE IF EXISTS search_vector_index"), params={} - ) + await self.repository.drop_vector_tables() await self.init_search_index() # Reindex all entities @@ -605,20 +592,8 @@ async def _clear_project_vectors_for_full_reindex(self) -> None: we need to clear the derived vector state first to force fresh embeddings. Outcome: the next batch sync recreates every eligible entity's vectors. """ - from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository - project_id = self.repository.project_id - params = {"project_id": project_id} - - # Constraint: sqlite-vec stores embeddings in a separate rowid table with - # no cascade delete, so embeddings must be removed before chunk rows. - if isinstance(self.repository, SQLiteSearchRepository): - await self.repository.delete_project_vector_rows() - else: - await self.repository.execute_query( - text("DELETE FROM search_vector_chunks WHERE project_id = :project_id"), - params, - ) + await self.repository.delete_project_vector_rows() logger.info("Cleared project vectors for full reindex", project_id=project_id) async def _purge_stale_search_rows(self) -> None: @@ -628,9 +603,6 @@ async def _purge_stale_search_rows(self) -> None: Why: stale rows inflate embedding coverage stats in project info Outcome: search tables only contain rows for entities that still exist """ - from basic_memory.repository.sqlite_search_repository import SQLiteSearchRepository - from sqlalchemy import text - project_id = self.repository.project_id stale_entity_filter = ( "entity_id NOT IN (SELECT id FROM entity WHERE project_id = :project_id)" @@ -645,18 +617,7 @@ async def _purge_stale_search_rows(self) -> None: params, ) - # SQLite vec has no CASCADE — must delete embeddings before chunks - if isinstance(self.repository, SQLiteSearchRepository): - await self.repository.delete_stale_vector_rows() - else: - # Postgres CASCADE handles embedding deletion automatically - await self.repository.execute_query( - text( - f"DELETE FROM search_vector_chunks " - f"WHERE project_id = :project_id AND {stale_entity_filter}" - ), - params, - ) + await self.repository.delete_stale_vector_rows() logger.info("Purged stale search rows for deleted entities", project_id=project_id) diff --git a/tests/repository/test_milvus_search_repository.py b/tests/repository/test_milvus_search_repository.py new file mode 100644 index 000000000..46e33f335 --- /dev/null +++ b/tests/repository/test_milvus_search_repository.py @@ -0,0 +1,397 @@ +"""Milvus-backed semantic vector repository tests.""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from typing import Any, cast + +import pytest +from sqlalchemy import text + +from basic_memory import db +from basic_memory.config import ( + BasicMemoryConfig, + DatabaseBackend, + ProjectEntry, + SemanticVectorBackend, +) +from basic_memory.repository.milvus_search_repository import ( + SQLiteMilvusSearchRepository, +) +from basic_memory.repository.search_index_row import SearchIndexRow +from basic_memory.repository.search_repository import create_search_repository +from basic_memory.repository.semantic_errors import SemanticDependenciesMissingError +from basic_memory.schemas.search import SearchItemType, SearchRetrievalMode + + +class StubEmbeddingProvider: + """Deterministic embedding provider for Milvus repository tests.""" + + model_name = "stub" + dimensions = 4 + + async def embed_query(self, text: str) -> list[float]: + return self._vectorize(text) + + async def embed_documents(self, texts: list[str]) -> list[list[float]]: + return [self._vectorize(text) for text in texts] + + def runtime_log_attrs(self) -> dict[str, object]: + return {} + + @staticmethod + def _vectorize(text: str) -> list[float]: + normalized = text.lower() + if any(token in normalized for token in ["auth", "token", "session", "login"]): + return [1.0, 0.0, 0.0, 0.0] + if any(token in normalized for token in ["schema", "migration", "database", "sql"]): + return [0.0, 1.0, 0.0, 0.0] + return [0.0, 0.0, 0.0, 1.0] + + +class FakeDataType: + """Small stand-in for pymilvus.DataType.""" + + VARCHAR = "VARCHAR" + INT64 = "INT64" + FLOAT_VECTOR = "FLOAT_VECTOR" + + +class FakeSchema: + """Collect schema fields added by the repository.""" + + def __init__(self) -> None: + self.fields: list[dict[str, Any]] = [] + + def add_field(self, **kwargs: Any) -> None: + params: dict[str, Any] = {} + if "dim" in kwargs: + params["dim"] = kwargs["dim"] + if "max_length" in kwargs: + params["max_length"] = kwargs["max_length"] + field = { + "name": kwargs["field_name"], + "type": kwargs["datatype"], + "params": params, + } + if kwargs.get("is_primary"): + field["is_primary"] = True + self.fields.append(field) + + +class FakeIndexParams: + """Collect index definitions added by the repository.""" + + def __init__(self) -> None: + self.indexes: list[dict[str, Any]] = [] + + def add_index(self, **kwargs: Any) -> None: + self.indexes.append(dict(kwargs)) + + +class FakeMilvusClient: + """In-memory MilvusClient double with collection-level state.""" + + collections: dict[str, dict[str, Any]] = {} + instances: list["FakeMilvusClient"] = [] + + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + self.instances.append(self) + + @classmethod + def reset(cls) -> None: + cls.collections = {} + cls.instances = [] + + def has_collection(self, *, collection_name: str) -> bool: + return collection_name in self.collections + + def create_schema(self, *, auto_id: bool, enable_dynamic_field: bool) -> FakeSchema: + assert auto_id is False + assert enable_dynamic_field is False + return FakeSchema() + + def prepare_index_params(self) -> FakeIndexParams: + return FakeIndexParams() + + def create_collection( + self, + *, + collection_name: str, + schema: FakeSchema, + index_params: FakeIndexParams, + ) -> None: + self.collections[collection_name] = { + "fields": list(schema.fields), + "indexes": list(index_params.indexes), + "records": {}, + } + + def describe_collection(self, *, collection_name: str) -> dict[str, Any]: + collection = self.collections[collection_name] + return {"fields": collection["fields"]} + + def upsert(self, *, collection_name: str, data: list[dict[str, Any]]) -> None: + records = self.collections[collection_name]["records"] + for record in data: + records[record["id"]] = dict(record) + + def search( + self, + *, + collection_name: str, + data: list[list[float]], + anns_field: str, + limit: int, + filter: str, + output_fields: list[str], + ) -> list[list[dict[str, Any]]]: + assert anns_field == "embedding" + assert output_fields == ["chunk_id", "entity_id", "chunk_key", "chunk_text"] + project_id = int(filter.split("==", 1)[1].strip()) + query = data[0] + hits = [] + for record in self.collections[collection_name]["records"].values(): + if int(record["project_id"]) != project_id: + continue + score = sum(a * b for a, b in zip(query, record["embedding"], strict=True)) + norm = math.sqrt(sum(value * value for value in record["embedding"])) or 1.0 + hits.append( + { + "id": record["id"], + "distance": score / norm, + "entity": {field: record[field] for field in output_fields}, + } + ) + hits.sort(key=lambda hit: hit["distance"], reverse=True) + return [hits[:limit]] + + def delete( + self, + *, + collection_name: str, + ids: list[str] | None = None, + filter: str | None = None, + ) -> None: + records = self.collections[collection_name]["records"] + if ids is not None: + for record_id in ids: + records.pop(record_id, None) + return + + assert filter is not None + filters = { + part.strip().split("==", 1)[0].strip(): int(part.strip().split("==", 1)[1]) + for part in filter.split("and") + } + for record_id, record in list(records.items()): + if all(int(record[field]) == expected for field, expected in filters.items()): + records.pop(record_id, None) + + +def _milvus_config(tmp_path) -> BasicMemoryConfig: + return BasicMemoryConfig( + env="test", + projects={"test-project": ProjectEntry(path=str(tmp_path))}, + default_project="test-project", + database_backend=DatabaseBackend.SQLITE, + semantic_search_enabled=True, + semantic_vector_backend=SemanticVectorBackend.MILVUS, + milvus_uri=str(tmp_path / "milvus.db"), + milvus_token="test-token", + semantic_embedding_sync_batch_size=8, + ) + + +def _row( + *, + project_id: int, + row_id: int, + entity_id: int, + title: str, + permalink: str, + content: str, +) -> SearchIndexRow: + now = datetime.now(timezone.utc) + return SearchIndexRow( + project_id=project_id, + id=row_id, + type=SearchItemType.ENTITY.value, + title=title, + permalink=permalink, + file_path=f"{permalink}.md", + metadata={"note_type": "spec"}, + entity_id=entity_id, + content_stems=content, + content_snippet=content, + created_at=now, + updated_at=now, + ) + + +def _repo(session_maker, test_project, tmp_path) -> SQLiteMilvusSearchRepository: + return SQLiteMilvusSearchRepository( + session_maker, + project_id=test_project.id, + app_config=_milvus_config(tmp_path), + embedding_provider=StubEmbeddingProvider(), + milvus_client_factory=FakeMilvusClient, + milvus_data_type=FakeDataType, + ) + + +@pytest.fixture(autouse=True) +def _reset_fake_milvus() -> None: + FakeMilvusClient.reset() + + +@pytest.mark.asyncio +async def test_milvus_backend_creates_collection_and_marker_rows( + session_maker, + test_project, + tmp_path, +): + """Milvus backend stores real vectors externally and SQL marker rows locally.""" + repo = _repo(session_maker, test_project, tmp_path) + + await repo.init_search_index() + await repo.bulk_index_items( + [ + _row( + project_id=test_project.id, + row_id=101, + entity_id=101, + title="Authentication", + permalink="specs/authentication", + content="auth token session login", + ), + _row( + project_id=test_project.id, + row_id=102, + entity_id=102, + title="Database", + permalink="specs/database", + content="database schema migration", + ), + ] + ) + + result = await repo.sync_entity_vectors_batch([101, 102]) + + assert result.entities_synced == 2 + assert FakeMilvusClient.instances[0].kwargs == { + "uri": str(tmp_path / "milvus.db"), + "token": "test-token", + } + collection = FakeMilvusClient.collections["basic_memory_vectors"] + assert collection["indexes"] == [ + { + "field_name": "embedding", + "index_type": "AUTOINDEX", + "metric_type": "COSINE", + } + ] + assert collection["records"] + + async with db.scoped_session(session_maker) as session: + marker_count = await session.execute( + text( + "SELECT COUNT(*) FROM search_vector_embeddings e " + "JOIN search_vector_chunks c ON c.id = e.chunk_id " + "WHERE c.project_id = :project_id" + ), + {"project_id": test_project.id}, + ) + assert int(marker_count.scalar_one()) == len(collection["records"]) + + results = await repo.search( + search_text="session token auth", + retrieval_mode=SearchRetrievalMode.VECTOR, + limit=5, + offset=0, + ) + + assert results + assert results[0].permalink == "specs/authentication" + + +@pytest.mark.asyncio +async def test_milvus_backend_deletes_external_records_with_entity_chunks( + session_maker, + test_project, + tmp_path, +): + """Deleting entity vectors clears both SQL marker rows and Milvus records.""" + repo = _repo(session_maker, test_project, tmp_path) + + await repo.init_search_index() + await repo.index_item( + _row( + project_id=test_project.id, + row_id=201, + entity_id=201, + title="Authentication", + permalink="specs/authentication", + content="auth token session login", + ) + ) + await repo.sync_entity_vectors(201) + assert FakeMilvusClient.collections["basic_memory_vectors"]["records"] + + await repo.delete_entity_vector_rows(201) + + assert FakeMilvusClient.collections["basic_memory_vectors"]["records"] == {} + async with db.scoped_session(session_maker) as session: + chunk_count = await session.execute( + text( + "SELECT COUNT(*) FROM search_vector_chunks " + "WHERE project_id = :project_id AND entity_id = :entity_id" + ), + {"project_id": test_project.id, "entity_id": 201}, + ) + marker_count = await session.execute( + text("SELECT COUNT(*) FROM search_vector_embeddings"), + ) + assert int(chunk_count.scalar_one()) == 0 + assert int(marker_count.scalar_one()) == 0 + + +def test_create_search_repository_selects_sqlite_milvus_backend(tmp_path): + """Factory should keep SQL backend selection but swap semantic vector storage.""" + config = _milvus_config(tmp_path) + config.semantic_search_enabled = False + + repo = create_search_repository(cast(Any, None), project_id=1, app_config=config) + + assert isinstance(repo, SQLiteMilvusSearchRepository) + + +@pytest.mark.asyncio +async def test_milvus_backend_reports_missing_optional_dependency( + session_maker, + test_project, + tmp_path, + monkeypatch, +): + """Selecting Milvus without pymilvus should fail with the semantic dependency error.""" + repo = SQLiteMilvusSearchRepository( + session_maker, + project_id=test_project.id, + app_config=_milvus_config(tmp_path), + embedding_provider=StubEmbeddingProvider(), + ) + + def _missing_pymilvus(name: str) -> Any: + if name == "pymilvus": + raise ImportError(name) + return __import__(name) + + monkeypatch.setattr( + "basic_memory.repository.milvus_search_repository.importlib.import_module", + _missing_pymilvus, + ) + + with pytest.raises(SemanticDependenciesMissingError): + await repo._ensure_vector_tables() diff --git a/tests/services/test_project_service_embedding_status.py b/tests/services/test_project_service_embedding_status.py index 2b12aa3c7..ada9b8398 100644 --- a/tests/services/test_project_service_embedding_status.py +++ b/tests/services/test_project_service_embedding_status.py @@ -7,6 +7,7 @@ from sqlalchemy import text from basic_memory import db +from basic_memory.config import SemanticVectorBackend from basic_memory.schemas.project_info import EmbeddingStatus from basic_memory.services.project_service import ProjectService @@ -297,6 +298,84 @@ async def test_embedding_status_healthy(project_service: ProjectService, test_gr assert status.reindex_reason is None +@pytest.mark.asyncio +async def test_embedding_status_milvus_sqlite_uses_plain_marker_table( + project_service: ProjectService, test_graph, test_project +): + """Milvus marker tables are regular SQL tables, not sqlite-vec virtual tables.""" + if _is_postgres(): + pytest.skip("This regression covers the SQLite marker-table branch.") + + await _execute(project_service, text("DELETE FROM search_vector_chunks"), {}) + await _execute( + project_service, + text("DROP TABLE IF EXISTS search_vector_embeddings"), {} + ) + await _execute( + project_service, + text( + "CREATE TABLE search_vector_embeddings (" + "chunk_id INTEGER PRIMARY KEY, project_id INTEGER, embedding_dims INTEGER)" + ), + {}, + ) + + entity_result = await _execute( + project_service, + text("SELECT DISTINCT entity_id FROM search_index WHERE project_id = :project_id LIMIT 1"), + {"project_id": test_project.id}, + ) + entity_id = entity_result.scalar_one() + await _execute( + project_service, + text( + "INSERT INTO search_vector_chunks " + "(id, entity_id, project_id, chunk_key, chunk_text, source_hash, " + "entity_fingerprint, embedding_model) " + "VALUES (1, :entity_id, :project_id, 'chunk-1', 'text', 'hash', " + "'fp-hash', 'stub')" + ), + {"entity_id": entity_id, "project_id": test_project.id}, + ) + await _execute( + project_service, + text( + "INSERT INTO search_vector_embeddings " + "(chunk_id, project_id, embedding_dims) VALUES (1, :project_id, 4)" + ), + {"project_id": test_project.id}, + ) + + async def fail_if_sqlite_vec_path_is_used(query, params=None): + raise AssertionError("Milvus marker tables should use regular SQL execution") + + with patch.object( + type(project_service), + "config_manager", + new_callable=lambda: property( + lambda self: _config_manager_with( + semantic_search_enabled=True, + semantic_vector_backend=SemanticVectorBackend.MILVUS, + ) + ), + ): + with patch.object( + project_service.repository, + "scalar_vec_query", + side_effect=fail_if_sqlite_vec_path_is_used, + ): + status = await project_service.get_embedding_status(test_project.id) + + await _execute( + project_service, + text("DROP TABLE IF EXISTS search_vector_embeddings"), {} + ) + + assert status.vector_tables_exist is True + assert status.total_embeddings == 1 + assert status.orphaned_chunks == 0 + + @pytest.mark.asyncio async def test_embedding_status_excludes_stale_entity_ids( project_service: ProjectService, test_graph, test_project @@ -364,11 +443,15 @@ async def test_get_project_info_includes_embedding_status( # --- Helper --- -def _config_manager_with(semantic_search_enabled: bool): +def _config_manager_with( + semantic_search_enabled: bool, + semantic_vector_backend: SemanticVectorBackend = SemanticVectorBackend.DATABASE, +): """Create a ConfigManager whose config has the given semantic_search_enabled value.""" from basic_memory.config import ConfigManager cm = ConfigManager() # Patch the config object in-place cm.config.semantic_search_enabled = semantic_search_enabled + cm.config.semantic_vector_backend = semantic_vector_backend return cm diff --git a/tests/test_config.py b/tests/test_config.py index c2f1588ca..89f917d84 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -12,6 +12,7 @@ ConfigManager, ProjectEntry, ProjectMode, + SemanticVectorBackend, default_fastembed_cache_dir, resolve_data_dir, ) @@ -1260,6 +1261,28 @@ def test_default_search_type_rejects_invalid_values(self): with pytest.raises(Exception): BasicMemoryConfig(default_search_type="invalid") + def test_semantic_vector_backend_defaults_to_database(self): + """Vector storage should stay in the active database by default.""" + config = BasicMemoryConfig() + assert config.semantic_vector_backend == SemanticVectorBackend.DATABASE + + def test_semantic_vector_backend_accepts_milvus(self, tmp_path): + """Milvus can be selected without changing the SQL database backend.""" + config = BasicMemoryConfig( + semantic_vector_backend="milvus", + milvus_uri=str(tmp_path / "milvus.db"), + milvus_token="token", + ) + assert config.semantic_vector_backend == SemanticVectorBackend.MILVUS + assert config.milvus_uri == str(tmp_path / "milvus.db") + assert config.milvus_token == "token" + + def test_milvus_uri_supports_unprefixed_env_var(self, monkeypatch): + """MILVUS_URI should work in addition to BASIC_MEMORY_MILVUS_URI.""" + monkeypatch.setenv("MILVUS_URI", "./custom-milvus.db") + config = BasicMemoryConfig() + assert config.milvus_uri == "./custom-milvus.db" + class TestFormattingConfig: """Test file formatting configuration options.""" diff --git a/uv.lock b/uv.lock index 703593816..2977f0995 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.12" resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version == '3.13.*'", - "python_full_version < '3.13'", + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -323,6 +329,11 @@ dependencies = [ { name = "watchfiles" }, ] +[package.optional-dependencies] +milvus = [ + { name = "pymilvus", extra = ["milvus-lite"] }, +] + [package.dev-dependencies] dev = [ { name = "cst-lsp" }, @@ -376,6 +387,7 @@ requires-dist = [ { name = "pydantic", extras = ["email", "timezone"], specifier = ">=2.12.0" }, { name = "pydantic-settings", specifier = ">=2.6.1" }, { name = "pyjwt", specifier = ">=2.10.1" }, + { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=2.6.0,<4.0.0" }, { name = "pyright", specifier = ">=1.1.390" }, { name = "pytest-aio", specifier = ">=1.9.0" }, { name = "pytest-asyncio", specifier = ">=1.2.0" }, @@ -391,6 +403,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32'", specifier = ">=0.21.0" }, { name = "watchfiles", specifier = ">=1.0.4" }, ] +provides-extras = ["milvus"] [package.metadata.requires-dev] dev = [ @@ -898,6 +911,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "faiss-cpu" +version = "1.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, + { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, +] + [[package]] name = "fastapi" version = "0.139.0" @@ -1429,6 +1459,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, ] +[[package]] +name = "grpcio" +version = "1.82.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/19/e29d3979b420b92d516ee97f0bff4fb88cbb3d724791318f50beb55db549/grpcio-1.82.0.tar.gz", hash = "sha256:bfe3247e0bb598585ce26565730970d21bccf3c9dc547dc6703a1a3c7888e8e1", size = 13184479, upload-time = "2026-07-06T04:22:54.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/e6/9ce68bc431224177b6f506cfb4896a742119c51255143069970955b6510b/grpcio-1.82.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:03cce11532da292215cd8e5f444de91982e39dfb41a916e0e0f91a5c128588ea", size = 6144680, upload-time = "2026-07-06T04:21:29.808Z" }, + { url = "https://files.pythonhosted.org/packages/59/93/00ecf28e1be7a70b4b4d148d3a551b6c9a37024a669c3650a2c374c64026/grpcio-1.82.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:01336fe31d1ea5d5154b0d803eb3584e69fb96e0e657acc87bd4d48d6abc9587", size = 11952213, upload-time = "2026-07-06T04:21:32.347Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/9be6bf4cddb5b5f39c0748cdf5639525cd4116a157c8f3802c9217e54882/grpcio-1.82.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe56f1bea709ddb2531fbd510a2dd6040e993985507c3b2bf3404507aee989b1", size = 6710770, upload-time = "2026-07-06T04:21:35.382Z" }, + { url = "https://files.pythonhosted.org/packages/00/97/62c0969e993673ebef16d526db2504f38bc4c73cf2593c28746791ab7956/grpcio-1.82.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:77a5d8fe331376c7aaa4e06e903a43bd144de4f98c6e414e13cbc5d64e5aa61e", size = 7450671, upload-time = "2026-07-06T04:21:37.923Z" }, + { url = "https://files.pythonhosted.org/packages/67/1f/d83d9fbaaef2b6ebbf70a6e467000f13f92b4cf0b84c561c162b43128439/grpcio-1.82.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9ce80ea66c803398fde9c5b1fd925920c9742da7f10f8b85acac0adac3e4d7e0", size = 6886855, upload-time = "2026-07-06T04:21:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/c5/36/5ed2e28b8c5f00599c7d5e94d5f82c32cf73a6fa42d13c2900cb37c861ec/grpcio-1.82.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4665dc8c9ec30acff455e2b15c47b825f18647c555483aa1ccf670adead8adcc", size = 7501322, upload-time = "2026-07-06T04:21:43.78Z" }, + { url = "https://files.pythonhosted.org/packages/89/4e/69113709e14e24289940d6aef067b797c73c8c96f580683af7d5f09b22b0/grpcio-1.82.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4405d19ecfc7a8caa9043ff5a651e1871b54fc620f917de5dede7e6fb78e9e14", size = 8536896, upload-time = "2026-07-06T04:21:46.387Z" }, + { url = "https://files.pythonhosted.org/packages/30/f6/08bd6eb89a558f0f59dacaef747afda7c21e2c2ecf138ae2ec533dea8aeb/grpcio-1.82.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:30e580c19178609799660f52b1e1eb09248969be20dc44caaa4dcaa3e8450dd9", size = 7913890, upload-time = "2026-07-06T04:21:49.733Z" }, + { url = "https://files.pythonhosted.org/packages/d5/a1/04b467e2ffc93f8e50b660bc6f47c3c8e9ba7603104a741e3d5663a261d4/grpcio-1.82.0-cp312-cp312-win32.whl", hash = "sha256:dbbd83dbaa856b387a4eba74ec3add7f594f090d3b4d390d829dd5834816aae4", size = 4240969, upload-time = "2026-07-06T04:21:52.281Z" }, + { url = "https://files.pythonhosted.org/packages/25/3c/15d3601384d5937e8d9fbe6d8e7b8171008d649744dd7b6c864932937ec9/grpcio-1.82.0-cp312-cp312-win_amd64.whl", hash = "sha256:73176d270699d76a9dc3753f25e01c19fb98f830f826a506e917d83629f1b39b", size = 5001575, upload-time = "2026-07-06T04:21:54.839Z" }, + { url = "https://files.pythonhosted.org/packages/f0/c8/7aace81e7cd12c6ffccf0d47ac389a3f19e5280e9ab04d301d02855ecd25/grpcio-1.82.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:841bcaa24882921d41cd7a82118f9a766edcf8807c2f486e7deeb0ff5ea36181", size = 6146066, upload-time = "2026-07-06T04:21:57.527Z" }, + { url = "https://files.pythonhosted.org/packages/10/17/b448f2265e4927188425c0b09fb943c39b50f7f53fbead644b44ccdf834b/grpcio-1.82.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:d4bb7219c39c735e6573ae3c33aeafb2a4e1f1cc93a762f7899c283defaed365", size = 11948617, upload-time = "2026-07-06T04:22:00.066Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a3/fba54e50fffc9f74b1ffd5eb4e47310e6650557ef136a165f1fe7df03a65/grpcio-1.82.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:254230d2cb773a87380858d0fc74653b4761bf2013c83fe1c5d77ed8f241fc9d", size = 6714591, upload-time = "2026-07-06T04:22:04.04Z" }, + { url = "https://files.pythonhosted.org/packages/29/6f/f0ec3f1a61a746dc7037a737ba4cbe60959645311ac8542bb1ad4e72ae8b/grpcio-1.82.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:16d4f215344057eb21038319b05a0c531306230c17b075fa8461a944581006dd", size = 7454986, upload-time = "2026-07-06T04:22:06.776Z" }, + { url = "https://files.pythonhosted.org/packages/b6/11/9d6ce94465d6a7f92c413430b3e77f0879b39427a217ffb3d23585df4bb3/grpcio-1.82.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a7c283b459151a2e84df32b28c31562ab3e7f624bccffce176c5608565b0212f", size = 6888623, upload-time = "2026-07-06T04:22:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/51/1a/b887adb7abb69081c4f24bc66c3612975e87caf5f7c6aa3916bb82d42e5f/grpcio-1.82.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:96a3be4d2a482ff004c036f6391f33553b36a7627138d1ca3ecc1e70d55d8af5", size = 7505067, upload-time = "2026-07-06T04:22:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/60/94/90f11c0f227ac653cff769e05c7f279da945e134c9351a1c37d475714686/grpcio-1.82.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bc481e6c7e6c8721797f2ab764092c843fd5cea06d4c7e9f4c3e3b7ecf331eb0", size = 8535382, upload-time = "2026-07-06T04:22:14.453Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3d/de8569386685213135dfbaab58e221add11858485524cf7d5987efc6d70b/grpcio-1.82.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:680f633bc788652222cb90a568ec374edcd91e1fe7715a7c248ece8e1032c2bd", size = 7910708, upload-time = "2026-07-06T04:22:17.994Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/3950b32369763818ae89ef76d52e84bd034d3a0ab46bf315669c913b32e2/grpcio-1.82.0-cp313-cp313-win32.whl", hash = "sha256:640b4715b9c206e5176e46601aa1422c47c779f642a869dfd27062e8fde13dfb", size = 4240351, upload-time = "2026-07-06T04:22:20.626Z" }, + { url = "https://files.pythonhosted.org/packages/a7/5c/c7b9a48efe973883f5707a311f87772a4c307a22ec80d6f3c851846a0a02/grpcio-1.82.0-cp313-cp313-win_amd64.whl", hash = "sha256:8523e81bd23f83e607aed28fbd8244b2be4aeb34e084fcef1bd1867709085c2a", size = 5000985, upload-time = "2026-07-06T04:22:23.365Z" }, + { url = "https://files.pythonhosted.org/packages/bd/6e/abc5efe11b73eefa0183531032eef2a53f33aeced6484b0d9da559c12ed6/grpcio-1.82.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:136969867c14fcc743fa39b12d7da133cae7da4f8e0e7767b6b8b7e75a8c24fd", size = 6146898, upload-time = "2026-07-06T04:22:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e4/d01ee88ceac221436dce1584ae1f046bd44d0dffc5a92e95aa34857c4516/grpcio-1.82.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:a7e246cd216662f513ae79ea3afadde6afe8a659b48a4170bae9a896e69ac254", size = 11954897, upload-time = "2026-07-06T04:22:29.114Z" }, + { url = "https://files.pythonhosted.org/packages/99/b9/2a35540ec08afc08859c35afd1a8a51e2585f39b12cf4eee68dcb1fa973d/grpcio-1.82.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:85c2a4e9e8a30d73d41e547a5101e57a1ed2093ec4b2daf6847c344adba00523", size = 6723102, upload-time = "2026-07-06T04:22:32.146Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ba/af71af59943b5d08879617a1d97495d45c53e43f3c6ef16f820c64e107f9/grpcio-1.82.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:a3a7802328e0d20e6ac458ff1974a8096cfaa3ca84dea0903235f641f1d703c1", size = 7454545, upload-time = "2026-07-06T04:22:34.776Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7e/258aae12b68910140725873657469ff166f8968bdb0674e12d6452b1812c/grpcio-1.82.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9331e6b91b26cffd63fb70a545f3c2cac8dc1b434b4436f43915c23e1e22f265", size = 6889585, upload-time = "2026-07-06T04:22:37.411Z" }, + { url = "https://files.pythonhosted.org/packages/5e/04/cfada8930306b9101cfb405b33ecb3b72abce51d725f900f167ffbc3146b/grpcio-1.82.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:670ab93a0059e707de53b9e715304b9f566b6defee80e5490cdd15820ce032f6", size = 7514162, upload-time = "2026-07-06T04:22:40.076Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/955f95989338857a6d73ee838b4d9e8198d50972a70ca83ec22322396f4c/grpcio-1.82.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:fbfe2e321bb30b84799677d6e8469896e487b494e9e4fbc9b8cc4425fe054d44", size = 8536163, upload-time = "2026-07-06T04:22:42.801Z" }, + { url = "https://files.pythonhosted.org/packages/06/62/826da0b0f01c7ba3f7ea0185968f551290deb28968d98b1edf604c895db8/grpcio-1.82.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2472b72a55da8570e089f1ca22f34a350d86c109be2755f6ee1cca5ebbdd9b54", size = 7912573, upload-time = "2026-07-06T04:22:46.187Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b4/d8824a66ac1ac7fbcdf63806f1a8725d95426654e859afa4606070c78740/grpcio-1.82.0-cp314-cp314-win32.whl", hash = "sha256:c9feb986151c2726789599b3672c1c9faa1d0824da921dc3f33a8cb1f93e2861", size = 4321824, upload-time = "2026-07-06T04:22:48.866Z" }, + { url = "https://files.pythonhosted.org/packages/da/08/02e581cc0bbb4c7b842f85e66a945b46d5bcc5927575c1086edfbc3360e7/grpcio-1.82.0-cp314-cp314-win_amd64.whl", hash = "sha256:b1eeb0480d86965263f8c8ae7ee5360c284d22176a196aab02fce7fba8d89dd8", size = 5141112, upload-time = "2026-07-06T04:22:51.443Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -2103,6 +2174,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "milvus-lite" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "faiss-cpu" }, + { name = "grpcio" }, + { name = "numpy" }, + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/9a/d80d260e6fe1246818a8ef782c374ba9c6ca46ca3b987c14eabe914ef805/milvus_lite-3.0.tar.gz", hash = "sha256:2c35d0d046b1faae3402cde1fb73d65f51ee8c6aba65f54de1dda46f7bb18b9b", size = 589749, upload-time = "2026-05-13T07:14:05.827Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/42/dee08ae3bfc1731572f193d00b248c9370b0b9dff12becb0ffd8b2ee8d56/milvus_lite-3.0-py3-none-any.whl", hash = "sha256:d9a094eab84bdaa4253da3721482282c939da1cce6f4e1759f947e8d3e53406e", size = 230490, upload-time = "2026-05-13T07:14:00.816Z" }, +] + [[package]] name = "mmh3" version = "5.2.1" @@ -2521,6 +2607,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/7a/7fe66f5f3682b1dd47d88cc4e11f1c6c0966b737de2d16671146e23c39a5/opentelemetry_semantic_conventions-0.63b1-py3-none-any.whl", hash = "sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682", size = 203713, upload-time = "2026-05-21T16:32:47.016Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -2530,6 +2669,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" }, + { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, +] + [[package]] name = "pathable" version = "0.6.0" @@ -2841,6 +3032,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/48/aa584cf3772e01231641c95dc1aa73327a7d986c562639d78d0013733acf/py_rust_stemmers-0.1.8-cp314-cp314-win_amd64.whl", hash = "sha256:931d13570962b093417e5443a9d1bd63d73fa239ebb81e5b1d346663571403e4", size = 209636, upload-time = "2026-05-22T11:00:18.662Z" }, ] +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, +] + [[package]] name = "pybars3" version = "0.9.7" @@ -3027,6 +3256,29 @@ version = "0.5.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/ce/af/409edba35fc597f1e386e3860303791ab5a28d6cc9a8aecbc567051b19a9/PyMeta3-0.5.1.tar.gz", hash = "sha256:18bda326d9a9bbf587bfc0ee0bc96864964d78b067288bcf55d4d98681d05bcb", size = 29566, upload-time = "2015-02-22T16:30:06.858Z" } +[[package]] +name = "pymilvus" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" }, +] + +[package.optional-dependencies] +milvus-lite = [ + { name = "milvus-lite", marker = "sys_platform != 'win32'" }, +] + [[package]] name = "pyperclip" version = "1.11.0" @@ -3696,6 +3948,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/21/3b/326ad4c03b5da89b5124c8890af66e8119c4d2e10abc0619e0d67d9f7c7f/sentry_sdk-2.65.0-py3-none-any.whl", hash = "sha256:3595169677a808e4d0e1ea6ffb89443459549c7a98392ed71c77c847182ab6bf", size = 503869, upload-time = "2026-07-13T11:33:17.71Z" }, ] +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" From 86605530f9ff5206c7ab9a139843465ac87701af Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Wed, 8 Jul 2026 10:04:09 +0000 Subject: [PATCH 2/3] fix: harden Milvus vector backend tests Signed-off-by: Cheney Zhang --- pyproject.toml | 2 +- .../repository/milvus_search_repository.py | 17 +- .../test_milvus_search_repository.py | 190 +++--------------- uv.lock | 2 +- 4 files changed, 42 insertions(+), 169 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7bafa7c8b..291aaa1bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ bm = "basic_memory.cli.main:app" [project.optional-dependencies] milvus = [ - "pymilvus[milvus-lite]>=2.6.0,<4.0.0", + "pymilvus[milvus-lite]>=3.0.0,<4.0.0", ] [build-system] diff --git a/src/basic_memory/repository/milvus_search_repository.py b/src/basic_memory/repository/milvus_search_repository.py index 61eb8fab5..58991e6d0 100644 --- a/src/basic_memory/repository/milvus_search_repository.py +++ b/src/basic_memory/repository/milvus_search_repository.py @@ -24,6 +24,8 @@ _COLLECTION_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _MILVUS_ID_MAX_LENGTH = 128 _MILVUS_TEXT_MAX_LENGTH = 65535 +_MILVUS_MARKER_COLUMNS = {"chunk_id", "project_id", "embedding_dims", "updated_at"} +_SQL_VECTOR_COLUMNS = {"embedding"} def _load_milvus_symbols() -> tuple[MilvusClientFactory, Any]: @@ -170,8 +172,9 @@ async def _ensure_sql_vector_metadata_tables(self, session: AsyncSession) -> Non embedding_columns = await self._table_columns( session, table_name="search_vector_embeddings", dialect_name=dialect_name ) - marker_columns = {"chunk_id", "project_id", "embedding_dims", "updated_at"} - marker_mismatch = bool(embedding_columns) and not marker_columns.issubset(embedding_columns) + marker_mismatch = bool(embedding_columns) and not self._is_milvus_marker_table( + embedding_columns + ) if marker_mismatch: # Trigger: switching from sqlite-vec/pgvector to Milvus leaves a vector # storage table under the marker table name. @@ -278,6 +281,11 @@ async def _ensure_sql_vector_metadata_tables(self, session: AsyncSession) -> Non ) ) + @staticmethod + def _is_milvus_marker_table(columns: set[str]) -> bool: + """Return whether search_vector_embeddings is the Milvus SQL marker table.""" + return _MILVUS_MARKER_COLUMNS.issubset(columns) and not (_SQL_VECTOR_COLUMNS & columns) + @staticmethod def _dialect_name(session: AsyncSession) -> str: if session.bind is None: @@ -415,6 +423,7 @@ async def _run_vector_query( anns_field="embedding", limit=candidate_limit, filter=f"project_id == {int(self.project_id)}", + search_params={"metric_type": "COSINE"}, output_fields=["chunk_id", "entity_id", "chunk_key", "chunk_text"], ) return self._milvus_search_rows(raw_results) @@ -650,8 +659,8 @@ async def drop_vector_tables(self) -> None: self._vector_tables_initialized = False def _distance_to_similarity(self, distance: float) -> float: - """Milvus COSINE search returns higher scores for better matches.""" - return max(0.0, min(1.0, distance)) + """Convert Milvus COSINE distance to Basic Memory's higher-is-better score.""" + return max(0.0, min(1.0, 1.0 - distance)) class SQLiteMilvusSearchRepository(MilvusVectorSearchMixin, SQLiteSearchRepository): diff --git a/tests/repository/test_milvus_search_repository.py b/tests/repository/test_milvus_search_repository.py index 46e33f335..20668500e 100644 --- a/tests/repository/test_milvus_search_repository.py +++ b/tests/repository/test_milvus_search_repository.py @@ -2,7 +2,6 @@ from __future__ import annotations -import math from datetime import datetime, timezone from typing import Any, cast @@ -50,147 +49,6 @@ def _vectorize(text: str) -> list[float]: return [0.0, 0.0, 0.0, 1.0] -class FakeDataType: - """Small stand-in for pymilvus.DataType.""" - - VARCHAR = "VARCHAR" - INT64 = "INT64" - FLOAT_VECTOR = "FLOAT_VECTOR" - - -class FakeSchema: - """Collect schema fields added by the repository.""" - - def __init__(self) -> None: - self.fields: list[dict[str, Any]] = [] - - def add_field(self, **kwargs: Any) -> None: - params: dict[str, Any] = {} - if "dim" in kwargs: - params["dim"] = kwargs["dim"] - if "max_length" in kwargs: - params["max_length"] = kwargs["max_length"] - field = { - "name": kwargs["field_name"], - "type": kwargs["datatype"], - "params": params, - } - if kwargs.get("is_primary"): - field["is_primary"] = True - self.fields.append(field) - - -class FakeIndexParams: - """Collect index definitions added by the repository.""" - - def __init__(self) -> None: - self.indexes: list[dict[str, Any]] = [] - - def add_index(self, **kwargs: Any) -> None: - self.indexes.append(dict(kwargs)) - - -class FakeMilvusClient: - """In-memory MilvusClient double with collection-level state.""" - - collections: dict[str, dict[str, Any]] = {} - instances: list["FakeMilvusClient"] = [] - - def __init__(self, **kwargs: Any) -> None: - self.kwargs = kwargs - self.instances.append(self) - - @classmethod - def reset(cls) -> None: - cls.collections = {} - cls.instances = [] - - def has_collection(self, *, collection_name: str) -> bool: - return collection_name in self.collections - - def create_schema(self, *, auto_id: bool, enable_dynamic_field: bool) -> FakeSchema: - assert auto_id is False - assert enable_dynamic_field is False - return FakeSchema() - - def prepare_index_params(self) -> FakeIndexParams: - return FakeIndexParams() - - def create_collection( - self, - *, - collection_name: str, - schema: FakeSchema, - index_params: FakeIndexParams, - ) -> None: - self.collections[collection_name] = { - "fields": list(schema.fields), - "indexes": list(index_params.indexes), - "records": {}, - } - - def describe_collection(self, *, collection_name: str) -> dict[str, Any]: - collection = self.collections[collection_name] - return {"fields": collection["fields"]} - - def upsert(self, *, collection_name: str, data: list[dict[str, Any]]) -> None: - records = self.collections[collection_name]["records"] - for record in data: - records[record["id"]] = dict(record) - - def search( - self, - *, - collection_name: str, - data: list[list[float]], - anns_field: str, - limit: int, - filter: str, - output_fields: list[str], - ) -> list[list[dict[str, Any]]]: - assert anns_field == "embedding" - assert output_fields == ["chunk_id", "entity_id", "chunk_key", "chunk_text"] - project_id = int(filter.split("==", 1)[1].strip()) - query = data[0] - hits = [] - for record in self.collections[collection_name]["records"].values(): - if int(record["project_id"]) != project_id: - continue - score = sum(a * b for a, b in zip(query, record["embedding"], strict=True)) - norm = math.sqrt(sum(value * value for value in record["embedding"])) or 1.0 - hits.append( - { - "id": record["id"], - "distance": score / norm, - "entity": {field: record[field] for field in output_fields}, - } - ) - hits.sort(key=lambda hit: hit["distance"], reverse=True) - return [hits[:limit]] - - def delete( - self, - *, - collection_name: str, - ids: list[str] | None = None, - filter: str | None = None, - ) -> None: - records = self.collections[collection_name]["records"] - if ids is not None: - for record_id in ids: - records.pop(record_id, None) - return - - assert filter is not None - filters = { - part.strip().split("==", 1)[0].strip(): int(part.strip().split("==", 1)[1]) - for part in filter.split("and") - } - for record_id, record in list(records.items()): - if all(int(record[field]) == expected for field, expected in filters.items()): - records.pop(record_id, None) - - def _milvus_config(tmp_path) -> BasicMemoryConfig: return BasicMemoryConfig( env="test", @@ -232,19 +90,23 @@ def _row( def _repo(session_maker, test_project, tmp_path) -> SQLiteMilvusSearchRepository: + pytest.importorskip("pymilvus") return SQLiteMilvusSearchRepository( session_maker, project_id=test_project.id, app_config=_milvus_config(tmp_path), embedding_provider=StubEmbeddingProvider(), - milvus_client_factory=FakeMilvusClient, - milvus_data_type=FakeDataType, ) -@pytest.fixture(autouse=True) -def _reset_fake_milvus() -> None: - FakeMilvusClient.reset() +async def _milvus_records(repo: SQLiteMilvusSearchRepository, project_id: int) -> list[dict]: + return await repo._milvus_call( + "query", + collection_name=repo._milvus_collection_name, + filter=f"project_id == {int(project_id)}", + output_fields=["id", "chunk_id", "entity_id", "chunk_key"], + limit=100, + ) @pytest.mark.asyncio @@ -281,19 +143,8 @@ async def test_milvus_backend_creates_collection_and_marker_rows( result = await repo.sync_entity_vectors_batch([101, 102]) assert result.entities_synced == 2 - assert FakeMilvusClient.instances[0].kwargs == { - "uri": str(tmp_path / "milvus.db"), - "token": "test-token", - } - collection = FakeMilvusClient.collections["basic_memory_vectors"] - assert collection["indexes"] == [ - { - "field_name": "embedding", - "index_type": "AUTOINDEX", - "metric_type": "COSINE", - } - ] - assert collection["records"] + records = await _milvus_records(repo, test_project.id) + assert {record["entity_id"] for record in records} == {101, 102} async with db.scoped_session(session_maker) as session: marker_count = await session.execute( @@ -304,7 +155,7 @@ async def test_milvus_backend_creates_collection_and_marker_rows( ), {"project_id": test_project.id}, ) - assert int(marker_count.scalar_one()) == len(collection["records"]) + assert int(marker_count.scalar_one()) == len(records) results = await repo.search( search_text="session token auth", @@ -338,11 +189,11 @@ async def test_milvus_backend_deletes_external_records_with_entity_chunks( ) ) await repo.sync_entity_vectors(201) - assert FakeMilvusClient.collections["basic_memory_vectors"]["records"] + assert await _milvus_records(repo, test_project.id) await repo.delete_entity_vector_rows(201) - assert FakeMilvusClient.collections["basic_memory_vectors"]["records"] == {} + assert await _milvus_records(repo, test_project.id) == [] async with db.scoped_session(session_maker) as session: chunk_count = await session.execute( text( @@ -358,6 +209,19 @@ async def test_milvus_backend_deletes_external_records_with_entity_chunks( assert int(marker_count.scalar_one()) == 0 +def test_milvus_marker_table_rejects_pgvector_table() -> None: + """A pgvector table is not a valid Milvus marker table even if marker columns match.""" + pgvector_columns = { + "chunk_id", + "project_id", + "embedding", + "embedding_dims", + "updated_at", + } + + assert not SQLiteMilvusSearchRepository._is_milvus_marker_table(pgvector_columns) + + def test_create_search_repository_selects_sqlite_milvus_backend(tmp_path): """Factory should keep SQL backend selection but swap semantic vector storage.""" config = _milvus_config(tmp_path) diff --git a/uv.lock b/uv.lock index 2977f0995..030d7b78f 100644 --- a/uv.lock +++ b/uv.lock @@ -387,7 +387,7 @@ requires-dist = [ { name = "pydantic", extras = ["email", "timezone"], specifier = ">=2.12.0" }, { name = "pydantic-settings", specifier = ">=2.6.1" }, { name = "pyjwt", specifier = ">=2.10.1" }, - { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=2.6.0,<4.0.0" }, + { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=3.0.0,<4.0.0" }, { name = "pyright", specifier = ">=1.1.390" }, { name = "pytest-aio", specifier = ">=1.9.0" }, { name = "pytest-asyncio", specifier = ">=1.2.0" }, From 218da162ce04ee6583498e2f9b15b2eae4c7dbcf Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Wed, 15 Jul 2026 00:50:33 +0000 Subject: [PATCH 3/3] style(core): format Milvus rebase changes Signed-off-by: Cheney Zhang --- src/basic_memory/services/project_service.py | 6 +----- .../services/test_project_service_embedding_status.py | 10 ++-------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/basic_memory/services/project_service.py b/src/basic_memory/services/project_service.py index 4495f41bf..3e4837c04 100644 --- a/src/basic_memory/services/project_service.py +++ b/src/basic_memory/services/project_service.py @@ -1188,11 +1188,7 @@ async def _vec_scalar(vec_sql) -> int: # Why: project info should degrade gracefully instead of crashing on stats queries. # Outcome: report vector tables as unavailable and point the user to install the # missing dependency before rebuilding embeddings. - if ( - is_postgres - or uses_milvus - or "no such module: vec0" not in str(exc).lower() - ): + if is_postgres or uses_milvus or "no such module: vec0" not in str(exc).lower(): raise return EmbeddingStatus( diff --git a/tests/services/test_project_service_embedding_status.py b/tests/services/test_project_service_embedding_status.py index ada9b8398..53f15dbec 100644 --- a/tests/services/test_project_service_embedding_status.py +++ b/tests/services/test_project_service_embedding_status.py @@ -307,10 +307,7 @@ async def test_embedding_status_milvus_sqlite_uses_plain_marker_table( pytest.skip("This regression covers the SQLite marker-table branch.") await _execute(project_service, text("DELETE FROM search_vector_chunks"), {}) - await _execute( - project_service, - text("DROP TABLE IF EXISTS search_vector_embeddings"), {} - ) + await _execute(project_service, text("DROP TABLE IF EXISTS search_vector_embeddings"), {}) await _execute( project_service, text( @@ -366,10 +363,7 @@ async def fail_if_sqlite_vec_path_is_used(query, params=None): ): status = await project_service.get_embedding_status(test_project.id) - await _execute( - project_service, - text("DROP TABLE IF EXISTS search_vector_embeddings"), {} - ) + await _execute(project_service, text("DROP TABLE IF EXISTS search_vector_embeddings"), {}) assert status.vector_tables_exist is True assert status.total_embeddings == 1