diff --git a/CLAUDE.md b/CLAUDE.md index 11dd52a..25959a4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,7 +107,7 @@ src/ ├── web/ # Dashboard server (server.ts) and page (dashboard.ts: do not edit) ├── lite/ # libscope/lite: createLite() preset of LibScope, createCodeChunker(), TreeSitterChunker, normalizeRawInput() ├── db/ # SQLite (schema.ts migrations, connection.ts, index-meta.ts embedding model of the index) -├── providers/ # Embedding providers (local/xenova, ollama, openai) +├── providers/ # Embedding providers (local, ollama, openai) ├── registry/ # Git-backed pack registries (config, git, sync, publish, search, resolve, checksum) ├── connectors/ # notion, slack, confluence, onenote, obsidian, docs; registry.ts (types, saved connections, sync) ├── config-schema.ts # The zod config schema: keys, defaults, env var names, docs table (getConfigKeyTable) diff --git a/Dockerfile b/Dockerfile index 9a667ed..da9f71f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ -# Debian (glibc) base: the local embedding model (onnxruntime-node) does not run on Alpine (musl). -# sharp and onnxruntime-node ship prebuilt binaries, so only better-sqlite3 is rebuilt. Skipping the -# onnxruntime-node install script also skips its optional CUDA download (about 270 MB). +# better-sqlite3 is the only native addon; the production stage rebuilds it. # Build stage FROM node:22-bookworm-slim AS builder diff --git a/README.md b/README.md index 6089f3c..12049a9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ libscope ask "How do I configure OAuth2?" libscope doctor ``` -The database is created on first use. The first command that embeds text downloads the local model [all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2) (about 80 MB, first run only). +The database is created on first use. The first command that embeds text downloads the local model [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) (about 90 MB, first run only). ### MCP (Claude, Cursor, VS Code, ...) diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 234cb4a..333895b 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -151,7 +151,7 @@ src/ │ ├── embedding.ts # EmbeddingProvider interface │ ├── index.ts # createEmbeddingProvider() from config │ ├── dimensions.ts # vector sizes of known models -│ ├── local.ts # @huggingface/transformers (all-MiniLM-L6-v2) +│ ├── local.ts # all-MiniLM-L6-v2 on onnxruntime-web (WebAssembly) │ ├── ollama.ts # Ollama HTTP API │ └── openai.ts # OpenAI embeddings API ├── registry/ # git pack registries: config, git, sync, search, resolve, publish, checksum diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index daa9026..8930704 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -67,11 +67,11 @@ Embeddings turn text into vectors for semantic search. LibScope supports three p | Provider | Default | Requires | Notes | | -------- | ------- | ---------------------- | --------------------------------------------- | -| `local` | ✅ | Nothing | all-MiniLM-L6-v2, ~80MB download on first use | +| `local` | ✅ | Nothing | all-MiniLM-L6-v2, ~90MB download on first use | | `ollama` | | Ollama running locally | Uses nomic-embed-text by default | | `openai` | | API key | Uses text-embedding-3-small by default | -The local provider works out of the box — no API keys, no external services. It runs the model in-process using `@huggingface/transformers` and caches it in `~/.libscope/models`. +The local provider works out of the box — no API keys, no external services. It runs the model in-process on the ONNX Runtime WebAssembly build (no native addon) and caches it in `~/.libscope/models`. ```bash # Switch to Ollama diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9b02d4b..7c55b43 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -18,7 +18,7 @@ Requires Node.js 20 or later. libscope doctor ``` -This shows the config files, the active workspace and its database (`~/.libscope/workspaces/default/libscope.db`), the embedding model, and which LLM `ask` will use. Each warning comes with the command that fixes it. You do not have to create anything first: the database is created when you add the first document (or run `libscope doctor --fix`). The first command that embeds text downloads the local embedding model (~80MB, once) and prints one line while it does. +This shows the config files, the active workspace and its database (`~/.libscope/workspaces/default/libscope.db`), the embedding model, and which LLM `ask` will use. Each warning comes with the command that fixes it. You do not have to create anything first: the database is created when you add the first document (or run `libscope doctor --fix`). The first command that embeds text downloads the local embedding model (~90MB, once) and prints one line while it does. ## Index Some Documents diff --git a/docs/guide/lite.md b/docs/guide/lite.md index cfd61a8..161b5e7 100644 --- a/docs/guide/lite.md +++ b/docs/guide/lite.md @@ -79,7 +79,7 @@ createLite(options: LiteOptions): LibScope ### Embedding providers -The default is the local `all-MiniLM-L6-v2` model (about 80 MB, downloaded on first use). To use OpenAI or Ollama, set it in `config`: +The default is the local `all-MiniLM-L6-v2` model (about 90 MB, downloaded on first use). To use OpenAI or Ollama, set it in `config`: ```ts const scope = createLite({ diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index db01657..b3591d8 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -24,7 +24,7 @@ If you're on an unsupported platform, LibScope will fall back to keyword-only se **Symptom:** First run hangs at "Downloading embedding model ... (first run only)..." or fails with a network error. -**Cause:** The local embedding model (~80MB) downloads from Hugging Face on first use. +**Cause:** The local embedding model (~90MB) downloads from Hugging Face on first use. **Fix:** diff --git a/docs/migration-v2.md b/docs/migration-v2.md index b12fac0..39bcde5 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -1,7 +1,8 @@ # Migrating to LibScope 2.0 -LibScope 2.0 removes old names without aliases. This page lists every removed or renamed name and what to use instead. Your indexed data is not changed by the upgrade. +LibScope 2.0 removes old names without aliases. This page lists every removed or renamed name and what to use instead. +- [Rebuild the vector index (2.0.1)](#rebuild-the-vector-index-201): required once - [Configuration](#configuration): config keys, environment variables, API keys - [Library API](#library-api): package root exports, the `LibScope` class, `libscope/lite` - [MCP](#mcp): server startup, renamed, merged and removed tools @@ -14,6 +15,20 @@ LibScope 2.0 removes old names without aliases. This page lists every removed or The references list the 2.0 names: [CLI](reference/cli.md), [MCP tools](reference/mcp-tools.md), [REST API](reference/rest-api.md) and [configuration](reference/configuration.md). +## Rebuild the vector index (2.0.1) + +LibScope 2.0.1 changes the vectors of the local embedding provider, so vector indexes built by earlier versions do not match it. Upgrade to 2.0.1 or later, then rebuild the vector index once: + +```bash +libscope admin reindex --rebuild +``` + +Until you do, commands that use the vector index stop with an error that names the old and the new model. Documents, chunks and keyword search are not changed. + +The local provider now runs the full-precision all-MiniLM-L6-v2 model on the ONNX Runtime WebAssembly build, and truncates input at 256 tokens as sentence-transformers does. Its vectors match the original model. Installing LibScope no longer runs a native install script for the model runtime, and no longer downloads CUDA binaries on Linux. The first command that embeds text downloads the model again (about 90 MB, once). + +Vector indexes built by LibScope 1.x with any provider must also be rebuilt, because 1.x did not record which model built them. + ## Configuration ### Config keys @@ -50,7 +65,7 @@ These variables do not change: `LIBSCOPE_EMBEDDING_PROVIDER`, `LIBSCOPE_LLM_PROV | `llm.provider` had no default; `ask` failed until it was set. | `llm.provider` defaults to `auto`: passthrough under MCP, else OpenAI if an OpenAI key is set, else Anthropic if an Anthropic key is set, else Ollama if `llm.url` is set or the embedding provider is `ollama`. | | An invalid value in a config file was used as is. | An invalid value is ignored with a warning, and the default (or lower layer) applies. | | `LIBSCOPE_ALLOW_PRIVATE_URLS=false` did not override `true` in a file. | `LIBSCOPE_INDEXING_ALLOW_PRIVATE_URLS=false` (or `0`) sets `false`. | -| The local embedding model ran on `@xenova/transformers` and was cached in `node_modules/@xenova/transformers/.cache`. | The local embedding model runs on `@huggingface/transformers` and is cached in `~/.libscope/models` (`/data/.libscope/models` in the Docker image). The first command that embeds text after the upgrade downloads the model again, once. The model (all-MiniLM-L6-v2, quantized) and its 384-dimension vectors do not change, so you do not have to reindex. | +| The local embedding model ran on `@xenova/transformers` and was cached in `node_modules/@xenova/transformers/.cache`. | The local embedding model is cached in `~/.libscope/models` (`/data/.libscope/models` in the Docker image). The first command that embeds text after the upgrade downloads the model again, once. Its vectors change in 2.0.1: [rebuild the vector index](#rebuild-the-vector-index-201). | ### Library (SDK) code diff --git a/docs/reference/cli.md b/docs/reference/cli.md index b21c7c3..359c582 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -45,7 +45,7 @@ Every result shows document IDs (and chunk IDs for search hits), so you can pass **Errors.** A failed command prints `✗ ` and a next step on stderr, and exits with code 1. For example, an unknown document ID suggests `libscope docs list`, and a configuration problem suggests `libscope doctor`. Add `--verbose` to see the stack trace. -**Progress.** Long operations (directory imports, crawls, reindexing, pack installs) show a progress line on stderr when it is a terminal. The first command that needs the local embedding model prints one line while the model downloads (about 80 MB, first run only). +**Progress.** Long operations (directory imports, crawls, reindexing, pack installs) show a progress line on stderr when it is a terminal. The first command that needs the local embedding model prints one line while the model downloads (about 90 MB, first run only). ### Document filters diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 70b22a6..52121da 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -25,7 +25,7 @@ Every key `section.field` has the environment variable `LIBSCOPE_
_=18" - } - }, "node_modules/@huggingface/tokenizers": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.2.0.tgz", "integrity": "sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==", "license": "Apache-2.0" }, - "node_modules/@huggingface/transformers": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.3.1.tgz", - "integrity": "sha512-5+voSeUCFK6PCKlY15dPhNOKq3/zOTxDN7oC+Wy9dFb1QZ0TSQomZRRy7URRK8YRuUoE9fGMQTTPC5d/pWBJhA==", - "license": "Apache-2.0", - "dependencies": { - "@huggingface/jinja": "^0.5.10", - "@huggingface/tokenizers": "^0.2.0", - "onnxruntime-node": "1.30.0", - "onnxruntime-web": "1.31.0-dev.20260914-8d85527a0", - "sharp": "^0.35.4" - } - }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -1181,506 +1150,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz", - "integrity": "sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.4" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz", - "integrity": "sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.4" - } - }, - "node_modules/@img/sharp-freebsd-wasm32": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz", - "integrity": "sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==", - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "dependencies": { - "@img/sharp-wasm32": "0.35.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz", - "integrity": "sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz", - "integrity": "sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz", - "integrity": "sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==", - "cpu": [ - "arm" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz", - "integrity": "sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz", - "integrity": "sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==", - "cpu": [ - "ppc64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz", - "integrity": "sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==", - "cpu": [ - "riscv64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz", - "integrity": "sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==", - "cpu": [ - "s390x" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz", - "integrity": "sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz", - "integrity": "sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz", - "integrity": "sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz", - "integrity": "sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==", - "cpu": [ - "arm" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.4" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz", - "integrity": "sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.4" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz", - "integrity": "sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==", - "cpu": [ - "ppc64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.4" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz", - "integrity": "sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==", - "cpu": [ - "riscv64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.4" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz", - "integrity": "sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==", - "cpu": [ - "s390x" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.4" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz", - "integrity": "sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.4" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz", - "integrity": "sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.4" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz", - "integrity": "sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.4" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", - "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.11.3" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-webcontainers-wasm32": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.5.tgz", - "integrity": "sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz", - "integrity": "sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz", - "integrity": "sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz", - "integrity": "sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -4241,40 +3710,6 @@ "dev": true, "license": "MIT" }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4555,6 +3990,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -5188,37 +4624,6 @@ "node": ">=10.13.0" } }, - "node_modules/global-agent": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", - "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", - "license": "BSD-3-Clause", - "dependencies": { - "globalthis": "^1.0.2", - "matcher": "^4.0.0", - "semver": "^7.3.5", - "serialize-error": "^8.1.0" - }, - "engines": { - "node": ">=10.0" - } - }, - "node_modules/globalthis": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", - "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "license": "MIT", - "dependencies": { - "define-properties": "^1.2.1", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5247,18 +4652,6 @@ "node": ">=8" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -6179,21 +5572,6 @@ "dev": true, "license": "MIT" }, - "node_modules/matcher": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", - "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -6582,15 +5960,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/obug": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", @@ -6669,52 +6038,20 @@ "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", "license": "MIT" }, - "node_modules/onnxruntime-node": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz", - "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==", - "hasInstallScript": true, - "license": "MIT", - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "adm-zip": "^0.6.0", - "global-agent": "^4.1.3", - "onnxruntime-common": "1.30.0" - } - }, - "node_modules/onnxruntime-node/node_modules/adm-zip": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", - "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", - "license": "MIT", - "engines": { - "node": ">=14.0" - } - }, "node_modules/onnxruntime-web": { - "version": "1.31.0-dev.20260914-8d85527a0", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.31.0-dev.20260914-8d85527a0.tgz", - "integrity": "sha512-Iy7rtadoBgxS/LLvDr3QW38DB1PNXRnr0GJMcL0TAt7c9qjgVQl83UlGCVyeAnK2InpmW8Uc3PL8XIuqtDeF6g==", + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.30.0.tgz", + "integrity": "sha512-q0y+JrrtukXSzsBWEMccVfqX25LRmosXHF+CaRJmg8pZClzcV7svNc4rKY3jL02Vb7QmRMDs1SigqR4CXAfKYQ==", "license": "MIT", "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", - "onnxruntime-common": "1.31.0-dev.20260911-2a43ec07e", + "onnxruntime-common": "1.30.0", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, - "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.31.0-dev.20260911-2a43ec07e", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.31.0-dev.20260911-2a43ec07e.tgz", - "integrity": "sha512-gBuF6U32YErKIAt+yD7DeGBpjRxhJ1uJwako6ygjokSZULDU6Pd+KWComX8Tumk1hYVHRxjPj7mdMnD0ryAPOw==", - "license": "MIT" - }, "node_modules/openai": { "version": "6.49.0", "resolved": "https://registry.npmjs.org/openai/-/openai-6.49.0.tgz", @@ -7517,21 +6854,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/serialize-error": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", - "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", - "license": "MIT", - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/serve-static": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", @@ -7564,55 +6886,6 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, - "node_modules/sharp": { - "version": "0.35.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", - "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", - "license": "Apache-2.0", - "dependencies": { - "@img/colour": "^1.1.0", - "detect-libc": "^2.1.2", - "semver": "^7.8.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.5", - "@img/sharp-darwin-x64": "0.35.5", - "@img/sharp-freebsd-wasm32": "0.35.5", - "@img/sharp-libvips-darwin-arm64": "1.3.4", - "@img/sharp-libvips-darwin-x64": "1.3.4", - "@img/sharp-libvips-linux-arm": "1.3.4", - "@img/sharp-libvips-linux-arm64": "1.3.4", - "@img/sharp-libvips-linux-ppc64": "1.3.4", - "@img/sharp-libvips-linux-riscv64": "1.3.4", - "@img/sharp-libvips-linux-s390x": "1.3.4", - "@img/sharp-libvips-linux-x64": "1.3.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.4", - "@img/sharp-libvips-linuxmusl-x64": "1.3.4", - "@img/sharp-linux-arm": "0.35.5", - "@img/sharp-linux-arm64": "0.35.5", - "@img/sharp-linux-ppc64": "0.35.5", - "@img/sharp-linux-riscv64": "0.35.5", - "@img/sharp-linux-s390x": "0.35.5", - "@img/sharp-linux-x64": "0.35.5", - "@img/sharp-linuxmusl-arm64": "0.35.5", - "@img/sharp-linuxmusl-x64": "0.35.5", - "@img/sharp-webcontainers-wasm32": "0.35.5", - "@img/sharp-win32-arm64": "0.35.5", - "@img/sharp-win32-ia32": "0.35.5", - "@img/sharp-win32-x64": "0.35.5" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -8218,13 +7491,6 @@ "typescript": ">=4.8.4" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true - }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -8250,18 +7516,6 @@ "node": ">= 0.8.0" } }, - "node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/type-is": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", diff --git a/package.json b/package.json index ef8e8e2..b31b9dd 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ }, "dependencies": { "@anthropic-ai/sdk": "^0.131.0", - "@huggingface/transformers": "^4.3.1", + "@huggingface/tokenizers": "^0.2.0", "@modelcontextprotocol/sdk": "^1.0.0", "better-sqlite3": "^12.11.1", "commander": "^14.0.3", @@ -69,6 +69,7 @@ "js-yaml": "^4.1.1", "node-cron": "^4.6.0", "node-html-markdown": "^2.0.0", + "onnxruntime-web": "^1.30.0", "openai": "^6.49.0", "pino": "^10.3.1", "pizzip": "^3.2.0", diff --git a/src/config-schema.ts b/src/config-schema.ts index e5a5bad..2972f9b 100644 --- a/src/config-schema.ts +++ b/src/config-schema.ts @@ -47,7 +47,7 @@ export const ConfigSchema = z.object({ nonEmpty() .optional() .describe( - "Embedding model. Ignored by the local provider (always Xenova/all-MiniLM-L6-v2).", + "Embedding model. Ignored by the local provider (always sentence-transformers/all-MiniLM-L6-v2).", ), { defaultText: "ollama: nomic-embed-text; openai: text-embedding-3-small" }, ), diff --git a/src/config.ts b/src/config.ts index 8455766..f3b08df 100644 --- a/src/config.ts +++ b/src/config.ts @@ -389,7 +389,7 @@ function validateProviderConfig(config: LibScopeConfig, warnings: string[]): voi } if (config.embedding.provider === "local" && config.embedding.model) { warnings.push( - `embedding.model "${config.embedding.model}" is ignored by the local provider (it always uses Xenova/all-MiniLM-L6-v2).`, + `embedding.model "${config.embedding.model}" is ignored by the local provider (it always uses sentence-transformers/all-MiniLM-L6-v2).`, ); } if (config.llm?.provider === "openai" && !openaiKey) { diff --git a/src/db/schema.ts b/src/db/schema.ts index 69965de..b33ea43 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -12,7 +12,10 @@ import { writeEmbeddingIdentity, } from "./index-meta.js"; -const SCHEMA_VERSION = 19; +const SCHEMA_VERSION = 20; + +/** Model recorded by migration 20 for a vector index whose model is unknown. */ +export const PRE_2_0_1_MODEL = "unknown (built before libscope 2.0.1)"; /** Copy every chunk into chunks_fts, keyed by the chunk's rowid. */ const FTS_BACKFILL_SQL = ` @@ -347,6 +350,18 @@ const MIGRATIONS: Record = { INSERT INTO schema_version (version) VALUES (19); `, + // libscope 1.x recorded no embedding model for its vectors, so a changed model was not + // detected. Record a placeholder so that opening such a vector index asks for + // `libscope admin reindex --rebuild`. (2.0.0 indexes of the local provider are caught + // because the recorded model, Xenova/all-MiniLM-L6-v2, no longer matches.) + 20: ` + INSERT INTO index_meta (key, value) + SELECT 'embedding_model', '${PRE_2_0_1_MODEL}' + WHERE EXISTS (SELECT 1 FROM sqlite_master WHERE name = 'chunk_embeddings') + AND NOT EXISTS (SELECT 1 FROM index_meta WHERE key = 'embedding_model'); + + INSERT INTO schema_version (version) VALUES (20); + `, }; /** diff --git a/src/providers/local.ts b/src/providers/local.ts index a6163c6..a4fe05d 100644 --- a/src/providers/local.ts +++ b/src/providers/local.ts @@ -1,28 +1,32 @@ import { existsSync } from "node:fs"; -import { homedir } from "node:os"; -import { join } from "node:path"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { availableParallelism, homedir } from "node:os"; +import { dirname, join } from "node:path"; +import type { InferenceSession, Tensor } from "onnxruntime-web"; import { EmbeddingError } from "../errors.js"; import type { EmbeddingProvider } from "./embedding.js"; import { getLogger } from "../logger.js"; -import { mapSequential } from "../utils/async.js"; +import { forEachSequential, mapSequential } from "../utils/async.js"; +import { withRetry } from "../utils/retry.js"; -/** Minimal typed interface for the @huggingface/transformers feature-extraction pipeline output. */ -interface TransformersOutput { - data: Float32Array; -} +/** Hugging Face repository and pinned revision of the ONNX export of all-MiniLM-L6-v2. */ +const MODEL_REPO = "Xenova/all-MiniLM-L6-v2"; +const MODEL_REVISION = "751bff37182d3f1213fa05d7196b954e230abad9"; +const TOKENIZER_FILE = "tokenizer.json"; +const TOKENIZER_CONFIG_FILE = "tokenizer_config.json"; +// Full precision: the quantized export's vectors drift from the original model's. +const ONNX_FILE = "onnx/model.onnx"; +const MODEL_FILES = [TOKENIZER_FILE, TOKENIZER_CONFIG_FILE, ONNX_FILE]; -/** Minimal typed interface for the @huggingface/transformers feature-extraction pipeline function. */ -type FeatureExtractionPipeline = ( - input: string, - options: { pooling: string; normalize: boolean }, -) => Promise; +/** The model's max_seq_length in sentence-transformers. Longer input is truncated. */ +export const MAX_TOKENS = 256; -/** Download events from @huggingface/transformers while the model is fetched (first use only). */ +/** Download events while the model files are fetched (first use only). */ export interface ModelDownloadEvent { - /** "initiate" | "download" | "progress" | "done" per file, "progress_total" for all files, then "ready". */ - status: string; + /** "progress" while a file downloads, then "ready" once all files are cached. */ + status: "progress" | "ready"; file?: string | undefined; - /** Percent of `file` downloaded (status "progress"), or of all files (status "progress_total"). */ + /** Percent of `file` downloaded (status "progress"). */ progress?: number | undefined; } @@ -32,65 +36,160 @@ export function getModelCacheDir(): string { } /** - * Local embedding provider using @huggingface/transformers (all-MiniLM-L6-v2). - * Downloads the model on first use (~80MB). Runs entirely in-process. + * Truncate token ids that start with [CLS] and end with [SEP] to MAX_TOKENS, + * keeping the final [SEP] as sentence-transformers does. + */ +export function truncateTokenIds(ids: number[]): number[] { + if (ids.length <= MAX_TOKENS) return ids; + return [...ids.slice(0, MAX_TOKENS - 1), ...ids.slice(-1)]; +} + +/** Mean of the token vectors (shape [tokens, dimensions], row-major), scaled to unit length. */ +export function meanPoolNormalized( + hidden: Float32Array, + tokens: number, + dimensions: number, +): number[] { + const sum = new Array(dimensions).fill(0); + for (let t = 0; t < tokens; t++) { + for (let d = 0; d < dimensions; d++) { + sum[d]! += hidden[t * dimensions + d]!; + } + } + const norm = Math.sqrt(sum.reduce((acc, x) => acc + x * x, 0)); + return sum.map((x) => x / norm); +} + +/** Download one model file into the cache, through a temporary file. */ +async function downloadModelFile( + file: string, + dest: string, + onProgress: ((event: ModelDownloadEvent) => void) | undefined, +): Promise { + const url = `https://huggingface.co/${MODEL_REPO}/resolve/${MODEL_REVISION}/${file}`; + const { body, total } = await withRetry(async () => { + const res = await fetch(url); + if (!res.ok || !res.body) { + throw new EmbeddingError(`Download of ${url} failed: HTTP ${res.status}`); + } + return { + body: res.body as AsyncIterable, + total: Number(res.headers.get("content-length")), + }; + }); + const parts: Uint8Array[] = []; + let received = 0; + for await (const part of body) { + parts.push(part); + received += part.length; + if (onProgress && total > 0) { + onProgress({ status: "progress", file, progress: (received / total) * 100 }); + } + } + await mkdir(dirname(dest), { recursive: true }); + const partial = `${dest}.partial`; + await writeFile(partial, Buffer.concat(parts)); + await rename(partial, dest); +} + +/** + * The part of the @huggingface/tokenizers Tokenizer used here. The package's own type + * declarations use extensionless imports, which do not resolve under NodeNext. + */ +interface Tokenizer { + encode(text: string): { ids: number[] }; +} + +interface TokenizersModule { + Tokenizer: new (tokenizerJson: object, tokenizerConfig: object) => Tokenizer; +} + +/** Tokenizer and inference session of the loaded model. */ +interface LoadedModel { + tokenizer: Tokenizer; + session: InferenceSession; + Tensor: typeof Tensor; +} + +/** + * Local embedding provider: all-MiniLM-L6-v2 run in-process on the ONNX Runtime + * WebAssembly build, with truncation and mean pooling as in sentence-transformers. + * Downloads the model on first use (~90MB). */ export class LocalEmbeddingProvider implements EmbeddingProvider { readonly name = "local"; - readonly model = "Xenova/all-MiniLM-L6-v2"; + readonly model = "sentence-transformers/all-MiniLM-L6-v2"; readonly dimensions = 384; - private pipeline: FeatureExtractionPipeline | null = null; - private initPromise: Promise | null = null; + private loading: Promise | null = null; /** Called while the model is downloaded (not when it is already cached), e.g. to show progress. */ onDownloadProgress: ((event: ModelDownloadEvent) => void) | undefined; - private async ensureInitialized(): Promise { - this.initPromise ??= this.doInitialize(); - await this.initPromise; + /** The loaded model, loading it on first call. A failed load is retried on the next call. */ + private load(): Promise { + this.loading ??= this.loadModel().catch((err: unknown) => { + this.loading = null; + throw new EmbeddingError("Failed to load local embedding model", err); + }); + return this.loading; } - private async doInitialize(): Promise { + private async loadModel(): Promise { const log = getLogger(); log.info("Loading local embedding model (all-MiniLM-L6-v2)..."); - try { - // Dynamic import to avoid loading transformers until needed - const { pipeline, env } = await import("@huggingface/transformers"); - // The library default is a folder inside its own package, which is not writable for a - // global install or a non-root container user; a failed cache write fails the load. - const cacheDir = getModelCacheDir(); - env.cacheDir = cacheDir; - const cached = existsSync(join(cacheDir, this.model)); - const onDownload = cached ? undefined : this.onDownloadProgress; - // dtype "q8" loads onnx/model_quantized.onnx, the model that @xenova/transformers 2.x - // used by default, so vectors stay compatible with existing indexes. - // Cast to the typed interface; the library's pipeline output types are broader than needed. - this.pipeline = (await pipeline("feature-extraction", this.model, { - dtype: "q8", - ...(onDownload ? { progress_callback: onDownload } : {}), - })) as unknown as FeatureExtractionPipeline; - log.info("Local embedding model loaded successfully"); - } catch (err) { - this.initPromise = null; - throw new EmbeddingError("Failed to load local embedding model", err); - } + const modelDir = join(getModelCacheDir(), MODEL_REPO); + const missing = MODEL_FILES.filter((file) => !existsSync(join(modelDir, file))); + // One file at a time, so that download progress reports one file at a time. + await forEachSequential(missing, (file) => + downloadModelFile(file, join(modelDir, file), this.onDownloadProgress), + ); + if (missing.length > 0) this.onDownloadProgress?.({ status: "ready" }); + + // Dynamic imports so that commands without embeddings do not load the runtime. + const [{ Tokenizer }, ort] = await Promise.all([ + import("@huggingface/tokenizers") as Promise as Promise, + import("onnxruntime-web"), + ]); + const [tokenizerJson, tokenizerConfig, onnx] = await Promise.all([ + readFile(join(modelDir, TOKENIZER_FILE), "utf8"), + readFile(join(modelDir, TOKENIZER_CONFIG_FILE), "utf8"), + readFile(join(modelDir, ONNX_FILE)), + ]); + // The runtime's default is min(4, cores / 2) threads. + ort.env.wasm.numThreads = availableParallelism(); + const loaded: LoadedModel = { + tokenizer: new Tokenizer( + JSON.parse(tokenizerJson) as object, + JSON.parse(tokenizerConfig) as object, + ), + session: await ort.InferenceSession.create(onnx), + Tensor: ort.Tensor, + }; + log.info("Local embedding model loaded successfully"); + return loaded; } async embed(text: string): Promise { if (!text.trim()) { throw new EmbeddingError("Input text must not be empty"); } - await this.ensureInitialized(); + const { tokenizer, session, Tensor } = await this.load(); try { - const output = await this.pipeline!(text, { pooling: "mean", normalize: true }); - const embedding = Array.from(output.data); - if (embedding.length !== this.dimensions) { + const ids = truncateTokenIds(tokenizer.encode(text).ids); + const shape = [1, ids.length]; + const output = await session.run({ + input_ids: new Tensor("int64", BigInt64Array.from(ids, BigInt), shape), + attention_mask: new Tensor("int64", new BigInt64Array(ids.length).fill(1n), shape), + token_type_ids: new Tensor("int64", new BigInt64Array(ids.length), shape), + }); + const hidden = output["last_hidden_state"]!.data as Float32Array; + if (hidden.length !== ids.length * this.dimensions) { throw new EmbeddingError( - `Expected embedding dimension ${this.dimensions}, got ${embedding.length}`, + `Expected embedding dimension ${this.dimensions}, got ${hidden.length / ids.length}`, ); } - return embedding; + return meanPoolNormalized(hidden, ids.length, this.dimensions); } catch (err) { throw new EmbeddingError(`Failed to generate embedding: ${String(err)}`, err); } diff --git a/tests/integration/retrieval-quality.test.ts b/tests/integration/retrieval-quality.test.ts index 9ebc88c..9cf23ab 100644 --- a/tests/integration/retrieval-quality.test.ts +++ b/tests/integration/retrieval-quality.test.ts @@ -21,7 +21,7 @@ * 4. Library-filter precision — searching with library= must return ONLY * docs from that library; zero cross-library leakage allowed. * - * 5. Neural model suite (conditional) — if @huggingface/transformers + + * 5. Neural model suite (conditional) — if onnxruntime-web + * all-MiniLM-L6-v2 is available locally, the same 15 queries run with * real embeddings under higher thresholds (MRR ≥ 0.82), plus 5 pure- * paraphrase queries that TF-IDF cannot handle but a semantic model should. @@ -513,7 +513,7 @@ function loadVec(db: Database.Database): void { function isNeuralModelAvailable(): Promise { try { const require = createRequire(import.meta.url); - require.resolve("@huggingface/transformers"); + require.resolve("onnxruntime-web"); // Attempt a quick model resolution without actually loading — just check the package exists return Promise.resolve(true); } catch { @@ -757,7 +757,7 @@ describe.runIf(isVecAvailable())( neuralAvailable = await isNeuralModelAvailable(); if (!neuralAvailable) return; - // Dynamic import to avoid loading transformers during TF-IDF suite + // Dynamic import to avoid loading the model runtime during TF-IDF suite const { LocalEmbeddingProvider } = await import("../../src/providers/local.js"); const provider = new LocalEmbeddingProvider(); diff --git a/tests/unit/index-meta.test.ts b/tests/unit/index-meta.test.ts index aa36085..6dc9510 100644 --- a/tests/unit/index-meta.test.ts +++ b/tests/unit/index-meta.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import type Database from "better-sqlite3"; import DatabaseConstructor from "better-sqlite3"; import { createDatabase } from "../../src/db/connection.js"; -import { runMigrations, createVectorTable } from "../../src/db/schema.js"; +import { PRE_2_0_1_MODEL, runMigrations, createVectorTable } from "../../src/db/schema.js"; import { getVectorTableDimensions, readEmbeddingIdentity, @@ -12,6 +12,7 @@ import { searchDocuments } from "../../src/core/search.js"; import { ConfigError, DatabaseError } from "../../src/errors.js"; import { initLogger } from "../../src/logger.js"; import type { EmbeddingProvider } from "../../src/providers/embedding.js"; +import { LocalEmbeddingProvider } from "../../src/providers/local.js"; /** Provider stub that returns constant vectors of a fixed size. */ class FixedProvider implements EmbeddingProvider { @@ -79,7 +80,7 @@ describe.skipIf(!vecAvailable)("vector index metadata (sqlite-vec)", () => { }); }); - it("populates metadata for a vector table built before schema v18", () => { + it("asks for a rebuild of a vector table built before schema v18 (libscope 1.x)", () => { runMigrations(db, 17); db.exec( "CREATE VIRTUAL TABLE chunk_embeddings USING vec0(chunk_id TEXT PRIMARY KEY, embedding float[4])", @@ -87,15 +88,25 @@ describe.skipIf(!vecAvailable)("vector index metadata (sqlite-vec)", () => { runMigrations(db); expect(readEmbeddingIdentity(db)).toEqual({ provider: undefined, - model: undefined, + model: PRE_2_0_1_MODEL, dimensions: undefined, }); createVectorTable(db, 4); expect(readEmbeddingIdentity(db).dimensions).toBe(4); - createVectorTable(db, new FixedProvider("mock", "m1", 4)); - expect(readEmbeddingIdentity(db)).toEqual({ provider: "mock", model: "m1", dimensions: 4 }); + expect(() => createVectorTable(db, new FixedProvider("mock", "m1", 4))).toThrow( + /libscope admin reindex --rebuild/, + ); + }); + + it("asks for a rebuild of a local-provider index built by 2.0.0", () => { + runMigrations(db); + createVectorTable(db, new FixedProvider("local", "Xenova/all-MiniLM-L6-v2", 384)); + + expect(() => createVectorTable(db, new LocalEmbeddingProvider())).toThrow( + /libscope admin reindex --rebuild/, + ); }); it("rejects a provider whose vector size differs from the index", () => { diff --git a/tests/unit/local-embedding.test.ts b/tests/unit/local-embedding.test.ts new file mode 100644 index 0000000..695a8ca --- /dev/null +++ b/tests/unit/local-embedding.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { + LocalEmbeddingProvider, + MAX_TOKENS, + meanPoolNormalized, + truncateTokenIds, +} from "../../src/providers/local.js"; +import { initLogger } from "../../src/logger.js"; + +const CLS = 101; +const SEP = 102; + +describe("truncateTokenIds", () => { + it("keeps input of at most MAX_TOKENS ids unchanged", () => { + const ids = [CLS, ...Array.from({ length: MAX_TOKENS - 2 }, (_, i) => 1000 + i), SEP]; + expect(truncateTokenIds(ids)).toEqual(ids); + }); + + it("cuts longer input to MAX_TOKENS ids and keeps the final [SEP]", () => { + const body = Array.from({ length: 600 }, (_, i) => 1000 + i); + const truncated = truncateTokenIds([CLS, ...body, SEP]); + expect(truncated).toHaveLength(MAX_TOKENS); + expect(truncated[0]).toBe(CLS); + expect(truncated.at(-1)).toBe(SEP); + expect(truncated.slice(1, -1)).toEqual(body.slice(0, MAX_TOKENS - 2)); + }); +}); + +describe("meanPoolNormalized", () => { + it("averages the token vectors and scales the result to unit length", () => { + // Two tokens, two dimensions: mean of [1, 0] and [3, 4] is [2, 2]. + const pooled = meanPoolNormalized(new Float32Array([1, 0, 3, 4]), 2, 2); + expect(pooled[0]).toBeCloseTo(Math.SQRT1_2, 6); + expect(pooled[1]).toBeCloseTo(Math.SQRT1_2, 6); + }); +}); + +// Reference values: the first 8 dimensions from Python onnxruntime (CPU) running the same +// full-precision model with the Hugging Face tokenizer, truncated to 256 tokens. +const REFERENCES = [ + { + text: "The quick brown fox jumps over the lazy dog.", + head: [0.043934, 0.058934, 0.048178, 0.077548, 0.026744, -0.03763, -0.002605, -0.059943], + }, + { + text: Array.from({ length: 400 }, (_, i) => `word${i % 50}`).join(" "), + head: [0.004672, -0.058774, -0.039712, -0.016036, 0.016489, 0.011557, 0.082817, -0.027109], + }, +]; + +describe("LocalEmbeddingProvider — skipped if the model cannot be loaded", () => { + const provider = new LocalEmbeddingProvider(); + let available = false; + + beforeAll(async () => { + initLogger("silent"); + try { + await provider.embed("warmup"); + available = true; + } catch { + available = false; + } + }, 300_000); + + it("matches the reference vectors for short and truncated long input", async () => { + if (!available) { + console.log(" [skip] all-MiniLM-L6-v2 model not available"); + return; + } + for (const { text, head } of REFERENCES) { + const vector = await provider.embed(text); + expect(vector).toHaveLength(provider.dimensions); + vector.slice(0, head.length).forEach((value, i) => { + expect(value).toBeCloseTo(head[i]!, 5); + }); + } + }); +}); diff --git a/tests/unit/schema-migrations.test.ts b/tests/unit/schema-migrations.test.ts index dec1f11..c3f1ecd 100644 --- a/tests/unit/schema-migrations.test.ts +++ b/tests/unit/schema-migrations.test.ts @@ -4,7 +4,8 @@ import type Database from "better-sqlite3"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { runMigrations } from "../../src/db/schema.js"; +import { PRE_2_0_1_MODEL, runMigrations } from "../../src/db/schema.js"; +import { readEmbeddingIdentity } from "../../src/db/index-meta.js"; import { searchDocuments } from "../../src/core/search.js"; import { initLogger } from "../../src/logger.js"; import { MockEmbeddingProvider } from "../fixtures/mock-provider.js"; @@ -234,4 +235,34 @@ describe("schema migrations", () => { expect(count(db, "index_meta")).toBe(0); }); }); + + describe("migration 20: model of vector indexes built by 1.x", () => { + it("records a placeholder model for a vector index without one", () => { + const db = newDb(); + runMigrations(db, 19); + // sqlite-vec is not loaded here; the migration only checks that the table exists. + db.exec("CREATE TABLE chunk_embeddings (chunk_id TEXT)"); + + runMigrations(db); + + expect(readEmbeddingIdentity(db).model).toBe(PRE_2_0_1_MODEL); + }); + + it("keeps a recorded model, and records nothing without a vector index", () => { + const withModel = newDb(); + runMigrations(withModel, 19); + withModel.exec("CREATE TABLE chunk_embeddings (chunk_id TEXT)"); + withModel + .prepare("INSERT INTO index_meta (key, value) VALUES ('embedding_model', ?)") + .run("nomic-embed-text"); + const withoutVectors = newDb(); + runMigrations(withoutVectors, 19); + + runMigrations(withModel); + runMigrations(withoutVectors); + + expect(readEmbeddingIdentity(withModel).model).toBe("nomic-embed-text"); + expect(count(withoutVectors, "index_meta")).toBe(0); + }); + }); }); diff --git a/tests/unit/schema.test.ts b/tests/unit/schema.test.ts index 3b1cd2e..5b4584c 100644 --- a/tests/unit/schema.test.ts +++ b/tests/unit/schema.test.ts @@ -38,7 +38,7 @@ describe("database schema", () => { const version = db.prepare("SELECT MAX(version) as v FROM schema_version").get() as { v: number; }; - expect(version.v).toBe(19); + expect(version.v).toBe(20); }); it("should create expected indexes", () => {