diff --git a/.dockerignore b/.dockerignore index b8d164d..925ca74 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,3 +17,4 @@ data/nlprule/ data/snapshots/ data/tagger/classes.txt data/tagger/weights.json +assets/all-MiniLM-L6-v2/ diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..bd246a7 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,59 @@ +name: Rust Coverage + +on: + push: + branches: + - main + - v0.7.2 + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + coverage: + name: Rust coverage + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CARGO_TERM_COLOR: always + + steps: + - name: Check out source + uses: actions/checkout@v6 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - name: Install nlprule tokenizer + env: + TOKENIZER_URL: https://cuemap.dev/assets/en_tokenizer.bin.gz + TOKENIZER_SHA256: f54fd31ec463f8646d0239bb531a64e0210ed1ae02bf5e3b42aeeb9bff8305ba + run: | + curl -fsSL --retry 3 "${TOKENIZER_URL}" -o "${RUNNER_TEMP}/en_tokenizer.bin.gz" + echo "${TOKENIZER_SHA256} ${RUNNER_TEMP}/en_tokenizer.bin.gz" | sha256sum -c - + gzip -dc "${RUNNER_TEMP}/en_tokenizer.bin.gz" > "${RUNNER_TEMP}/en_tokenizer.bin" + echo "TOKENIZER_PATH=${RUNNER_TEMP}/en_tokenizer.bin" >> "${GITHUB_ENV}" + + - name: Generate LCOV report + run: | + cargo llvm-cov clean --workspace + cargo llvm-cov --all-features --lib --no-report -- --test-threads=1 + cargo llvm-cov --all-features --bin cuemap --no-report -- --test-threads=1 + cargo llvm-cov --all-features --tests --no-report -- --test-threads=1 + cargo llvm-cov report --lcov --output-path lcov.info + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + flags: rust-engine + name: rust-engine-${{ github.sha }} + fail_ci_if_error: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8115fc2..2745613 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,8 +1,8 @@ name: Build and Publish Optional NPM Binaries on: - release: - types: [created] + push: + branches: [main] workflow_dispatch: # Allow manual triggering jobs: @@ -55,44 +55,64 @@ jobs: targets: ${{ matrix.target }} - name: Build Rust Engine - run: cargo build --release --target ${{ matrix.target }} + run: cargo build --locked --release --target ${{ matrix.target }} - name: Prepare NPM Package Structure shell: bash run: | - mkdir -p npm-packages/${{ matrix.pkg_name }}/bin + package_dir="npm-packages/${{ matrix.pkg_name }}" + mkdir -p "${package_dir}/bin" "${package_dir}/assets" - # Copy binary based on OS extension (.exe for Windows) - if [ "${{ matrix.os }}" = "windows-latest" ]; then - cp target/${{ matrix.target }}/release/cuemap.exe npm-packages/${{ matrix.pkg_name }}/bin/cuemap.exe - else - cp target/${{ matrix.target }}/release/cuemap npm-packages/${{ matrix.pkg_name }}/bin/cuemap - chmod +x npm-packages/${{ matrix.pkg_name }}/bin/cuemap - fi + cp target/${{ matrix.target }}/release/cuemap "${package_dir}/bin/cuemap-native" + cp scripts/npm-native-wrapper.cjs "${package_dir}/bin/cuemap" + cp scripts/npm-native-README.md "${package_dir}/README.md" + cp LICENSE "${package_dir}/LICENSE" + chmod +x "${package_dir}/bin/cuemap" "${package_dir}/bin/cuemap-native" + + tokenizer_url="https://cuemap.dev/assets/en_tokenizer.bin.gz" + tokenizer_sha256="f54fd31ec463f8646d0239bb531a64e0210ed1ae02bf5e3b42aeeb9bff8305ba" + curl -fsSL --retry 3 "${tokenizer_url}" -o "${package_dir}/assets/en_tokenizer.bin.gz" + node -e ' + const fs = require("node:fs"); + const crypto = require("node:crypto"); + const [file, expected] = process.argv.slice(1); + const actual = crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); + if (actual !== expected) throw new Error(`Tokenizer checksum mismatch: ${actual}`); + ' "${package_dir}/assets/en_tokenizer.bin.gz" "${tokenizer_sha256}" + gzip -dc "${package_dir}/assets/en_tokenizer.bin.gz" > "${package_dir}/assets/en_tokenizer.bin" + rm "${package_dir}/assets/en_tokenizer.bin.gz" - # Create the package.json for this specific architecture package - cat < npm-packages/${{ matrix.pkg_name }}/package.json - { - "name": "@cuemap-dev/${{ matrix.pkg_name }}", - "version": "${{ github.event.release.tag_name || '0.6.6' }}", - "description": "Pre-compiled CueMap Engine for ${{ matrix.npm_os }} ${{ matrix.npm_arch }}", - "engines": { - "node": ">= 18" - }, - "os": ["${{ matrix.npm_os }}"], - "cpu": ["${{ matrix.npm_arch }}"], - "bin": { - "cuemap": "bin/cuemap${{ matrix.os == 'windows-latest' && '.exe' || '' }}" - }, - "repository": { - "type": "git", - "url": "https://github.com/cuemap-dev/cuemap.git" - }, - "author": "Kaan Demirel", - "license": "BSL-1.1" - } - EOF + version="$(awk -F '"' '/^version = "/ { print $2; exit }' Cargo.toml)" + node -e ' + const fs = require("node:fs"); + const [output, version, os, cpu, packageName] = process.argv.slice(1); + const manifest = { + name: `@cuemap-dev/${packageName}`, + version, + description: `Pre-compiled CueMap Engine for ${os} ${cpu}`, + engines: { node: ">=18" }, + os: [os], + cpu: [cpu], + bin: { cuemap: "bin/cuemap" }, + files: ["bin", "assets", "README.md", "LICENSE"], + repository: { type: "git", url: "https://github.com/cuemap-dev/cuemap.git" }, + author: "Kaan Demirel", + license: "BSL-1.1", + publishConfig: { access: "public" }, + }; + if (os === "linux") manifest.libc = ["glibc"]; + fs.writeFileSync(output, `${JSON.stringify(manifest, null, 2)}\n`); + ' "${package_dir}/package.json" "${version}" "${{ matrix.npm_os }}" "${{ matrix.npm_arch }}" "${{ matrix.pkg_name }}" + + - name: Verify NPM package contents + working-directory: npm-packages/${{ matrix.pkg_name }} + shell: bash + run: | + test -x bin/cuemap + test -x bin/cuemap-native + test -s assets/en_tokenizer.bin + npm pack --dry-run - name: Publish to NPM working-directory: npm-packages/${{ matrix.pkg_name }} - run: npm publish --access public --verbose + run: npm publish --access public --provenance --verbose diff --git a/.gitignore b/.gitignore index b737bda..f47dd8b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,20 @@ data/reports/ data/snapshots/ tests/data/ snapshots/ -scripts/ +__pycache__/ +*.py[cod] +evals/**/bin/ +vendor/onnxruntime/archives/ +vendor/onnxruntime/ios-arm64/ +vendor/onnxruntime/ios-arm64-sim/ +assets/all-MiniLM-L6-v2/ +coverage/ +lcov.info +codecov.json +evals/intent/ +vendor/ + +# Local release outputs and helpers +benchmarks/benchmark_nl_results.json +evals/release-20260815_122352/ +scripts/run-sdk-integration-tests.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e66cd1..d133553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ All notable changes to the CueMap Rust Engine will be documented in this file. +## [0.7.2] - 2026-08-04 + +### Added +- **Bundled semantic reranking**: The default `semantic-encoder` build now uses qint8 `all-MiniLM-L3-v2` for memory/query embeddings and bounded hybrid reranking; no runtime model download or external service is required. The `edge` profile uses a bundled q4 build of the same model. +- **Intent classification and reranking**: Added local query/memory intent classification, confidence-weighted hybrid reranking, persisted memory annotations, the `/intent/classify` API, and intent coverage/progress fields in `/jobs/status`. +- **Caller-provided vectors**: Added per-memory `embedding`, per-query `query_embedding`, semantic recall modes, and one-vector-per-produced-chunk `embeddings` for `/ingest/content`. +- **Edge semantic profile**: Added the `edge` profile for constrained devices. It selects q4 MiniLM-L3 while lowering ANN fanout and the vector memory budget. +- **External unit-test files**: Moved all Rust unit-test modules out of `src` into `tests/unit`, preserving parent-module privacy while keeping production files focused on implementation. +- **Coverage reporting**: Added a `cargo-llvm-cov` GitHub Actions workflow with Codecov upload and a Rust README coverage badge. +- **CLI startup and handler coverage**: Added deterministic tests for layered profile/CLI configuration, snapshot-directory recovery, encryption-key precedence, context signing, KDF salt selection, configurable file/URL ingestion, lexicon inspection, default-project persistence, HTTP-backed add/recall/status/project/alias/memory/ingest flows, connection failures, recall modes, log rendering modes, static/live server bootstrap, detached readiness, and stop lifecycle handling. +- **Critical-path coverage**: Added deterministic HTTP API guard/reload and semantic-recall tests, parent-fusion and projection helper tests, CueBridge gap-expansion coverage, read-only/global route guards, recursive URL ingestion and persisted web recall tests, unconfigured backup checks, persistence corruption and local-backup tests, job-processing tests, ingester state/preview/deduplication tests, watcher event tests, and offline web-search parsing tests. +- **Engine coverage gate**: Added focused lifecycle, disk-content, temporal-chunking, semantic budget/error, structured-reranking, source-order expansion, decay/consolidation, and generic MainStats/LexiconStats tests. `src/engine.rs` now measures 94.10% lines, 93.79% regions, and 90.91% functions across the library and dedicated engine integration suite. +- **Server-health coverage gate**: Added project-context lifecycle, cue resolution/cache, symbol routing, alias filtering, snapshot version/corruption, atomic save/load, background snapshot, and local cloud-backup tests. `src/projects.rs` now measures 95.30% lines, 94.67% regions, and 95.83% functions; `src/persistence.rs` measures 90.40% lines, 87.30% regions, and 88.24% functions in the focused library/project coverage run. +- **Snapshot recovery**: New snapshots use zstd-compressed JSON so arbitrary `serde_json::Value` metadata survives restart; legacy bincode snapshots remain supported where decodable. Startup now reports per-project snapshot load failures and can discover the legacy sibling snapshot directory. Pre-v0.7.2 bincode snapshots containing dynamic JSON metadata may still require reingestion because bincode cannot decode `deserialize_any` values. + +### Changed +- **Semantic defaults**: The quality/default profile reports `all-MiniLM-L3-v2` (`bundled-qint8-minilm-l3`) with a 128-token window; the previous bundled L6 model is no longer part of the release binary. +- **Structural-only core semantics**: Removed CuePacks and domain-ontology facet rules from ingestion/query planning. The remaining deterministic planner emits structural evidence, metadata, grammatical perspective, answer shape, ordering, and reference-time signals. +- **Trained embedding-only intent categories**: Removed exact-match semantic phrase/vocabulary adjustments and runtime semantic anchor lists. A tiny model-specific linear head now maps frozen MiniLM embeddings to intent scores; syntax-only query shape can only admit an uncertain recall check without relabelling the intent or changing durable-memory eligibility. +- **Leakage-guarded intent training**: Added deterministic NumPy training for the L3-qint8 and L3-q4 intent heads. +- **Release package hygiene**: Native builds and Cargo packages now expose only the `cuemap` server binary and exclude local diagnostics, benchmarks, evals, vendor archives, caches, and the retired L6 assets. + +### Fixed +- **CLI stop PID validation**: Reject Unix PID values that cannot be represented as a positive `pid_t`, preventing malformed or stale PID files from turning a targeted shutdown into a process-wide signal. +- **Native npm Publishing**: Updated the GitHub Actions publisher to ship the checksum-verified tokenizer and package launcher on every supported platform, matching the local release packager. +- **Container semantic build**: Added bundled model assets to the Docker build context and synchronized the image version metadata. +- **Intent job completion**: Failed intent annotations now reach a terminal job phase while keeping `intent_ready=false`, rather than leaving ingestion permanently in `processing`. +- **Snapshot Coverage**: Added regression coverage for periodically persisting projects created after the snapshot scheduler starts. +- **Watcher deletion path normalization**: Deletion events now canonicalize the surviving parent path so files under macOS `/var` symlinked temporary roots remove the same tracking keys created during ingestion. +- **VerifyFile deadlock**: Verification now releases the cue-index read guard before deleting stale memories, preventing worker hangs during file re-ingestion cleanup. + ## [0.7.1] - 2026-07-17 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index a529e4f..b68f97e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,6 +36,7 @@ dependencies = [ "cfg-if", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -242,6 +243,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -254,6 +261,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bimap" version = "0.6.3" @@ -348,6 +361,15 @@ dependencies = [ "zip", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.56" @@ -482,6 +504,33 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -613,7 +662,7 @@ dependencies = [ [[package]] name = "cuemap" -version = "0.7.1" +version = "0.7.2" dependencies = [ "ahash", "aho-corasick 1.1.4", @@ -633,6 +682,7 @@ dependencies = [ "docx-rs", "futures", "globset", + "half 1.8.3", "hex", "hmac", "ignore", @@ -642,6 +692,7 @@ dependencies = [ "nlprule", "notify", "object_store", + "ort", "pbkdf2", "pdf-extract", "quick-xml 0.39.2", @@ -660,6 +711,7 @@ dependencies = [ "subtle", "tempfile", "time", + "tokenizers", "tokio", "toml", "tower 0.4.13", @@ -686,6 +738,56 @@ dependencies = [ "zstd", ] +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -699,6 +801,16 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -719,6 +831,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -797,6 +940,12 @@ dependencies = [ "serde", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -834,6 +983,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + [[package]] name = "euclid" version = "0.20.14" @@ -906,6 +1064,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1211,6 +1384,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "html5ever" version = "0.29.1" @@ -1449,6 +1628,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1516,6 +1701,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inotify" version = "0.11.0" @@ -1594,6 +1792,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1718,12 +1925,34 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "mac" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "markup5ever" version = "0.14.1" @@ -1764,6 +1993,16 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1814,6 +2053,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "moxcms" version = "0.7.11" @@ -1841,6 +2102,38 @@ dependencies = [ "version_check", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1869,7 +2162,7 @@ dependencies = [ "once_cell", "onig", "rayon", - "rayon-cond", + "rayon-cond 0.1.0", "serde", "serde_json", "srx", @@ -1923,12 +2216,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2010,12 +2321,73 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2039,6 +2411,18 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -2064,6 +2448,15 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2190,6 +2583,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postscript" version = "0.14.1" @@ -2424,6 +2832,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.11.0" @@ -2445,6 +2859,17 @@ dependencies = [ "rayon", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -2937,12 +3362,35 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "srx" version = "0.1.4" @@ -2960,6 +3408,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "streaming-iterator" version = "0.1.9" @@ -3188,6 +3642,40 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "indicatif", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.2", + "rayon", + "rayon-cond 0.4.0", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.49.0" @@ -3587,6 +4075,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -3605,6 +4102,18 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "universal-hash" version = "0.5.1" @@ -3627,6 +4136,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -3645,6 +4184,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3676,6 +4221,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -3851,6 +4402,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -3866,6 +4426,22 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3875,6 +4451,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index 7c81efe..f75cf2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,33 @@ [package] name = "cuemap" -version = "0.7.1" +version = "0.7.2" edition = "2021" +autobins = false license = "BSL-1.1" +description = "High-performance temporal-associative memory engine for AI agents" +readme = "README.md" +repository = "https://github.com/cuemap-dev/cuemap" +homepage = "https://cuemap.dev" +keywords = ["ai", "memory", "agents", "retrieval", "rag"] +categories = ["database-implementations", "text-processing"] +exclude = [ + "/.github", + "/benchmarks", + "/evals", + "/scripts", + "/src/bin", + "/tests/evals", + "/vendor", + "/assets/all-MiniLM-L6-v2", + "**/.DS_Store", + "**/__pycache__", + "**/*.pyc", +] [[bin]] name = "cuemap" path = "src/main.rs" -[[bin]] -name = "check_bin" -path = "src/bin/check_bin.rs" - [dependencies] axum = { version = "0.7", features = ["macros"] } tokio = { version = "1", features = ["full", "signal"] } @@ -49,6 +65,7 @@ tree-sitter-java = "0.23.0" tree-sitter-php = "0.23.0" csv = "1.3" serde_yaml = "0.9" +half = { version = "1.8.3", features = ["serde"] } roxmltree = "0.20" pdf-extract = "0.7.2" docx-rs = "0.4" @@ -81,6 +98,12 @@ base64 = "0.21" ahash = "0.8" toml = "1.0.3" quick-xml = "0.39.2" +ort = { version = "2.0.0-rc.13", optional = true, features = ["coreml", "lax-feature-matching"] } +tokenizers = { version = "0.23.1", optional = true } + +[features] +default = ["semantic-encoder"] +semantic-encoder = ["dep:ort", "dep:tokenizers"] [dev-dependencies] tokio = { version = "1.0", features = ["full", "test-util"] } @@ -130,10 +153,6 @@ path = "tests/agent/mod.rs" name = "facets" path = "tests/facets/mod.rs" -[[test]] -name = "cuepacks" -path = "tests/cuepacks/mod.rs" - [[test]] name = "cuebridge" path = "tests/cuebridge/mod.rs" diff --git a/Dockerfile b/Dockerfile index ad62ff3..65f2999 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,16 @@ # syntax=docker/dockerfile:1 -FROM rust:1.93-slim-bookworm AS builder +FROM rust:1.93-slim-trixie AS builder WORKDIR /build +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev \ + && rm -rf /var/lib/apt/lists/* + COPY Cargo.toml Cargo.lock ./ COPY src ./src -COPY cuepacks ./cuepacks +COPY assets ./assets COPY lemma_exceptions.json ./ COPY data/tagger/tags.json ./data/tagger/tags.json @@ -19,7 +23,7 @@ FROM scratch AS native-binary COPY --from=builder /build/cuemap /cuemap -FROM debian:bookworm-slim AS tokenizer +FROM debian:trixie-slim AS tokenizer ARG TOKENIZER_URL="https://cuemap.dev/assets/en_tokenizer.bin.gz" ARG TOKENIZER_SHA256="f54fd31ec463f8646d0239bb531a64e0210ed1ae02bf5e3b42aeeb9bff8305ba" @@ -32,9 +36,9 @@ RUN apt-get update \ && gzip -dc /tmp/en_tokenizer.bin.gz > /en_tokenizer.bin \ && rm /tmp/en_tokenizer.bin.gz -FROM debian:bookworm-slim AS runtime +FROM debian:trixie-slim AS runtime -ARG VERSION=0.7.1 +ARG VERSION=0.7.2 ARG REVISION="" LABEL org.opencontainers.image.title="CueMap Engine" \ diff --git a/README.md b/README.md index 5160adf..1c8c46b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # CueMap Rust Engine +[![CI](https://github.com/cuemap-dev/cuemap/actions/workflows/coverage.yml/badge.svg?branch=v0.7.2)](https://github.com/cuemap-dev/cuemap/actions/workflows/coverage.yml) +[![Coverage](https://codecov.io/github/cuemap-dev/cuemap/branch/v0.7.2/graph/badge.svg?flag=rust-engine)](https://app.codecov.io/github/cuemap-dev/cuemap) + **High-performance temporal-associative memory store** designed for dynamic contextual retrieval. ## Overview @@ -7,14 +10,14 @@ CueMap implements a **Continuous Gradient Algorithm** optimized for associative data structures: 1. **Intersection (Context Filter)**: Triangulates relevant memories by overlapping cues -2. **CuePack-Guided Intent Routing**: Uses compiled deterministic rules to add structural facets and weighted intent cues without runtime model calls. +2. **Structural Extraction**: Emits deterministic cues for observable evidence such as dates, numbers, lists, source metadata, and surface entities. 3. **Recency & Salience (Signal Dynamics)**: Balances fresh data with salient, high-signal events prioritized by an adaptive impact scoring module. 4. **Reinforcement (Access-based Learning)**: Frequently accessed memories gain signal strength, remaining highly accessible even as they age. -5. **Deterministic Facets & Intent Routing**: Extracts synchronous source, evidence, temporal, type, and entity facets, then uses sparse intent cues and reranking during recall. +5. **Sparse Recall**: Uses normalized lexical cues, structural facets, recency, salience, and bounded deterministic reranking. -As of v0.7.0, CueMap's core path is deterministic and embedding-free. GloVe/Ollama cue generation, WordNet/POS expansion, semantic bridges, pattern completion, external lexicon graphs, context expansion/speculation endpoints, and autonomous consolidation have been removed from the core engine. +As of v0.7.2, CueMap's default core path is deterministic and ontology-free. GloVe/Ollama cue generation, WordNet/POS expansion, semantic bridges, pattern completion, external lexicon graphs, context expansion/speculation endpoints, and autonomous consolidation have been removed from the default engine path. v0.7.2 bundles a qint8 `all-MiniLM-L3-v2` vector layer for semantic reranking, intent classification, and query embeddings; the `edge` profile selects a q4 build of the same model. The encoder can still be disabled for constrained builds or deployments. -v0.7.0 also uses numeric per-project memory IDs everywhere. If callers need deterministic upsert/dedupe identity, pass `source_key`; memory IDs remain compact runtime addresses. +v0.7.2 also uses numeric per-project memory IDs everywhere. If callers need deterministic upsert/dedupe identity, pass `source_key`; memory IDs remain compact runtime addresses. Built with Rust for maximum performance and reliability. @@ -36,8 +39,8 @@ CueMap treats the nlprule tokenizer as a runtime asset, not a build artifact. Se ### Docker ```bash -docker build -t cuemap/engine:0.7.1 . -docker run -p 8080:8080 -v "$(pwd)/local_snapshot_dir:/app/data" cuemap/engine:0.7.1 +docker build -t cuemap/engine:0.7.2 . +docker run -p 8080:8080 -v "$(pwd)/local_snapshot_dir:/app/data" cuemap/engine:0.7.2 ``` The container runs as the unprivileged `cuemap` user. Ensure a bind-mounted data directory is writable by UID/GID `10001`, or use a Docker-managed volume. Runtime defaults can be overridden with `CUEMAP_PORT`, `CUEMAP_DATA_DIR`, `CUEMAP_SNAPSHOT_INTERVAL_SECONDS`, `TOKENIZER_PATH`, and `RUST_LOG`. @@ -51,7 +54,7 @@ Build the Darwin ARM64, Darwin x64, and Linux x64 native packages without publis ./scripts/verify-npm-native-packages.sh ``` -The packager builds Linux on Debian Bookworm for an older glibc baseline, bundles the checksum-pinned tokenizer, and writes package tarballs plus `SHA256SUMS` under `dist/npm-native/tarballs`. +The packager builds Linux on Debian Trixie, bundles the checksum-pinned tokenizer, and writes package tarballs plus `SHA256SUMS` under `dist/npm-native/tarballs`. ### CLI Commands @@ -85,7 +88,6 @@ cuemap [OPTIONS] #### Deterministic Semantics - **`lexicon`**: Inspect lexicon entries and wire/unwire cues. - **`alias`**: Manage explicit deterministic aliases. -- **`cuepack`**: List, inspect, and validate deterministic semantic packs. Hint: Use `cuemap --help` to see available commands and options. @@ -162,21 +164,16 @@ CueMap provides complete project isolation with automatic persistence: ### Usage -CueMap runs in multi-tenant mode by default. Simply specify a project ID in your requests. +CueMap runs in multi-tenant mode by default. Select a project for CLI commands with `cuemap set-project` or pass `--project` to an individual command. ```bash -# Start server +# Start the server ./target/release/cuemap start --port 8080 -``` - -### Example -```bash -# Add memory to project -curl -X POST http://localhost:8080/memories \ - -H "X-Project-ID: my-project" \ - -H "Content-Type: application/json" \ - -d '{"content": "Important data", "cues": ["test"]}' +# Choose a project and use the local CLI +cuemap set-project my-project +cuemap add "Important data" +cuemap recall "What is important?" # Stop server (Ctrl+C) - saves all projects when persistence is enabled # Restart server - loads persisted snapshots @@ -189,8 +186,9 @@ Snapshots are automatically managed: - **Created**: Periodically and on graceful shutdown (SIGINT/Ctrl+C) when persistence is enabled. - **Loaded**: On server startup - **Disabled**: `--disable-snapshots` turns off periodic and shutdown snapshot saves. -- **Location**: `~/.cuemap/data/snapshots/` by default, or `<--data-dir>/snapshots` when `--data-dir` is set. -- **Format**: Bincode binary +- **Location**: `~/.cuemap/data/snapshots/` by default, or `<--data-dir>/snapshots` when `--data-dir` is set. Older installs may also be discovered under the legacy sibling `snapshots/` directory. +- **Format**: zstd-compressed JSON inside `.bin` files. This preserves arbitrary metadata reliably while keeping snapshots compact; older uncompressed bincode snapshots remain readable when their metadata can be decoded. +- **Migration note**: Some pre-v0.7.2 bincode snapshots that contain dynamic JSON metadata cannot be decoded by bincode's `deserialize_any` limitation. Those projects are reported at startup and must be reingested or exported from a compatible older binary before upgrading. - **Files**: `{project-id}.bin`, `{project-id}_lexicon.bin`, `{project-id}_aliases.bin` ### Cloud Backup @@ -215,7 +213,7 @@ Enable cloud backup via CLI flags or `~/.cuemap/server_config.toml`. - `local`: Local path (for testing/replication) **Management**: -Backups can be triggered manually via API (`/backup/upload`, `/backup/download`). +Manual backup operations are documented in the [HTTP API reference](https://cuemap.dev/docs/api-reference). ## Authentication @@ -241,52 +239,7 @@ Or configure keys in `~/.cuemap/server_config.toml`: api_keys = ["your-secret-key"] ``` -### Using Authentication - -Include the API key in the `X-API-Key` header: - -```bash -# Without auth (fails if enabled) -curl http://localhost:8080/stats -# Response: Missing X-API-Key header - -# With correct key -curl -H "X-API-Key: your-secret-key" -H "X-Project-ID: default" http://localhost:8080/stats -# Response: {"total_memories": 1000, ...} - -# With wrong key -curl -H "X-API-Key: wrong-key" -H "X-Project-ID: default" http://localhost:8080/stats -# Response: Invalid API key -``` - -### SDK Usage - -#### Standard SDKs - -Python: -```python -from cuemap import CueMap - -# With authentication -client = CueMap( - url="http://localhost:8080", - api_key="your-secret-key" -) - -client.add("Memory", cues=["test"]) -``` - -TypeScript: -```typescript -import CueMap from 'cuemap'; - -const client = new CueMap({ - url: 'http://localhost:8080', - apiKey: 'your-secret-key' -}); - -await client.add('Memory', ['test']); -``` +Clients send the configured key in the `X-API-Key` header. See the [HTTP API reference](https://cuemap.dev/docs/api-reference) for request headers and SDK examples. ### Docker with Authentication @@ -324,61 +277,88 @@ To optimize storage efficiency, especially for large textual memories, CueMap em ## Performance -### Benchmark Results (v0.7.0) +### Benchmark Results (v0.7.2) Tests performed on **Real-World Data** (Wikipedia Articles), processing full natural language sentences with the complete NLP pipeline. -**Hardware:** MacBook Pro M-series, 64GB RAM, single node. These are local v0.7 benchmark numbers from the numeric-ID engine. +**Hardware:** MacBook Pro M-series, 64GB RAM, single node. The v0.7.2 release table below records completed lexical and hybrid runs at 10K, 100K, and 1M memories. Lexical runs isolate the sparse core with the semantic encoder disabled; hybrid runs include the bundled local encoder. P95 is the release headline percentile, while P99 remains available in the JSON diagnostics. #### Benchmark Methodology The NL benchmark script lives at `benchmarks/benchmark_nl.py`. Benchmark setup: -- Uses Wikipedia parquet files as the source corpus. You can obtain Wikipedia source data from official [Wikimedia dumps](https://dumps.wikimedia.org/) or an Internet Archive mirror, then prepare/extract text into parquet files with a `text` column. +- Uses the public [Wikipedia Plaintext (2023-07-01) Kaggle dataset](https://www.kaggle.com/datasets/jjinho/wikipedia-20230701) as the release corpus. The benchmark script downloads it automatically when `--wikipedia-path` is omitted, caches it under `~/.cache/cuemap/benchmarks/wikipedia-20230701`, and samples parquet files with a `text` column. Install the downloader first with `python -m pip install kagglehub`; configure Kaggle access if Kaggle prompts for authentication. To avoid the download or use another corpus, pass `--wikipedia-path /path/to/parquet-or-directory`. - Deduplicates sampled snippets and consumes them without replacement, so 100K and 1M write runs do not reuse the same text. - Writes use HTTP `POST /memories` with `minimal_response=true` and no explicit cues, forcing CueMap to run deterministic cue/facet extraction and indexing. - Reads generate keyword-style natural-language queries from retained ingested snippets. -- Recall numbers use the script's lean recall mode: `auto_reinforce=false`, salience disabled, alias expansion disabled, CueBridge artifacts disabled, `depth=1`, `expansion_depth=1`, and parent/order/evidence reconstruction disabled. This isolates the core sparse recall path. +- Recall numbers use the script's lean recall mode: `semantic_mode=lexical`, `auto_reinforce=false`, salience disabled, alias expansion disabled, CueBridge artifacts disabled, `depth=1`, `expansion_depth=1`, and parent/order/evidence reconstruction disabled. This isolates the core sparse recall path from the bundled semantic encoder and reranker. +- Each requested size runs in its own run-scoped project, so a 1M pass is not layered on top of a previous 100K pass or stale state from an earlier invocation. - `--trace-timing` records engine timing breakdowns but is not required for throughput measurements. -Example run: +Example run with the checked-in release fixture: ```bash -cuemap start --disable-snapshots --disable-bg-jobs +CUEMAP_SEMANTIC_ENCODER_ENABLED=false cuemap start --disable-snapshots --disable-bg-jobs python benchmarks/benchmark_nl.py \ - --sizes 100000,1000000 \ + --sizes 10000,100000,1000000 \ --project-id nl_test \ - --wikipedia-path /Users//Downloads/wikipedia/ \ - --trace-timing \ + --semantic-mode lexical \ --wiki-reservoir-size 20000 \ --query-sample-size 5000 \ --payload-buffer-size 500 ``` -#### 1. Ingestion (Write) Performance -*Measures HTTP ingestion, deterministic cue/facet extraction, source-key upsert, and indexing.* +Restart the engine without `CUEMAP_SEMANTIC_ENCODER_ENABLED=false`, then run the +same command with `--semantic-mode hybrid` to produce the hybrid comparison. The +dataset is downloaded only once and reused from the local cache on subsequent +runs. + +#### v0.7.2 latency comparison -| Dataset Scale | Avg Latency | P50 | P99 | Throughput | +The lexical release rerun now covers 10K, 100K, and 1M writes plus lean recall +queries. The compact comparison below records the 10K, 100K, and 1M hybrid +runs as well. + +| Mode | Write avg | Write P50 | Write P95 | Write throughput | Read avg | Read P50 | Read P95 | Read throughput | +|:---|---:|---:|---:|---:|---:|---:|---:|---:| +| Lexical | 2.13 ms | 1.88 ms | 4.13 ms | 470 ops/s | 1.05 ms | 1.01 ms | 1.60 ms | 939 ops/s | +| Hybrid | 11.52 ms | 10.38 ms | 17.45 ms | 87 ops/s | 6.54 ms | 6.80 ms | 8.41 ms | 152 ops/s | + +The script still stores p99 in the JSON result for diagnostics, but p95 is the +headline percentile used by the console output and release chart. + +| Hybrid scale | Write avg | Write P50 | Write P95 | Write throughput | Read avg | Read P50 | Read P95 | Read throughput | +|:---|---:|---:|---:|---:|---:|---:|---:|---:| +| **10,000** | 11.52 ms | 10.38 ms | 17.45 ms | 87 ops/s | 6.54 ms | 6.80 ms | 8.41 ms | 152 ops/s | +| **100,000** | 11.21 ms | 10.27 ms | 16.43 ms | 89 ops/s | 6.94 ms | 7.14 ms | 8.83 ms | 144 ops/s | +| **1,000,000** | 11.28 ms | 10.38 ms | 16.81 ms | 89 ops/s | 8.36 ms | 8.12 ms | 10.67 ms | 119 ops/s | + +#### 1. Ingestion (Write) Performance — lexical +*Measures HTTP ingestion, deterministic cue/facet extraction, memory allocation, and indexing.* + +| Dataset Scale | Avg Latency | P50 | P95 | Throughput | |:---|:---|:---|:---|:---| -| **100,000** | 3.13 ms | 2.56 ms | 10.98 ms | 320 ops/s | -| **1,000,000** | 2.85 ms | 2.39 ms | 11.23 ms | 351 ops/s | +| **10,000** | 2.13 ms | 1.88 ms | 4.13 ms | 470 ops/s | +| **100,000** | 2.92 ms | 2.41 ms | 5.70 ms | 343 ops/s | +| **1,000,000** | 3.33 ms | 2.74 ms | 6.18 ms | 301 ops/s | Write latency remains mostly flat with project size; the dominant cost is per-memory extraction/indexing rather than corpus scan time. -#### 2. Recall (Read) Performance +#### 2. Recall (Read) Performance — lexical *Measures the time to parse a query, resolve deterministic cues, and score sparse candidate intersections.* -| Dataset Scale | Avg Latency | P50 | P99 | -|:---|:---|:---|:---| -| **100,000** | 1.73 ms | 1.65 ms | 3.67 ms | -| **1,000,000** | 2.70 ms | 2.06 ms | 5.10 ms | +| Dataset Scale | Avg Latency | P50 | P95 | Throughput | +|:---|:---|:---|:---|---:| +| **10,000** | 1.05 ms | 1.01 ms | 1.60 ms | 939 ops/s | +| **100,000** | 1.86 ms | 1.71 ms | 3.16 ms | 535 ops/s | +| **1,000,000** | 2.63 ms | 2.06 ms | 3.72 ms | 378 ops/s | **Key Metrics**: -- **Low-latency recall:** 1M-memory natural-language recall stays around 2.7ms average with about 5.1ms p99 in the current v0.7 run. +- **Low-latency recall:** The lexical v0.7.2 1M run measured 2.63ms average with 3.72ms p95; hybrid measurements remain separate because they include bundled encoder work. - **Numeric ID memory reduction:** 1M in-memory footprint dropped from about 5.25GB to about 1.93GB after the v0.7 numeric memory-ID refactor. -- **Deterministic hot path:** recall uses in-memory sparse indexes and does not call embeddings, LLMs, network services, or disk scans. +- **Controlled hot path:** the release benchmark disables the local semantic encoder, LLMs, network services, and disk scans; normal v0.7.2 hybrid recall can use the bundled local encoder for bounded reranking. ## Architecture @@ -398,379 +378,15 @@ Write latency remains mostly flat with project size; the dominant cost is per-me - **Unstable sorting**: 2-3x faster than stable sort - **Iterative deepening**: Early termination on hot paths -## API - -### Deterministic Cue Extraction - -CueMap extracts cues synchronously from content and metadata using deterministic tokenization, normalization, facets, aliases, and CuePack rules. The recall path does not call embeddings, LLMs, WordNet, external APIs, or runtime graph expansion. - -```bash -# 1. Start CueMap -./target/release/cuemap start - -# 2. Add memory in natural language -curl -X POST http://localhost:8080/memories \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "content": "The payments service is down due to a timeout." - }' -# Deterministic extraction adds normalized lexical cues plus structural/facet cues. -``` - -## API Reference - -### Add Memory - -```bash -# Basic manual cues -curl -X POST http://localhost:8080/memories \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "content": "API Rate Limit Policy: 1000/min", - "cues": ["api", "rate_limit", "policy"], - "source_key": "doc:api-rate-limit-policy" - }' - -# Deterministic cues are extracted from content when `cues` is omitted or empty -curl -X POST http://localhost:8080/memories \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "content": "The payments service is down due to a timeout." - }' -``` - -### Recall Memories - -#### Explicit Cues -```bash -curl -X POST http://localhost:8080/recall \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "cues": ["api", "rate_limit"], - "limit": 10 - }' -``` - -#### Natural Language Search (Symbol-First Intent Routing) -```bash -curl -X POST http://localhost:8080/recall \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "query_text": "where is process_data used?", - "limit": 10, - "expansion_depth": 2 - }' -``` -Returns surgical code recall. The engine uses a deterministic **Symbol-First Router** with sparse BM25-style scoring to convert fuzzy queries into structural cues (e.g., `calls_function:process_data`). Set `expansion_depth` above 1 to include nearby source-order chunks when session/order metadata is available. - -```json -{ - "explain": { - "query_cues": ["payments"], - "expanded_cues": [ - ["payments", 1.0], - ["service:payments", 0.85] - ] - }, - "results": [ - { - "content": "...", - "score": 145.2, - "explain": { - "intersection_weighted": 1.85, - "recency_component": 0.5 - } - } - ] -} -``` - -### Reinforce Memory - -```bash -curl -X PATCH http://localhost:8080/memories/{id}/reinforce \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "cues": ["important", "urgent"] - }' -``` -Reinforcement is used to boost the relevance of a memory. It is a way to tell CueMap that a memory is important and should be recalled more often. Standard `POST /recall` requests do not auto-reinforce by default; enable `auto_reinforce` explicitly or reinforce a memory manually through the API. - -### Get Memory - -```bash -curl -H "X-Project-ID: default" http://localhost:8080/memories/{id} -``` - -### Get Stats -```bash -curl -H "X-Project-ID: default" http://localhost:8080/stats -``` - -### Alias Management - -Manage synonyms and semantic mappings deterministically. - -#### Add Alias -```bash -curl -X POST http://localhost:8080/aliases \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "from": "pay", - "to": "service:payment", - "weight": 0.9 - }' -``` - -#### Merge Aliases (Bulk) -```bash -curl -X POST http://localhost:8080/aliases/merge \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "cues": ["bill", "invoice", "statement"], - "to": "service:billing" - }' -``` - -#### Get Aliases -```bash -# Reverse lookup: Find all aliases for "service:payment" -curl -H "X-Project-ID: default" "http://localhost:8080/aliases?cue=service:payment" -``` - -### Project Management - -#### Create Project -```bash -curl -X POST http://localhost:8080/projects \ - -H "Content-Type: application/json" \ - -d '{"project_id": "my-project"}' -``` - -#### List Projects -```bash -curl http://localhost:8080/projects -``` - -#### Delete Project -```bash -curl -X DELETE "http://localhost:8080/projects/default" -``` - -### Lexicon Management - -#### Inspect Cue -View incoming tokens and manually wired canonical cue mappings. -```bash -curl "http://localhost:8080/lexicon/inspect/service:payment" -``` - -#### Wire Token (Manual Connection) -Manually connect a token to a canonical cue. -```bash -curl -X POST http://localhost:8080/lexicon/wire \ - -H "Content-Type: application/json" \ - -d '{ - "token": "stripe", - "canonical": "service:payment" - }' -``` - -#### Unwire/Delete Entry -Remove a specific token from the lexicon. -```bash -curl -X DELETE "http://localhost:8080/lexicon/entry/cue:stripe" -``` - -### CuePacks - -CuePacks are deterministic semantic packages. They are the maintainable place for domain vocabulary, semantic phrase families, facet rules, query-intent rules, aliases, and policy metadata. CuePacks are compiled at startup or request setup; recall does not call a network service, run embeddings, scan raw memory content, or read pack files from disk. - -Bundled defaults are enabled unless disabled. Place custom packs in `~/.cuemap/cuepacks/` as TOML files and inspect them with: - -```bash -cuemap cuepack list -cuemap cuepack inspect memory-general -cuemap cuepack validate ./my-domain-pack.toml -``` - -Select packs per request: - -```bash -cuemap recall -p my_project --cuepacks memory-general "which transit app did I use?" -cuemap recall -p my_project --disable-default-cuepacks "core-only recall" -``` - -API requests accept a separate `cuepacks` field. Use `["off"]` for core-only behavior, omit the field for bundled defaults, or pass explicit pack names. - -### CueBridge Artifacts - -CueBridge artifacts are offline-compiled lexical-gap packages. CueMap loads them into memory and uses them deterministically during recall: - -- **AliasPack**: safe lexical variants applied during query cue resolution. -- **GapPack**: gated expansion cues applied only when exact recall is weak. - -Install artifacts into the project artifact directory, then reload them: - -```bash -curl -X POST http://localhost:8080/projects/my-project/artifacts -``` - -Inspect active artifacts: - -```bash -curl http://localhost:8080/projects/my-project/artifacts -``` - -Recall can disable installed artifacts for baseline checks: - -```bash -cuemap recall -p my_project --disable-cuebridge-artifacts "what foundation did we choose?" -``` - -### Cloud Backup Management - -#### Upload Snapshot -```bash -curl -X POST http://localhost:8080/backup/upload \ - -H "Content-Type: application/json" \ - -d '{"project_id": "default"}' -``` - -#### Download Snapshot -```bash -curl -X POST http://localhost:8080/backup/download \ - -H "Content-Type: application/json" \ - -d '{"project_id": "default"}' -``` - -#### List Backups -```bash -curl http://localhost:8080/backup/list -``` - -### Monitoring - -#### Prometheus Metrics -Exposes internal system metrics for scraping (Prometheus format). - -```bash -curl http://localhost:8080/metrics -# Output: -# cuemap_ingestion_rate 120.0 -# cuemap_recall_latency_p99 0.8 -# cuemap_memory_usage_bytes 1024 -# ... -``` - -### Ingestion - -#### Ingest URL -Extract content from a web page and ingest it. -```bash -curl -X POST http://localhost:8080/ingest/url \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://example.com" - }' -``` - -#### Ingest Raw Content -Ingest text directly, simulating a file. -```bash -curl -X POST http://localhost:8080/ingest/content \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "content": "The quick brown fox jumps over the lazy dog.", - "filename": "fox.txt" - }' -``` - -#### Ingest File (Multipart) -Upload a file for processing by the Agent (supports Text, PDF, JSON, etc. if Agent is configured). -```bash -curl -X POST http://localhost:8080/ingest/file \ - -H "X-Project-ID: default" \ - --form "file=@/path/to/document.pdf" -``` - -#### Grounded Recall (Budgeted) - -```bash -curl -X POST http://localhost:8080/recall/grounded \ - -H "X-Project-ID: default" \ - -H "Content-Type: application/json" \ - -d '{ - "query_text": "Why is the server down?", - "token_budget": 500, - "limit": 10 - }' -``` - -Grounded recall deterministically fills a token budget with the highest-scoring memories and returns a context block designed to be passed to an LLM alongside a structured proof. - -Grounded recall enables `auto_reinforce` by default; set `"auto_reinforce": false` for read-only evaluation or benchmark runs. - -**Response** (example with Ed25519 signing configured): -```json -{ - "verified_context": "[VERIFIED CONTEXT] (1) Fact... Rules:...", - "proof": { - "trace_id": "966579b1-...", - "selected": [...], - "excluded_top": [...] - }, - "signature_alg": "ed25519", - "signature": "9b2d...", - "public_key": "ed25519:4f8c...", - "engine_latency_ms": 0.83 -} -``` - -### Signed Context (Immutable RAG) - -CueMap can sign grounded recall context so clients can verify that the `verified_context` block was produced by the configured CueMap server and was not modified in transit before it reaches an LLM. - -Preferred setup uses Ed25519 asymmetric signatures. Generate one 32-byte private seed, store it securely, and reuse it across restarts: - -```bash -openssl rand -hex 32 > ~/.cuemap/signing_ed25519_seed.hex -CUEMAP_SIGNING_PRIVATE_KEY="$(cat ~/.cuemap/signing_ed25519_seed.hex)" cuemap start -``` - -Or configure it in `~/.cuemap/server_config.toml`: - -```toml -[security] -signing_private_key = "ed25519:<32-byte-hex-seed>" -``` - -Grounded recall responses include the signature algorithm and public key: - -```json -{ - "verified_context": "...", - "signature_alg": "ed25519", - "signature": "9b2d...", - "public_key": "ed25519:4f8c..." -} -``` +## HTTP API and SDK documentation -Clients verify `signature` over the exact UTF-8 bytes of `verified_context` using the pinned Ed25519 public key. The signature is a lowercase hex-encoded 64-byte Ed25519 signature. The public key is `ed25519:` plus the lowercase hex-encoded raw 32-byte Ed25519 public key. +The complete HTTP/OpenAPI contract, request and response schemas, authentication headers, ingestion and recall routes, and Python/TypeScript SDK examples live in the [CueMap documentation](https://cuemap.dev/docs/api-reference). -Treat the response `public_key` as discovery metadata; production clients should pin the expected public key from deployment config rather than trusting a key delivered by the same response they are verifying. +- [Quick start and operations](https://cuemap.dev/docs/) +- [HTTP API reference](https://cuemap.dev/docs/api-reference) +- [OpenAPI 3.1 schema](https://cuemap.dev/openapi.yml) -For compatibility, `CUEMAP_SECRET_KEY` still enables legacy `hmac-sha256` signatures. HMAC verification requires sharing the same secret with verifiers, so Ed25519 is recommended for client-side or third-party verification. +The website docs are the source of truth for endpoint behavior and are kept aligned with the checked-in Rust router. This README stays focused on building, operating, and understanding the engine; use the CLI and MCP sections above for the fastest local workflows. ## System Architecture @@ -802,7 +418,7 @@ graph TB subgraph "Intelligence" NL[NL Tokenizer
Lemmatization + RAKE] - PACKS[CuePacks
Facet + Intent Rules] + STRUCT[Structural Facets
Evidence + Metadata] end subgraph "Persistence" @@ -821,7 +437,6 @@ graph TB AXUM --> SESSION QUEUE --> LEX - AXUM --> PACKS MAIN <-.-> PERSIST LEX <-.-> PERSIST @@ -832,18 +447,18 @@ graph TB style QUEUE fill:#9C27B0 ``` -### 2. Write Flow (POST /memories) +### 2. Write Flow ```mermaid sequenceDiagram participant C as Client - participant API as API Handler + participant API as HTTP Handler participant NL as NL Tokenizer participant Norm as Normalizer participant Tax as Taxonomy participant Main as CueMap Engine - C->>API: POST /memories
{content, cues[]} + C->>API: Memory write request
{content, cues[]} alt cues[] is empty API->>NL: tokenize_to_cues(content) @@ -865,19 +480,19 @@ sequenceDiagram Note over API,Main: Cue extraction and indexing happen synchronously ``` -### 3. Read Flow (POST /recall) +### 3. Read Flow ```mermaid sequenceDiagram participant C as Client - participant API as API Handler + participant API as HTTP Handler participant Lex as Lexicon participant Alias as Alias Engine participant Art as CueBridge Artifacts participant Main as CueMap Engine participant Q as Job Queue - C->>API: POST /recall
{query_text?, cues[], limit} + C->>API: Recall request
{query_text?, cues[], limit} alt query_text provided API->>Lex: resolve_cues_from_text(query) @@ -915,8 +530,8 @@ sequenceDiagram ```mermaid graph TB subgraph "Job Sources" - INGEST[Ingest API] - RECALL[POST /recall] + INGEST[Ingestion] + RECALL[Recall] AGENT[Self-Learning Agent] TIMER[60s Heatmap Tick] end @@ -973,23 +588,22 @@ The agent transforms your local filesystem into a deterministic structural knowl * **Documents**: PDF (text extraction), Word (DOCX), Excel (XLSX). * **Data**: CSV (row-aware), JSON (key-aware), YAML, XML. * **Tree-sitter Powered Chunking**: Smartly splits code into functions, classes, and modules while preserving context. -* **Deterministic Knowledge Extraction**: Uses tree-sitter structure, document parsers, metadata facets, token normalization, and CuePack rules; no runtime model call is required. +* **Deterministic Knowledge Extraction**: Uses tree-sitter structure, document parsers, metadata facets, and token normalization; no runtime model call is required. * **Idempotent Updates**: Uses content-aware hashing (`file::`) to prevent memory duplication and ensure stale memories are pruned. * **Background Verification Loop**: Continuously verifies that memories in the engine still exist on disk, pruning stale references automatically. ### 2. Deterministic Natural Language Engine -CueMap bridges unstructured text to sparse deterministic recall without vector search, runtime models, or background semantic expansion. +CueMap bridges unstructured text to sparse deterministic recall without vector search, runtime models, or background semantic expansion by default. Optional vector retrieval can add externally computed semantic candidates without changing the structural extraction path. #### How It Works At add-time, CueMap extracts cues synchronously from real structure: - normalized lexical cues -- entity, quote, model-like, and quantity-object cues +- surface entity, quote, model-like, and structural evidence cues - evidence facets such as numbers, money, dates, durations, and lists - source facets from metadata such as role, channel, session, and order -- CuePack-derived deterministic facet and intent cues At query-time, CueMap uses the same deterministic normalization path, then applies only bounded in-memory expansions: @@ -1000,9 +614,8 @@ At query-time, CueMap uses the same deterministic normalization path, then appli #### Semantic Boundary -CueMap Core does not try to infer broad semantic relationships from local co-occurrence. That keeps recall fast, deterministic, and inspectable. Semantic gap closure belongs in explicit artifacts: +CueMap Core does not try to infer broad semantic relationships from local co-occurrence or ontology rules. That keeps the default recall fast, deterministic, and inspectable. Semantic gap closure can come from externally precomputed vectors or explicit artifacts: -- **CuePacks**: deterministic domain rules, facets, aliases, and query intent policies. - **Manual Lexicon Wiring**: explicit token-to-canonical cue connections for project owners. - **CueBridge Artifacts**: offline-compiled GapPack/AliasPack files generated by CueBridge Local or Cloud and loaded into CueMap. diff --git a/assets/all-MiniLM-L3-v2/SHA256SUMS b/assets/all-MiniLM-L3-v2/SHA256SUMS new file mode 100644 index 0000000..c988a71 --- /dev/null +++ b/assets/all-MiniLM-L3-v2/SHA256SUMS @@ -0,0 +1,5 @@ +44a2d6852c28e7a4cff4d77a8bc8d5ca2a8b38884c1fa746e6652d830109e0c2 model_qint8_arm64.onnx +f4b6a11fd27a22983c1945696f348d452021a0c26a4712b2f603ba4862c16b95 model_int4.onnx +a9576e4dadfe7f78f071a1c00ba6902cc83ec6ed7e9b590ee974950bd4d54393 tokenizer.json +3cfd13c9dec40ad5c8a99341e99f061cf8e2b598e2697c391f978ada41bbd1bd intent_probe_qint8.head +e741a9169a56b33c8f27f22e8f15eaadc9156cd09a15076db61d85ca2923d9cd intent_probe_q4.head diff --git a/assets/all-MiniLM-L3-v2/intent_probe_q4.head b/assets/all-MiniLM-L3-v2/intent_probe_q4.head new file mode 100644 index 0000000..5431813 Binary files /dev/null and b/assets/all-MiniLM-L3-v2/intent_probe_q4.head differ diff --git a/assets/all-MiniLM-L3-v2/intent_probe_qint8.head b/assets/all-MiniLM-L3-v2/intent_probe_qint8.head new file mode 100644 index 0000000..306b23c Binary files /dev/null and b/assets/all-MiniLM-L3-v2/intent_probe_qint8.head differ diff --git a/assets/all-MiniLM-L3-v2/model_int4.onnx b/assets/all-MiniLM-L3-v2/model_int4.onnx new file mode 100644 index 0000000..d0a33a8 Binary files /dev/null and b/assets/all-MiniLM-L3-v2/model_int4.onnx differ diff --git a/assets/all-MiniLM-L3-v2/model_qint8_arm64.onnx b/assets/all-MiniLM-L3-v2/model_qint8_arm64.onnx new file mode 100644 index 0000000..3124850 Binary files /dev/null and b/assets/all-MiniLM-L3-v2/model_qint8_arm64.onnx differ diff --git a/assets/all-MiniLM-L3-v2/tokenizer.json b/assets/all-MiniLM-L3-v2/tokenizer.json new file mode 100644 index 0000000..13a88ab --- /dev/null +++ b/assets/all-MiniLM-L3-v2/tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":{"max_length":128,"strategy":"LongestFirst","stride":0},"padding":{"strategy":"BatchLongest","direction":"Right","pad_to_multiple_of":null,"pad_id":0,"pad_type_id":0,"pad_token":"[PAD]"},"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":100,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":101,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":102,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":103,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[101],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[102],"tokens":["[SEP]"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"type":"WordPiece","unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[PAD]":0,"[unused0]":1,"[unused1]":2,"[unused2]":3,"[unused3]":4,"[unused4]":5,"[unused5]":6,"[unused6]":7,"[unused7]":8,"[unused8]":9,"[unused9]":10,"[unused10]":11,"[unused11]":12,"[unused12]":13,"[unused13]":14,"[unused14]":15,"[unused15]":16,"[unused16]":17,"[unused17]":18,"[unused18]":19,"[unused19]":20,"[unused20]":21,"[unused21]":22,"[unused22]":23,"[unused23]":24,"[unused24]":25,"[unused25]":26,"[unused26]":27,"[unused27]":28,"[unused28]":29,"[unused29]":30,"[unused30]":31,"[unused31]":32,"[unused32]":33,"[unused33]":34,"[unused34]":35,"[unused35]":36,"[unused36]":37,"[unused37]":38,"[unused38]":39,"[unused39]":40,"[unused40]":41,"[unused41]":42,"[unused42]":43,"[unused43]":44,"[unused44]":45,"[unused45]":46,"[unused46]":47,"[unused47]":48,"[unused48]":49,"[unused49]":50,"[unused50]":51,"[unused51]":52,"[unused52]":53,"[unused53]":54,"[unused54]":55,"[unused55]":56,"[unused56]":57,"[unused57]":58,"[unused58]":59,"[unused59]":60,"[unused60]":61,"[unused61]":62,"[unused62]":63,"[unused63]":64,"[unused64]":65,"[unused65]":66,"[unused66]":67,"[unused67]":68,"[unused68]":69,"[unused69]":70,"[unused70]":71,"[unused71]":72,"[unused72]":73,"[unused73]":74,"[unused74]":75,"[unused75]":76,"[unused76]":77,"[unused77]":78,"[unused78]":79,"[unused79]":80,"[unused80]":81,"[unused81]":82,"[unused82]":83,"[unused83]":84,"[unused84]":85,"[unused85]":86,"[unused86]":87,"[unused87]":88,"[unused88]":89,"[unused89]":90,"[unused90]":91,"[unused91]":92,"[unused92]":93,"[unused93]":94,"[unused94]":95,"[unused95]":96,"[unused96]":97,"[unused97]":98,"[unused98]":99,"[UNK]":100,"[CLS]":101,"[SEP]":102,"[MASK]":103,"[unused99]":104,"[unused100]":105,"[unused101]":106,"[unused102]":107,"[unused103]":108,"[unused104]":109,"[unused105]":110,"[unused106]":111,"[unused107]":112,"[unused108]":113,"[unused109]":114,"[unused110]":115,"[unused111]":116,"[unused112]":117,"[unused113]":118,"[unused114]":119,"[unused115]":120,"[unused116]":121,"[unused117]":122,"[unused118]":123,"[unused119]":124,"[unused120]":125,"[unused121]":126,"[unused122]":127,"[unused123]":128,"[unused124]":129,"[unused125]":130,"[unused126]":131,"[unused127]":132,"[unused128]":133,"[unused129]":134,"[unused130]":135,"[unused131]":136,"[unused132]":137,"[unused133]":138,"[unused134]":139,"[unused135]":140,"[unused136]":141,"[unused137]":142,"[unused138]":143,"[unused139]":144,"[unused140]":145,"[unused141]":146,"[unused142]":147,"[unused143]":148,"[unused144]":149,"[unused145]":150,"[unused146]":151,"[unused147]":152,"[unused148]":153,"[unused149]":154,"[unused150]":155,"[unused151]":156,"[unused152]":157,"[unused153]":158,"[unused154]":159,"[unused155]":160,"[unused156]":161,"[unused157]":162,"[unused158]":163,"[unused159]":164,"[unused160]":165,"[unused161]":166,"[unused162]":167,"[unused163]":168,"[unused164]":169,"[unused165]":170,"[unused166]":171,"[unused167]":172,"[unused168]":173,"[unused169]":174,"[unused170]":175,"[unused171]":176,"[unused172]":177,"[unused173]":178,"[unused174]":179,"[unused175]":180,"[unused176]":181,"[unused177]":182,"[unused178]":183,"[unused179]":184,"[unused180]":185,"[unused181]":186,"[unused182]":187,"[unused183]":188,"[unused184]":189,"[unused185]":190,"[unused186]":191,"[unused187]":192,"[unused188]":193,"[unused189]":194,"[unused190]":195,"[unused191]":196,"[unused192]":197,"[unused193]":198,"[unused194]":199,"[unused195]":200,"[unused196]":201,"[unused197]":202,"[unused198]":203,"[unused199]":204,"[unused200]":205,"[unused201]":206,"[unused202]":207,"[unused203]":208,"[unused204]":209,"[unused205]":210,"[unused206]":211,"[unused207]":212,"[unused208]":213,"[unused209]":214,"[unused210]":215,"[unused211]":216,"[unused212]":217,"[unused213]":218,"[unused214]":219,"[unused215]":220,"[unused216]":221,"[unused217]":222,"[unused218]":223,"[unused219]":224,"[unused220]":225,"[unused221]":226,"[unused222]":227,"[unused223]":228,"[unused224]":229,"[unused225]":230,"[unused226]":231,"[unused227]":232,"[unused228]":233,"[unused229]":234,"[unused230]":235,"[unused231]":236,"[unused232]":237,"[unused233]":238,"[unused234]":239,"[unused235]":240,"[unused236]":241,"[unused237]":242,"[unused238]":243,"[unused239]":244,"[unused240]":245,"[unused241]":246,"[unused242]":247,"[unused243]":248,"[unused244]":249,"[unused245]":250,"[unused246]":251,"[unused247]":252,"[unused248]":253,"[unused249]":254,"[unused250]":255,"[unused251]":256,"[unused252]":257,"[unused253]":258,"[unused254]":259,"[unused255]":260,"[unused256]":261,"[unused257]":262,"[unused258]":263,"[unused259]":264,"[unused260]":265,"[unused261]":266,"[unused262]":267,"[unused263]":268,"[unused264]":269,"[unused265]":270,"[unused266]":271,"[unused267]":272,"[unused268]":273,"[unused269]":274,"[unused270]":275,"[unused271]":276,"[unused272]":277,"[unused273]":278,"[unused274]":279,"[unused275]":280,"[unused276]":281,"[unused277]":282,"[unused278]":283,"[unused279]":284,"[unused280]":285,"[unused281]":286,"[unused282]":287,"[unused283]":288,"[unused284]":289,"[unused285]":290,"[unused286]":291,"[unused287]":292,"[unused288]":293,"[unused289]":294,"[unused290]":295,"[unused291]":296,"[unused292]":297,"[unused293]":298,"[unused294]":299,"[unused295]":300,"[unused296]":301,"[unused297]":302,"[unused298]":303,"[unused299]":304,"[unused300]":305,"[unused301]":306,"[unused302]":307,"[unused303]":308,"[unused304]":309,"[unused305]":310,"[unused306]":311,"[unused307]":312,"[unused308]":313,"[unused309]":314,"[unused310]":315,"[unused311]":316,"[unused312]":317,"[unused313]":318,"[unused314]":319,"[unused315]":320,"[unused316]":321,"[unused317]":322,"[unused318]":323,"[unused319]":324,"[unused320]":325,"[unused321]":326,"[unused322]":327,"[unused323]":328,"[unused324]":329,"[unused325]":330,"[unused326]":331,"[unused327]":332,"[unused328]":333,"[unused329]":334,"[unused330]":335,"[unused331]":336,"[unused332]":337,"[unused333]":338,"[unused334]":339,"[unused335]":340,"[unused336]":341,"[unused337]":342,"[unused338]":343,"[unused339]":344,"[unused340]":345,"[unused341]":346,"[unused342]":347,"[unused343]":348,"[unused344]":349,"[unused345]":350,"[unused346]":351,"[unused347]":352,"[unused348]":353,"[unused349]":354,"[unused350]":355,"[unused351]":356,"[unused352]":357,"[unused353]":358,"[unused354]":359,"[unused355]":360,"[unused356]":361,"[unused357]":362,"[unused358]":363,"[unused359]":364,"[unused360]":365,"[unused361]":366,"[unused362]":367,"[unused363]":368,"[unused364]":369,"[unused365]":370,"[unused366]":371,"[unused367]":372,"[unused368]":373,"[unused369]":374,"[unused370]":375,"[unused371]":376,"[unused372]":377,"[unused373]":378,"[unused374]":379,"[unused375]":380,"[unused376]":381,"[unused377]":382,"[unused378]":383,"[unused379]":384,"[unused380]":385,"[unused381]":386,"[unused382]":387,"[unused383]":388,"[unused384]":389,"[unused385]":390,"[unused386]":391,"[unused387]":392,"[unused388]":393,"[unused389]":394,"[unused390]":395,"[unused391]":396,"[unused392]":397,"[unused393]":398,"[unused394]":399,"[unused395]":400,"[unused396]":401,"[unused397]":402,"[unused398]":403,"[unused399]":404,"[unused400]":405,"[unused401]":406,"[unused402]":407,"[unused403]":408,"[unused404]":409,"[unused405]":410,"[unused406]":411,"[unused407]":412,"[unused408]":413,"[unused409]":414,"[unused410]":415,"[unused411]":416,"[unused412]":417,"[unused413]":418,"[unused414]":419,"[unused415]":420,"[unused416]":421,"[unused417]":422,"[unused418]":423,"[unused419]":424,"[unused420]":425,"[unused421]":426,"[unused422]":427,"[unused423]":428,"[unused424]":429,"[unused425]":430,"[unused426]":431,"[unused427]":432,"[unused428]":433,"[unused429]":434,"[unused430]":435,"[unused431]":436,"[unused432]":437,"[unused433]":438,"[unused434]":439,"[unused435]":440,"[unused436]":441,"[unused437]":442,"[unused438]":443,"[unused439]":444,"[unused440]":445,"[unused441]":446,"[unused442]":447,"[unused443]":448,"[unused444]":449,"[unused445]":450,"[unused446]":451,"[unused447]":452,"[unused448]":453,"[unused449]":454,"[unused450]":455,"[unused451]":456,"[unused452]":457,"[unused453]":458,"[unused454]":459,"[unused455]":460,"[unused456]":461,"[unused457]":462,"[unused458]":463,"[unused459]":464,"[unused460]":465,"[unused461]":466,"[unused462]":467,"[unused463]":468,"[unused464]":469,"[unused465]":470,"[unused466]":471,"[unused467]":472,"[unused468]":473,"[unused469]":474,"[unused470]":475,"[unused471]":476,"[unused472]":477,"[unused473]":478,"[unused474]":479,"[unused475]":480,"[unused476]":481,"[unused477]":482,"[unused478]":483,"[unused479]":484,"[unused480]":485,"[unused481]":486,"[unused482]":487,"[unused483]":488,"[unused484]":489,"[unused485]":490,"[unused486]":491,"[unused487]":492,"[unused488]":493,"[unused489]":494,"[unused490]":495,"[unused491]":496,"[unused492]":497,"[unused493]":498,"[unused494]":499,"[unused495]":500,"[unused496]":501,"[unused497]":502,"[unused498]":503,"[unused499]":504,"[unused500]":505,"[unused501]":506,"[unused502]":507,"[unused503]":508,"[unused504]":509,"[unused505]":510,"[unused506]":511,"[unused507]":512,"[unused508]":513,"[unused509]":514,"[unused510]":515,"[unused511]":516,"[unused512]":517,"[unused513]":518,"[unused514]":519,"[unused515]":520,"[unused516]":521,"[unused517]":522,"[unused518]":523,"[unused519]":524,"[unused520]":525,"[unused521]":526,"[unused522]":527,"[unused523]":528,"[unused524]":529,"[unused525]":530,"[unused526]":531,"[unused527]":532,"[unused528]":533,"[unused529]":534,"[unused530]":535,"[unused531]":536,"[unused532]":537,"[unused533]":538,"[unused534]":539,"[unused535]":540,"[unused536]":541,"[unused537]":542,"[unused538]":543,"[unused539]":544,"[unused540]":545,"[unused541]":546,"[unused542]":547,"[unused543]":548,"[unused544]":549,"[unused545]":550,"[unused546]":551,"[unused547]":552,"[unused548]":553,"[unused549]":554,"[unused550]":555,"[unused551]":556,"[unused552]":557,"[unused553]":558,"[unused554]":559,"[unused555]":560,"[unused556]":561,"[unused557]":562,"[unused558]":563,"[unused559]":564,"[unused560]":565,"[unused561]":566,"[unused562]":567,"[unused563]":568,"[unused564]":569,"[unused565]":570,"[unused566]":571,"[unused567]":572,"[unused568]":573,"[unused569]":574,"[unused570]":575,"[unused571]":576,"[unused572]":577,"[unused573]":578,"[unused574]":579,"[unused575]":580,"[unused576]":581,"[unused577]":582,"[unused578]":583,"[unused579]":584,"[unused580]":585,"[unused581]":586,"[unused582]":587,"[unused583]":588,"[unused584]":589,"[unused585]":590,"[unused586]":591,"[unused587]":592,"[unused588]":593,"[unused589]":594,"[unused590]":595,"[unused591]":596,"[unused592]":597,"[unused593]":598,"[unused594]":599,"[unused595]":600,"[unused596]":601,"[unused597]":602,"[unused598]":603,"[unused599]":604,"[unused600]":605,"[unused601]":606,"[unused602]":607,"[unused603]":608,"[unused604]":609,"[unused605]":610,"[unused606]":611,"[unused607]":612,"[unused608]":613,"[unused609]":614,"[unused610]":615,"[unused611]":616,"[unused612]":617,"[unused613]":618,"[unused614]":619,"[unused615]":620,"[unused616]":621,"[unused617]":622,"[unused618]":623,"[unused619]":624,"[unused620]":625,"[unused621]":626,"[unused622]":627,"[unused623]":628,"[unused624]":629,"[unused625]":630,"[unused626]":631,"[unused627]":632,"[unused628]":633,"[unused629]":634,"[unused630]":635,"[unused631]":636,"[unused632]":637,"[unused633]":638,"[unused634]":639,"[unused635]":640,"[unused636]":641,"[unused637]":642,"[unused638]":643,"[unused639]":644,"[unused640]":645,"[unused641]":646,"[unused642]":647,"[unused643]":648,"[unused644]":649,"[unused645]":650,"[unused646]":651,"[unused647]":652,"[unused648]":653,"[unused649]":654,"[unused650]":655,"[unused651]":656,"[unused652]":657,"[unused653]":658,"[unused654]":659,"[unused655]":660,"[unused656]":661,"[unused657]":662,"[unused658]":663,"[unused659]":664,"[unused660]":665,"[unused661]":666,"[unused662]":667,"[unused663]":668,"[unused664]":669,"[unused665]":670,"[unused666]":671,"[unused667]":672,"[unused668]":673,"[unused669]":674,"[unused670]":675,"[unused671]":676,"[unused672]":677,"[unused673]":678,"[unused674]":679,"[unused675]":680,"[unused676]":681,"[unused677]":682,"[unused678]":683,"[unused679]":684,"[unused680]":685,"[unused681]":686,"[unused682]":687,"[unused683]":688,"[unused684]":689,"[unused685]":690,"[unused686]":691,"[unused687]":692,"[unused688]":693,"[unused689]":694,"[unused690]":695,"[unused691]":696,"[unused692]":697,"[unused693]":698,"[unused694]":699,"[unused695]":700,"[unused696]":701,"[unused697]":702,"[unused698]":703,"[unused699]":704,"[unused700]":705,"[unused701]":706,"[unused702]":707,"[unused703]":708,"[unused704]":709,"[unused705]":710,"[unused706]":711,"[unused707]":712,"[unused708]":713,"[unused709]":714,"[unused710]":715,"[unused711]":716,"[unused712]":717,"[unused713]":718,"[unused714]":719,"[unused715]":720,"[unused716]":721,"[unused717]":722,"[unused718]":723,"[unused719]":724,"[unused720]":725,"[unused721]":726,"[unused722]":727,"[unused723]":728,"[unused724]":729,"[unused725]":730,"[unused726]":731,"[unused727]":732,"[unused728]":733,"[unused729]":734,"[unused730]":735,"[unused731]":736,"[unused732]":737,"[unused733]":738,"[unused734]":739,"[unused735]":740,"[unused736]":741,"[unused737]":742,"[unused738]":743,"[unused739]":744,"[unused740]":745,"[unused741]":746,"[unused742]":747,"[unused743]":748,"[unused744]":749,"[unused745]":750,"[unused746]":751,"[unused747]":752,"[unused748]":753,"[unused749]":754,"[unused750]":755,"[unused751]":756,"[unused752]":757,"[unused753]":758,"[unused754]":759,"[unused755]":760,"[unused756]":761,"[unused757]":762,"[unused758]":763,"[unused759]":764,"[unused760]":765,"[unused761]":766,"[unused762]":767,"[unused763]":768,"[unused764]":769,"[unused765]":770,"[unused766]":771,"[unused767]":772,"[unused768]":773,"[unused769]":774,"[unused770]":775,"[unused771]":776,"[unused772]":777,"[unused773]":778,"[unused774]":779,"[unused775]":780,"[unused776]":781,"[unused777]":782,"[unused778]":783,"[unused779]":784,"[unused780]":785,"[unused781]":786,"[unused782]":787,"[unused783]":788,"[unused784]":789,"[unused785]":790,"[unused786]":791,"[unused787]":792,"[unused788]":793,"[unused789]":794,"[unused790]":795,"[unused791]":796,"[unused792]":797,"[unused793]":798,"[unused794]":799,"[unused795]":800,"[unused796]":801,"[unused797]":802,"[unused798]":803,"[unused799]":804,"[unused800]":805,"[unused801]":806,"[unused802]":807,"[unused803]":808,"[unused804]":809,"[unused805]":810,"[unused806]":811,"[unused807]":812,"[unused808]":813,"[unused809]":814,"[unused810]":815,"[unused811]":816,"[unused812]":817,"[unused813]":818,"[unused814]":819,"[unused815]":820,"[unused816]":821,"[unused817]":822,"[unused818]":823,"[unused819]":824,"[unused820]":825,"[unused821]":826,"[unused822]":827,"[unused823]":828,"[unused824]":829,"[unused825]":830,"[unused826]":831,"[unused827]":832,"[unused828]":833,"[unused829]":834,"[unused830]":835,"[unused831]":836,"[unused832]":837,"[unused833]":838,"[unused834]":839,"[unused835]":840,"[unused836]":841,"[unused837]":842,"[unused838]":843,"[unused839]":844,"[unused840]":845,"[unused841]":846,"[unused842]":847,"[unused843]":848,"[unused844]":849,"[unused845]":850,"[unused846]":851,"[unused847]":852,"[unused848]":853,"[unused849]":854,"[unused850]":855,"[unused851]":856,"[unused852]":857,"[unused853]":858,"[unused854]":859,"[unused855]":860,"[unused856]":861,"[unused857]":862,"[unused858]":863,"[unused859]":864,"[unused860]":865,"[unused861]":866,"[unused862]":867,"[unused863]":868,"[unused864]":869,"[unused865]":870,"[unused866]":871,"[unused867]":872,"[unused868]":873,"[unused869]":874,"[unused870]":875,"[unused871]":876,"[unused872]":877,"[unused873]":878,"[unused874]":879,"[unused875]":880,"[unused876]":881,"[unused877]":882,"[unused878]":883,"[unused879]":884,"[unused880]":885,"[unused881]":886,"[unused882]":887,"[unused883]":888,"[unused884]":889,"[unused885]":890,"[unused886]":891,"[unused887]":892,"[unused888]":893,"[unused889]":894,"[unused890]":895,"[unused891]":896,"[unused892]":897,"[unused893]":898,"[unused894]":899,"[unused895]":900,"[unused896]":901,"[unused897]":902,"[unused898]":903,"[unused899]":904,"[unused900]":905,"[unused901]":906,"[unused902]":907,"[unused903]":908,"[unused904]":909,"[unused905]":910,"[unused906]":911,"[unused907]":912,"[unused908]":913,"[unused909]":914,"[unused910]":915,"[unused911]":916,"[unused912]":917,"[unused913]":918,"[unused914]":919,"[unused915]":920,"[unused916]":921,"[unused917]":922,"[unused918]":923,"[unused919]":924,"[unused920]":925,"[unused921]":926,"[unused922]":927,"[unused923]":928,"[unused924]":929,"[unused925]":930,"[unused926]":931,"[unused927]":932,"[unused928]":933,"[unused929]":934,"[unused930]":935,"[unused931]":936,"[unused932]":937,"[unused933]":938,"[unused934]":939,"[unused935]":940,"[unused936]":941,"[unused937]":942,"[unused938]":943,"[unused939]":944,"[unused940]":945,"[unused941]":946,"[unused942]":947,"[unused943]":948,"[unused944]":949,"[unused945]":950,"[unused946]":951,"[unused947]":952,"[unused948]":953,"[unused949]":954,"[unused950]":955,"[unused951]":956,"[unused952]":957,"[unused953]":958,"[unused954]":959,"[unused955]":960,"[unused956]":961,"[unused957]":962,"[unused958]":963,"[unused959]":964,"[unused960]":965,"[unused961]":966,"[unused962]":967,"[unused963]":968,"[unused964]":969,"[unused965]":970,"[unused966]":971,"[unused967]":972,"[unused968]":973,"[unused969]":974,"[unused970]":975,"[unused971]":976,"[unused972]":977,"[unused973]":978,"[unused974]":979,"[unused975]":980,"[unused976]":981,"[unused977]":982,"[unused978]":983,"[unused979]":984,"[unused980]":985,"[unused981]":986,"[unused982]":987,"[unused983]":988,"[unused984]":989,"[unused985]":990,"[unused986]":991,"[unused987]":992,"[unused988]":993,"[unused989]":994,"[unused990]":995,"[unused991]":996,"[unused992]":997,"[unused993]":998,"!":999,"\"":1000,"#":1001,"$":1002,"%":1003,"&":1004,"'":1005,"(":1006,")":1007,"*":1008,"+":1009,",":1010,"-":1011,".":1012,"/":1013,"0":1014,"1":1015,"2":1016,"3":1017,"4":1018,"5":1019,"6":1020,"7":1021,"8":1022,"9":1023,":":1024,";":1025,"<":1026,"=":1027,">":1028,"?":1029,"@":1030,"[":1031,"\\":1032,"]":1033,"^":1034,"_":1035,"`":1036,"a":1037,"b":1038,"c":1039,"d":1040,"e":1041,"f":1042,"g":1043,"h":1044,"i":1045,"j":1046,"k":1047,"l":1048,"m":1049,"n":1050,"o":1051,"p":1052,"q":1053,"r":1054,"s":1055,"t":1056,"u":1057,"v":1058,"w":1059,"x":1060,"y":1061,"z":1062,"{":1063,"|":1064,"}":1065,"~":1066,"¡":1067,"¢":1068,"£":1069,"¤":1070,"¥":1071,"¦":1072,"§":1073,"¨":1074,"©":1075,"ª":1076,"«":1077,"¬":1078,"®":1079,"°":1080,"±":1081,"²":1082,"³":1083,"´":1084,"µ":1085,"¶":1086,"·":1087,"¹":1088,"º":1089,"»":1090,"¼":1091,"½":1092,"¾":1093,"¿":1094,"×":1095,"ß":1096,"æ":1097,"ð":1098,"÷":1099,"ø":1100,"þ":1101,"đ":1102,"ħ":1103,"ı":1104,"ł":1105,"ŋ":1106,"œ":1107,"ƒ":1108,"ɐ":1109,"ɑ":1110,"ɒ":1111,"ɔ":1112,"ɕ":1113,"ə":1114,"ɛ":1115,"ɡ":1116,"ɣ":1117,"ɨ":1118,"ɪ":1119,"ɫ":1120,"ɬ":1121,"ɯ":1122,"ɲ":1123,"ɴ":1124,"ɹ":1125,"ɾ":1126,"ʀ":1127,"ʁ":1128,"ʂ":1129,"ʃ":1130,"ʉ":1131,"ʊ":1132,"ʋ":1133,"ʌ":1134,"ʎ":1135,"ʐ":1136,"ʑ":1137,"ʒ":1138,"ʔ":1139,"ʰ":1140,"ʲ":1141,"ʳ":1142,"ʷ":1143,"ʸ":1144,"ʻ":1145,"ʼ":1146,"ʾ":1147,"ʿ":1148,"ˈ":1149,"ː":1150,"ˡ":1151,"ˢ":1152,"ˣ":1153,"ˤ":1154,"α":1155,"β":1156,"γ":1157,"δ":1158,"ε":1159,"ζ":1160,"η":1161,"θ":1162,"ι":1163,"κ":1164,"λ":1165,"μ":1166,"ν":1167,"ξ":1168,"ο":1169,"π":1170,"ρ":1171,"ς":1172,"σ":1173,"τ":1174,"υ":1175,"φ":1176,"χ":1177,"ψ":1178,"ω":1179,"а":1180,"б":1181,"в":1182,"г":1183,"д":1184,"е":1185,"ж":1186,"з":1187,"и":1188,"к":1189,"л":1190,"м":1191,"н":1192,"о":1193,"п":1194,"р":1195,"с":1196,"т":1197,"у":1198,"ф":1199,"х":1200,"ц":1201,"ч":1202,"ш":1203,"щ":1204,"ъ":1205,"ы":1206,"ь":1207,"э":1208,"ю":1209,"я":1210,"ђ":1211,"є":1212,"і":1213,"ј":1214,"љ":1215,"њ":1216,"ћ":1217,"ӏ":1218,"ա":1219,"բ":1220,"գ":1221,"դ":1222,"ե":1223,"թ":1224,"ի":1225,"լ":1226,"կ":1227,"հ":1228,"մ":1229,"յ":1230,"ն":1231,"ո":1232,"պ":1233,"ս":1234,"վ":1235,"տ":1236,"ր":1237,"ւ":1238,"ք":1239,"־":1240,"א":1241,"ב":1242,"ג":1243,"ד":1244,"ה":1245,"ו":1246,"ז":1247,"ח":1248,"ט":1249,"י":1250,"ך":1251,"כ":1252,"ל":1253,"ם":1254,"מ":1255,"ן":1256,"נ":1257,"ס":1258,"ע":1259,"ף":1260,"פ":1261,"ץ":1262,"צ":1263,"ק":1264,"ר":1265,"ש":1266,"ת":1267,"،":1268,"ء":1269,"ا":1270,"ب":1271,"ة":1272,"ت":1273,"ث":1274,"ج":1275,"ح":1276,"خ":1277,"د":1278,"ذ":1279,"ر":1280,"ز":1281,"س":1282,"ش":1283,"ص":1284,"ض":1285,"ط":1286,"ظ":1287,"ع":1288,"غ":1289,"ـ":1290,"ف":1291,"ق":1292,"ك":1293,"ل":1294,"م":1295,"ن":1296,"ه":1297,"و":1298,"ى":1299,"ي":1300,"ٹ":1301,"پ":1302,"چ":1303,"ک":1304,"گ":1305,"ں":1306,"ھ":1307,"ہ":1308,"ی":1309,"ے":1310,"अ":1311,"आ":1312,"उ":1313,"ए":1314,"क":1315,"ख":1316,"ग":1317,"च":1318,"ज":1319,"ट":1320,"ड":1321,"ण":1322,"त":1323,"थ":1324,"द":1325,"ध":1326,"न":1327,"प":1328,"ब":1329,"भ":1330,"म":1331,"य":1332,"र":1333,"ल":1334,"व":1335,"श":1336,"ष":1337,"स":1338,"ह":1339,"ा":1340,"ि":1341,"ी":1342,"ो":1343,"।":1344,"॥":1345,"ং":1346,"অ":1347,"আ":1348,"ই":1349,"উ":1350,"এ":1351,"ও":1352,"ক":1353,"খ":1354,"গ":1355,"চ":1356,"ছ":1357,"জ":1358,"ট":1359,"ড":1360,"ণ":1361,"ত":1362,"থ":1363,"দ":1364,"ধ":1365,"ন":1366,"প":1367,"ব":1368,"ভ":1369,"ম":1370,"য":1371,"র":1372,"ল":1373,"শ":1374,"ষ":1375,"স":1376,"হ":1377,"া":1378,"ি":1379,"ী":1380,"ে":1381,"க":1382,"ச":1383,"ட":1384,"த":1385,"ந":1386,"ன":1387,"ப":1388,"ம":1389,"ய":1390,"ர":1391,"ல":1392,"ள":1393,"வ":1394,"ா":1395,"ி":1396,"ு":1397,"ே":1398,"ை":1399,"ನ":1400,"ರ":1401,"ಾ":1402,"ක":1403,"ය":1404,"ර":1405,"ල":1406,"ව":1407,"ා":1408,"ก":1409,"ง":1410,"ต":1411,"ท":1412,"น":1413,"พ":1414,"ม":1415,"ย":1416,"ร":1417,"ล":1418,"ว":1419,"ส":1420,"อ":1421,"า":1422,"เ":1423,"་":1424,"།":1425,"ག":1426,"ང":1427,"ད":1428,"ན":1429,"པ":1430,"བ":1431,"མ":1432,"འ":1433,"ར":1434,"ལ":1435,"ས":1436,"မ":1437,"ა":1438,"ბ":1439,"გ":1440,"დ":1441,"ე":1442,"ვ":1443,"თ":1444,"ი":1445,"კ":1446,"ლ":1447,"მ":1448,"ნ":1449,"ო":1450,"რ":1451,"ს":1452,"ტ":1453,"უ":1454,"ᄀ":1455,"ᄂ":1456,"ᄃ":1457,"ᄅ":1458,"ᄆ":1459,"ᄇ":1460,"ᄉ":1461,"ᄊ":1462,"ᄋ":1463,"ᄌ":1464,"ᄎ":1465,"ᄏ":1466,"ᄐ":1467,"ᄑ":1468,"ᄒ":1469,"ᅡ":1470,"ᅢ":1471,"ᅥ":1472,"ᅦ":1473,"ᅧ":1474,"ᅩ":1475,"ᅪ":1476,"ᅭ":1477,"ᅮ":1478,"ᅯ":1479,"ᅲ":1480,"ᅳ":1481,"ᅴ":1482,"ᅵ":1483,"ᆨ":1484,"ᆫ":1485,"ᆯ":1486,"ᆷ":1487,"ᆸ":1488,"ᆼ":1489,"ᴬ":1490,"ᴮ":1491,"ᴰ":1492,"ᴵ":1493,"ᴺ":1494,"ᵀ":1495,"ᵃ":1496,"ᵇ":1497,"ᵈ":1498,"ᵉ":1499,"ᵍ":1500,"ᵏ":1501,"ᵐ":1502,"ᵒ":1503,"ᵖ":1504,"ᵗ":1505,"ᵘ":1506,"ᵢ":1507,"ᵣ":1508,"ᵤ":1509,"ᵥ":1510,"ᶜ":1511,"ᶠ":1512,"‐":1513,"‑":1514,"‒":1515,"–":1516,"—":1517,"―":1518,"‖":1519,"‘":1520,"’":1521,"‚":1522,"“":1523,"”":1524,"„":1525,"†":1526,"‡":1527,"•":1528,"…":1529,"‰":1530,"′":1531,"″":1532,"›":1533,"‿":1534,"⁄":1535,"⁰":1536,"ⁱ":1537,"⁴":1538,"⁵":1539,"⁶":1540,"⁷":1541,"⁸":1542,"⁹":1543,"⁺":1544,"⁻":1545,"ⁿ":1546,"₀":1547,"₁":1548,"₂":1549,"₃":1550,"₄":1551,"₅":1552,"₆":1553,"₇":1554,"₈":1555,"₉":1556,"₊":1557,"₍":1558,"₎":1559,"ₐ":1560,"ₑ":1561,"ₒ":1562,"ₓ":1563,"ₕ":1564,"ₖ":1565,"ₗ":1566,"ₘ":1567,"ₙ":1568,"ₚ":1569,"ₛ":1570,"ₜ":1571,"₤":1572,"₩":1573,"€":1574,"₱":1575,"₹":1576,"ℓ":1577,"№":1578,"ℝ":1579,"™":1580,"⅓":1581,"⅔":1582,"←":1583,"↑":1584,"→":1585,"↓":1586,"↔":1587,"↦":1588,"⇄":1589,"⇌":1590,"⇒":1591,"∂":1592,"∅":1593,"∆":1594,"∇":1595,"∈":1596,"−":1597,"∗":1598,"∘":1599,"√":1600,"∞":1601,"∧":1602,"∨":1603,"∩":1604,"∪":1605,"≈":1606,"≡":1607,"≤":1608,"≥":1609,"⊂":1610,"⊆":1611,"⊕":1612,"⊗":1613,"⋅":1614,"─":1615,"│":1616,"■":1617,"▪":1618,"●":1619,"★":1620,"☆":1621,"☉":1622,"♠":1623,"♣":1624,"♥":1625,"♦":1626,"♭":1627,"♯":1628,"⟨":1629,"⟩":1630,"ⱼ":1631,"⺩":1632,"⺼":1633,"⽥":1634,"、":1635,"。":1636,"〈":1637,"〉":1638,"《":1639,"》":1640,"「":1641,"」":1642,"『":1643,"』":1644,"〜":1645,"あ":1646,"い":1647,"う":1648,"え":1649,"お":1650,"か":1651,"き":1652,"く":1653,"け":1654,"こ":1655,"さ":1656,"し":1657,"す":1658,"せ":1659,"そ":1660,"た":1661,"ち":1662,"っ":1663,"つ":1664,"て":1665,"と":1666,"な":1667,"に":1668,"ぬ":1669,"ね":1670,"の":1671,"は":1672,"ひ":1673,"ふ":1674,"へ":1675,"ほ":1676,"ま":1677,"み":1678,"む":1679,"め":1680,"も":1681,"や":1682,"ゆ":1683,"よ":1684,"ら":1685,"り":1686,"る":1687,"れ":1688,"ろ":1689,"を":1690,"ん":1691,"ァ":1692,"ア":1693,"ィ":1694,"イ":1695,"ウ":1696,"ェ":1697,"エ":1698,"オ":1699,"カ":1700,"キ":1701,"ク":1702,"ケ":1703,"コ":1704,"サ":1705,"シ":1706,"ス":1707,"セ":1708,"タ":1709,"チ":1710,"ッ":1711,"ツ":1712,"テ":1713,"ト":1714,"ナ":1715,"ニ":1716,"ノ":1717,"ハ":1718,"ヒ":1719,"フ":1720,"ヘ":1721,"ホ":1722,"マ":1723,"ミ":1724,"ム":1725,"メ":1726,"モ":1727,"ャ":1728,"ュ":1729,"ョ":1730,"ラ":1731,"リ":1732,"ル":1733,"レ":1734,"ロ":1735,"ワ":1736,"ン":1737,"・":1738,"ー":1739,"一":1740,"三":1741,"上":1742,"下":1743,"不":1744,"世":1745,"中":1746,"主":1747,"久":1748,"之":1749,"也":1750,"事":1751,"二":1752,"五":1753,"井":1754,"京":1755,"人":1756,"亻":1757,"仁":1758,"介":1759,"代":1760,"仮":1761,"伊":1762,"会":1763,"佐":1764,"侍":1765,"保":1766,"信":1767,"健":1768,"元":1769,"光":1770,"八":1771,"公":1772,"内":1773,"出":1774,"分":1775,"前":1776,"劉":1777,"力":1778,"加":1779,"勝":1780,"北":1781,"区":1782,"十":1783,"千":1784,"南":1785,"博":1786,"原":1787,"口":1788,"古":1789,"史":1790,"司":1791,"合":1792,"吉":1793,"同":1794,"名":1795,"和":1796,"囗":1797,"四":1798,"国":1799,"國":1800,"土":1801,"地":1802,"坂":1803,"城":1804,"堂":1805,"場":1806,"士":1807,"夏":1808,"外":1809,"大":1810,"天":1811,"太":1812,"夫":1813,"奈":1814,"女":1815,"子":1816,"学":1817,"宀":1818,"宇":1819,"安":1820,"宗":1821,"定":1822,"宣":1823,"宮":1824,"家":1825,"宿":1826,"寺":1827,"將":1828,"小":1829,"尚":1830,"山":1831,"岡":1832,"島":1833,"崎":1834,"川":1835,"州":1836,"巿":1837,"帝":1838,"平":1839,"年":1840,"幸":1841,"广":1842,"弘":1843,"張":1844,"彳":1845,"後":1846,"御":1847,"德":1848,"心":1849,"忄":1850,"志":1851,"忠":1852,"愛":1853,"成":1854,"我":1855,"戦":1856,"戸":1857,"手":1858,"扌":1859,"政":1860,"文":1861,"新":1862,"方":1863,"日":1864,"明":1865,"星":1866,"春":1867,"昭":1868,"智":1869,"曲":1870,"書":1871,"月":1872,"有":1873,"朝":1874,"木":1875,"本":1876,"李":1877,"村":1878,"東":1879,"松":1880,"林":1881,"森":1882,"楊":1883,"樹":1884,"橋":1885,"歌":1886,"止":1887,"正":1888,"武":1889,"比":1890,"氏":1891,"民":1892,"水":1893,"氵":1894,"氷":1895,"永":1896,"江":1897,"沢":1898,"河":1899,"治":1900,"法":1901,"海":1902,"清":1903,"漢":1904,"瀬":1905,"火":1906,"版":1907,"犬":1908,"王":1909,"生":1910,"田":1911,"男":1912,"疒":1913,"発":1914,"白":1915,"的":1916,"皇":1917,"目":1918,"相":1919,"省":1920,"真":1921,"石":1922,"示":1923,"社":1924,"神":1925,"福":1926,"禾":1927,"秀":1928,"秋":1929,"空":1930,"立":1931,"章":1932,"竹":1933,"糹":1934,"美":1935,"義":1936,"耳":1937,"良":1938,"艹":1939,"花":1940,"英":1941,"華":1942,"葉":1943,"藤":1944,"行":1945,"街":1946,"西":1947,"見":1948,"訁":1949,"語":1950,"谷":1951,"貝":1952,"貴":1953,"車":1954,"軍":1955,"辶":1956,"道":1957,"郎":1958,"郡":1959,"部":1960,"都":1961,"里":1962,"野":1963,"金":1964,"鈴":1965,"镇":1966,"長":1967,"門":1968,"間":1969,"阝":1970,"阿":1971,"陳":1972,"陽":1973,"雄":1974,"青":1975,"面":1976,"風":1977,"食":1978,"香":1979,"馬":1980,"高":1981,"龍":1982,"龸":1983,"fi":1984,"fl":1985,"！":1986,"（":1987,"）":1988,"，":1989,"－":1990,"．":1991,"／":1992,"：":1993,"？":1994,"～":1995,"the":1996,"of":1997,"and":1998,"in":1999,"to":2000,"was":2001,"he":2002,"is":2003,"as":2004,"for":2005,"on":2006,"with":2007,"that":2008,"it":2009,"his":2010,"by":2011,"at":2012,"from":2013,"her":2014,"##s":2015,"she":2016,"you":2017,"had":2018,"an":2019,"were":2020,"but":2021,"be":2022,"this":2023,"are":2024,"not":2025,"my":2026,"they":2027,"one":2028,"which":2029,"or":2030,"have":2031,"him":2032,"me":2033,"first":2034,"all":2035,"also":2036,"their":2037,"has":2038,"up":2039,"who":2040,"out":2041,"been":2042,"when":2043,"after":2044,"there":2045,"into":2046,"new":2047,"two":2048,"its":2049,"##a":2050,"time":2051,"would":2052,"no":2053,"what":2054,"about":2055,"said":2056,"we":2057,"over":2058,"then":2059,"other":2060,"so":2061,"more":2062,"##e":2063,"can":2064,"if":2065,"like":2066,"back":2067,"them":2068,"only":2069,"some":2070,"could":2071,"##i":2072,"where":2073,"just":2074,"##ing":2075,"during":2076,"before":2077,"##n":2078,"do":2079,"##o":2080,"made":2081,"school":2082,"through":2083,"than":2084,"now":2085,"years":2086,"most":2087,"world":2088,"may":2089,"between":2090,"down":2091,"well":2092,"three":2093,"##d":2094,"year":2095,"while":2096,"will":2097,"##ed":2098,"##r":2099,"##y":2100,"later":2101,"##t":2102,"city":2103,"under":2104,"around":2105,"did":2106,"such":2107,"being":2108,"used":2109,"state":2110,"people":2111,"part":2112,"know":2113,"against":2114,"your":2115,"many":2116,"second":2117,"university":2118,"both":2119,"national":2120,"##er":2121,"these":2122,"don":2123,"known":2124,"off":2125,"way":2126,"until":2127,"re":2128,"how":2129,"even":2130,"get":2131,"head":2132,"...":2133,"didn":2134,"##ly":2135,"team":2136,"american":2137,"because":2138,"de":2139,"##l":2140,"born":2141,"united":2142,"film":2143,"since":2144,"still":2145,"long":2146,"work":2147,"south":2148,"us":2149,"became":2150,"any":2151,"high":2152,"again":2153,"day":2154,"family":2155,"see":2156,"right":2157,"man":2158,"eyes":2159,"house":2160,"season":2161,"war":2162,"states":2163,"including":2164,"took":2165,"life":2166,"north":2167,"same":2168,"each":2169,"called":2170,"name":2171,"much":2172,"place":2173,"however":2174,"go":2175,"four":2176,"group":2177,"another":2178,"found":2179,"won":2180,"area":2181,"here":2182,"going":2183,"10":2184,"away":2185,"series":2186,"left":2187,"home":2188,"music":2189,"best":2190,"make":2191,"hand":2192,"number":2193,"company":2194,"several":2195,"never":2196,"last":2197,"john":2198,"000":2199,"very":2200,"album":2201,"take":2202,"end":2203,"good":2204,"too":2205,"following":2206,"released":2207,"game":2208,"played":2209,"little":2210,"began":2211,"district":2212,"##m":2213,"old":2214,"want":2215,"those":2216,"side":2217,"held":2218,"own":2219,"early":2220,"county":2221,"ll":2222,"league":2223,"use":2224,"west":2225,"##u":2226,"face":2227,"think":2228,"##es":2229,"2010":2230,"government":2231,"##h":2232,"march":2233,"came":2234,"small":2235,"general":2236,"town":2237,"june":2238,"##on":2239,"line":2240,"based":2241,"something":2242,"##k":2243,"september":2244,"thought":2245,"looked":2246,"along":2247,"international":2248,"2011":2249,"air":2250,"july":2251,"club":2252,"went":2253,"january":2254,"october":2255,"our":2256,"august":2257,"april":2258,"york":2259,"12":2260,"few":2261,"2012":2262,"2008":2263,"east":2264,"show":2265,"member":2266,"college":2267,"2009":2268,"father":2269,"public":2270,"##us":2271,"come":2272,"men":2273,"five":2274,"set":2275,"station":2276,"church":2277,"##c":2278,"next":2279,"former":2280,"november":2281,"room":2282,"party":2283,"located":2284,"december":2285,"2013":2286,"age":2287,"got":2288,"2007":2289,"##g":2290,"system":2291,"let":2292,"love":2293,"2006":2294,"though":2295,"every":2296,"2014":2297,"look":2298,"song":2299,"water":2300,"century":2301,"without":2302,"body":2303,"black":2304,"night":2305,"within":2306,"great":2307,"women":2308,"single":2309,"ve":2310,"building":2311,"large":2312,"population":2313,"river":2314,"named":2315,"band":2316,"white":2317,"started":2318,"##an":2319,"once":2320,"15":2321,"20":2322,"should":2323,"18":2324,"2015":2325,"service":2326,"top":2327,"built":2328,"british":2329,"open":2330,"death":2331,"king":2332,"moved":2333,"local":2334,"times":2335,"children":2336,"february":2337,"book":2338,"why":2339,"11":2340,"door":2341,"need":2342,"president":2343,"order":2344,"final":2345,"road":2346,"wasn":2347,"although":2348,"due":2349,"major":2350,"died":2351,"village":2352,"third":2353,"knew":2354,"2016":2355,"asked":2356,"turned":2357,"st":2358,"wanted":2359,"say":2360,"##p":2361,"together":2362,"received":2363,"main":2364,"son":2365,"served":2366,"different":2367,"##en":2368,"behind":2369,"himself":2370,"felt":2371,"members":2372,"power":2373,"football":2374,"law":2375,"voice":2376,"play":2377,"##in":2378,"near":2379,"park":2380,"history":2381,"30":2382,"having":2383,"2005":2384,"16":2385,"##man":2386,"saw":2387,"mother":2388,"##al":2389,"army":2390,"point":2391,"front":2392,"help":2393,"english":2394,"street":2395,"art":2396,"late":2397,"hands":2398,"games":2399,"award":2400,"##ia":2401,"young":2402,"14":2403,"put":2404,"published":2405,"country":2406,"division":2407,"across":2408,"told":2409,"13":2410,"often":2411,"ever":2412,"french":2413,"london":2414,"center":2415,"six":2416,"red":2417,"2017":2418,"led":2419,"days":2420,"include":2421,"light":2422,"25":2423,"find":2424,"tell":2425,"among":2426,"species":2427,"really":2428,"according":2429,"central":2430,"half":2431,"2004":2432,"form":2433,"original":2434,"gave":2435,"office":2436,"making":2437,"enough":2438,"lost":2439,"full":2440,"opened":2441,"must":2442,"included":2443,"live":2444,"given":2445,"german":2446,"player":2447,"run":2448,"business":2449,"woman":2450,"community":2451,"cup":2452,"might":2453,"million":2454,"land":2455,"2000":2456,"court":2457,"development":2458,"17":2459,"short":2460,"round":2461,"ii":2462,"km":2463,"seen":2464,"class":2465,"story":2466,"always":2467,"become":2468,"sure":2469,"research":2470,"almost":2471,"director":2472,"council":2473,"la":2474,"##2":2475,"career":2476,"things":2477,"using":2478,"island":2479,"##z":2480,"couldn":2481,"car":2482,"##is":2483,"24":2484,"close":2485,"force":2486,"##1":2487,"better":2488,"free":2489,"support":2490,"control":2491,"field":2492,"students":2493,"2003":2494,"education":2495,"married":2496,"##b":2497,"nothing":2498,"worked":2499,"others":2500,"record":2501,"big":2502,"inside":2503,"level":2504,"anything":2505,"continued":2506,"give":2507,"james":2508,"##3":2509,"military":2510,"established":2511,"non":2512,"returned":2513,"feel":2514,"does":2515,"title":2516,"written":2517,"thing":2518,"feet":2519,"william":2520,"far":2521,"co":2522,"association":2523,"hard":2524,"already":2525,"2002":2526,"##ra":2527,"championship":2528,"human":2529,"western":2530,"100":2531,"##na":2532,"department":2533,"hall":2534,"role":2535,"various":2536,"production":2537,"21":2538,"19":2539,"heart":2540,"2001":2541,"living":2542,"fire":2543,"version":2544,"##ers":2545,"##f":2546,"television":2547,"royal":2548,"##4":2549,"produced":2550,"working":2551,"act":2552,"case":2553,"society":2554,"region":2555,"present":2556,"radio":2557,"period":2558,"looking":2559,"least":2560,"total":2561,"keep":2562,"england":2563,"wife":2564,"program":2565,"per":2566,"brother":2567,"mind":2568,"special":2569,"22":2570,"##le":2571,"am":2572,"works":2573,"soon":2574,"##6":2575,"political":2576,"george":2577,"services":2578,"taken":2579,"created":2580,"##7":2581,"further":2582,"able":2583,"reached":2584,"david":2585,"union":2586,"joined":2587,"upon":2588,"done":2589,"important":2590,"social":2591,"information":2592,"either":2593,"##ic":2594,"##x":2595,"appeared":2596,"position":2597,"ground":2598,"lead":2599,"rock":2600,"dark":2601,"election":2602,"23":2603,"board":2604,"france":2605,"hair":2606,"course":2607,"arms":2608,"site":2609,"police":2610,"girl":2611,"instead":2612,"real":2613,"sound":2614,"##v":2615,"words":2616,"moment":2617,"##te":2618,"someone":2619,"##8":2620,"summer":2621,"project":2622,"announced":2623,"san":2624,"less":2625,"wrote":2626,"past":2627,"followed":2628,"##5":2629,"blue":2630,"founded":2631,"al":2632,"finally":2633,"india":2634,"taking":2635,"records":2636,"america":2637,"##ne":2638,"1999":2639,"design":2640,"considered":2641,"northern":2642,"god":2643,"stop":2644,"battle":2645,"toward":2646,"european":2647,"outside":2648,"described":2649,"track":2650,"today":2651,"playing":2652,"language":2653,"28":2654,"call":2655,"26":2656,"heard":2657,"professional":2658,"low":2659,"australia":2660,"miles":2661,"california":2662,"win":2663,"yet":2664,"green":2665,"##ie":2666,"trying":2667,"blood":2668,"##ton":2669,"southern":2670,"science":2671,"maybe":2672,"everything":2673,"match":2674,"square":2675,"27":2676,"mouth":2677,"video":2678,"race":2679,"recorded":2680,"leave":2681,"above":2682,"##9":2683,"daughter":2684,"points":2685,"space":2686,"1998":2687,"museum":2688,"change":2689,"middle":2690,"common":2691,"##0":2692,"move":2693,"tv":2694,"post":2695,"##ta":2696,"lake":2697,"seven":2698,"tried":2699,"elected":2700,"closed":2701,"ten":2702,"paul":2703,"minister":2704,"##th":2705,"months":2706,"start":2707,"chief":2708,"return":2709,"canada":2710,"person":2711,"sea":2712,"release":2713,"similar":2714,"modern":2715,"brought":2716,"rest":2717,"hit":2718,"formed":2719,"mr":2720,"##la":2721,"1997":2722,"floor":2723,"event":2724,"doing":2725,"thomas":2726,"1996":2727,"robert":2728,"care":2729,"killed":2730,"training":2731,"star":2732,"week":2733,"needed":2734,"turn":2735,"finished":2736,"railway":2737,"rather":2738,"news":2739,"health":2740,"sent":2741,"example":2742,"ran":2743,"term":2744,"michael":2745,"coming":2746,"currently":2747,"yes":2748,"forces":2749,"despite":2750,"gold":2751,"areas":2752,"50":2753,"stage":2754,"fact":2755,"29":2756,"dead":2757,"says":2758,"popular":2759,"2018":2760,"originally":2761,"germany":2762,"probably":2763,"developed":2764,"result":2765,"pulled":2766,"friend":2767,"stood":2768,"money":2769,"running":2770,"mi":2771,"signed":2772,"word":2773,"songs":2774,"child":2775,"eventually":2776,"met":2777,"tour":2778,"average":2779,"teams":2780,"minutes":2781,"festival":2782,"current":2783,"deep":2784,"kind":2785,"1995":2786,"decided":2787,"usually":2788,"eastern":2789,"seemed":2790,"##ness":2791,"episode":2792,"bed":2793,"added":2794,"table":2795,"indian":2796,"private":2797,"charles":2798,"route":2799,"available":2800,"idea":2801,"throughout":2802,"centre":2803,"addition":2804,"appointed":2805,"style":2806,"1994":2807,"books":2808,"eight":2809,"construction":2810,"press":2811,"mean":2812,"wall":2813,"friends":2814,"remained":2815,"schools":2816,"study":2817,"##ch":2818,"##um":2819,"institute":2820,"oh":2821,"chinese":2822,"sometimes":2823,"events":2824,"possible":2825,"1992":2826,"australian":2827,"type":2828,"brown":2829,"forward":2830,"talk":2831,"process":2832,"food":2833,"debut":2834,"seat":2835,"performance":2836,"committee":2837,"features":2838,"character":2839,"arts":2840,"herself":2841,"else":2842,"lot":2843,"strong":2844,"russian":2845,"range":2846,"hours":2847,"peter":2848,"arm":2849,"##da":2850,"morning":2851,"dr":2852,"sold":2853,"##ry":2854,"quickly":2855,"directed":2856,"1993":2857,"guitar":2858,"china":2859,"##w":2860,"31":2861,"list":2862,"##ma":2863,"performed":2864,"media":2865,"uk":2866,"players":2867,"smile":2868,"##rs":2869,"myself":2870,"40":2871,"placed":2872,"coach":2873,"province":2874,"towards":2875,"wouldn":2876,"leading":2877,"whole":2878,"boy":2879,"official":2880,"designed":2881,"grand":2882,"census":2883,"##el":2884,"europe":2885,"attack":2886,"japanese":2887,"henry":2888,"1991":2889,"##re":2890,"##os":2891,"cross":2892,"getting":2893,"alone":2894,"action":2895,"lower":2896,"network":2897,"wide":2898,"washington":2899,"japan":2900,"1990":2901,"hospital":2902,"believe":2903,"changed":2904,"sister":2905,"##ar":2906,"hold":2907,"gone":2908,"sir":2909,"hadn":2910,"ship":2911,"##ka":2912,"studies":2913,"academy":2914,"shot":2915,"rights":2916,"below":2917,"base":2918,"bad":2919,"involved":2920,"kept":2921,"largest":2922,"##ist":2923,"bank":2924,"future":2925,"especially":2926,"beginning":2927,"mark":2928,"movement":2929,"section":2930,"female":2931,"magazine":2932,"plan":2933,"professor":2934,"lord":2935,"longer":2936,"##ian":2937,"sat":2938,"walked":2939,"hill":2940,"actually":2941,"civil":2942,"energy":2943,"model":2944,"families":2945,"size":2946,"thus":2947,"aircraft":2948,"completed":2949,"includes":2950,"data":2951,"captain":2952,"##or":2953,"fight":2954,"vocals":2955,"featured":2956,"richard":2957,"bridge":2958,"fourth":2959,"1989":2960,"officer":2961,"stone":2962,"hear":2963,"##ism":2964,"means":2965,"medical":2966,"groups":2967,"management":2968,"self":2969,"lips":2970,"competition":2971,"entire":2972,"lived":2973,"technology":2974,"leaving":2975,"federal":2976,"tournament":2977,"bit":2978,"passed":2979,"hot":2980,"independent":2981,"awards":2982,"kingdom":2983,"mary":2984,"spent":2985,"fine":2986,"doesn":2987,"reported":2988,"##ling":2989,"jack":2990,"fall":2991,"raised":2992,"itself":2993,"stay":2994,"true":2995,"studio":2996,"1988":2997,"sports":2998,"replaced":2999,"paris":3000,"systems":3001,"saint":3002,"leader":3003,"theatre":3004,"whose":3005,"market":3006,"capital":3007,"parents":3008,"spanish":3009,"canadian":3010,"earth":3011,"##ity":3012,"cut":3013,"degree":3014,"writing":3015,"bay":3016,"christian":3017,"awarded":3018,"natural":3019,"higher":3020,"bill":3021,"##as":3022,"coast":3023,"provided":3024,"previous":3025,"senior":3026,"ft":3027,"valley":3028,"organization":3029,"stopped":3030,"onto":3031,"countries":3032,"parts":3033,"conference":3034,"queen":3035,"security":3036,"interest":3037,"saying":3038,"allowed":3039,"master":3040,"earlier":3041,"phone":3042,"matter":3043,"smith":3044,"winning":3045,"try":3046,"happened":3047,"moving":3048,"campaign":3049,"los":3050,"##ley":3051,"breath":3052,"nearly":3053,"mid":3054,"1987":3055,"certain":3056,"girls":3057,"date":3058,"italian":3059,"african":3060,"standing":3061,"fell":3062,"artist":3063,"##ted":3064,"shows":3065,"deal":3066,"mine":3067,"industry":3068,"1986":3069,"##ng":3070,"everyone":3071,"republic":3072,"provide":3073,"collection":3074,"library":3075,"student":3076,"##ville":3077,"primary":3078,"owned":3079,"older":3080,"via":3081,"heavy":3082,"1st":3083,"makes":3084,"##able":3085,"attention":3086,"anyone":3087,"africa":3088,"##ri":3089,"stated":3090,"length":3091,"ended":3092,"fingers":3093,"command":3094,"staff":3095,"skin":3096,"foreign":3097,"opening":3098,"governor":3099,"okay":3100,"medal":3101,"kill":3102,"sun":3103,"cover":3104,"job":3105,"1985":3106,"introduced":3107,"chest":3108,"hell":3109,"feeling":3110,"##ies":3111,"success":3112,"meet":3113,"reason":3114,"standard":3115,"meeting":3116,"novel":3117,"1984":3118,"trade":3119,"source":3120,"buildings":3121,"##land":3122,"rose":3123,"guy":3124,"goal":3125,"##ur":3126,"chapter":3127,"native":3128,"husband":3129,"previously":3130,"unit":3131,"limited":3132,"entered":3133,"weeks":3134,"producer":3135,"operations":3136,"mountain":3137,"takes":3138,"covered":3139,"forced":3140,"related":3141,"roman":3142,"complete":3143,"successful":3144,"key":3145,"texas":3146,"cold":3147,"##ya":3148,"channel":3149,"1980":3150,"traditional":3151,"films":3152,"dance":3153,"clear":3154,"approximately":3155,"500":3156,"nine":3157,"van":3158,"prince":3159,"question":3160,"active":3161,"tracks":3162,"ireland":3163,"regional":3164,"silver":3165,"author":3166,"personal":3167,"sense":3168,"operation":3169,"##ine":3170,"economic":3171,"1983":3172,"holding":3173,"twenty":3174,"isbn":3175,"additional":3176,"speed":3177,"hour":3178,"edition":3179,"regular":3180,"historic":3181,"places":3182,"whom":3183,"shook":3184,"movie":3185,"km²":3186,"secretary":3187,"prior":3188,"report":3189,"chicago":3190,"read":3191,"foundation":3192,"view":3193,"engine":3194,"scored":3195,"1982":3196,"units":3197,"ask":3198,"airport":3199,"property":3200,"ready":3201,"immediately":3202,"lady":3203,"month":3204,"listed":3205,"contract":3206,"##de":3207,"manager":3208,"themselves":3209,"lines":3210,"##ki":3211,"navy":3212,"writer":3213,"meant":3214,"##ts":3215,"runs":3216,"##ro":3217,"practice":3218,"championships":3219,"singer":3220,"glass":3221,"commission":3222,"required":3223,"forest":3224,"starting":3225,"culture":3226,"generally":3227,"giving":3228,"access":3229,"attended":3230,"test":3231,"couple":3232,"stand":3233,"catholic":3234,"martin":3235,"caught":3236,"executive":3237,"##less":3238,"eye":3239,"##ey":3240,"thinking":3241,"chair":3242,"quite":3243,"shoulder":3244,"1979":3245,"hope":3246,"decision":3247,"plays":3248,"defeated":3249,"municipality":3250,"whether":3251,"structure":3252,"offered":3253,"slowly":3254,"pain":3255,"ice":3256,"direction":3257,"##ion":3258,"paper":3259,"mission":3260,"1981":3261,"mostly":3262,"200":3263,"noted":3264,"individual":3265,"managed":3266,"nature":3267,"lives":3268,"plant":3269,"##ha":3270,"helped":3271,"except":3272,"studied":3273,"computer":3274,"figure":3275,"relationship":3276,"issue":3277,"significant":3278,"loss":3279,"die":3280,"smiled":3281,"gun":3282,"ago":3283,"highest":3284,"1972":3285,"##am":3286,"male":3287,"bring":3288,"goals":3289,"mexico":3290,"problem":3291,"distance":3292,"commercial":3293,"completely":3294,"location":3295,"annual":3296,"famous":3297,"drive":3298,"1976":3299,"neck":3300,"1978":3301,"surface":3302,"caused":3303,"italy":3304,"understand":3305,"greek":3306,"highway":3307,"wrong":3308,"hotel":3309,"comes":3310,"appearance":3311,"joseph":3312,"double":3313,"issues":3314,"musical":3315,"companies":3316,"castle":3317,"income":3318,"review":3319,"assembly":3320,"bass":3321,"initially":3322,"parliament":3323,"artists":3324,"experience":3325,"1974":3326,"particular":3327,"walk":3328,"foot":3329,"engineering":3330,"talking":3331,"window":3332,"dropped":3333,"##ter":3334,"miss":3335,"baby":3336,"boys":3337,"break":3338,"1975":3339,"stars":3340,"edge":3341,"remember":3342,"policy":3343,"carried":3344,"train":3345,"stadium":3346,"bar":3347,"sex":3348,"angeles":3349,"evidence":3350,"##ge":3351,"becoming":3352,"assistant":3353,"soviet":3354,"1977":3355,"upper":3356,"step":3357,"wing":3358,"1970":3359,"youth":3360,"financial":3361,"reach":3362,"##ll":3363,"actor":3364,"numerous":3365,"##se":3366,"##st":3367,"nodded":3368,"arrived":3369,"##ation":3370,"minute":3371,"##nt":3372,"believed":3373,"sorry":3374,"complex":3375,"beautiful":3376,"victory":3377,"associated":3378,"temple":3379,"1968":3380,"1973":3381,"chance":3382,"perhaps":3383,"metal":3384,"##son":3385,"1945":3386,"bishop":3387,"##et":3388,"lee":3389,"launched":3390,"particularly":3391,"tree":3392,"le":3393,"retired":3394,"subject":3395,"prize":3396,"contains":3397,"yeah":3398,"theory":3399,"empire":3400,"##ce":3401,"suddenly":3402,"waiting":3403,"trust":3404,"recording":3405,"##to":3406,"happy":3407,"terms":3408,"camp":3409,"champion":3410,"1971":3411,"religious":3412,"pass":3413,"zealand":3414,"names":3415,"2nd":3416,"port":3417,"ancient":3418,"tom":3419,"corner":3420,"represented":3421,"watch":3422,"legal":3423,"anti":3424,"justice":3425,"cause":3426,"watched":3427,"brothers":3428,"45":3429,"material":3430,"changes":3431,"simply":3432,"response":3433,"louis":3434,"fast":3435,"##ting":3436,"answer":3437,"60":3438,"historical":3439,"1969":3440,"stories":3441,"straight":3442,"create":3443,"feature":3444,"increased":3445,"rate":3446,"administration":3447,"virginia":3448,"el":3449,"activities":3450,"cultural":3451,"overall":3452,"winner":3453,"programs":3454,"basketball":3455,"legs":3456,"guard":3457,"beyond":3458,"cast":3459,"doctor":3460,"mm":3461,"flight":3462,"results":3463,"remains":3464,"cost":3465,"effect":3466,"winter":3467,"##ble":3468,"larger":3469,"islands":3470,"problems":3471,"chairman":3472,"grew":3473,"commander":3474,"isn":3475,"1967":3476,"pay":3477,"failed":3478,"selected":3479,"hurt":3480,"fort":3481,"box":3482,"regiment":3483,"majority":3484,"journal":3485,"35":3486,"edward":3487,"plans":3488,"##ke":3489,"##ni":3490,"shown":3491,"pretty":3492,"irish":3493,"characters":3494,"directly":3495,"scene":3496,"likely":3497,"operated":3498,"allow":3499,"spring":3500,"##j":3501,"junior":3502,"matches":3503,"looks":3504,"mike":3505,"houses":3506,"fellow":3507,"##tion":3508,"beach":3509,"marriage":3510,"##ham":3511,"##ive":3512,"rules":3513,"oil":3514,"65":3515,"florida":3516,"expected":3517,"nearby":3518,"congress":3519,"sam":3520,"peace":3521,"recent":3522,"iii":3523,"wait":3524,"subsequently":3525,"cell":3526,"##do":3527,"variety":3528,"serving":3529,"agreed":3530,"please":3531,"poor":3532,"joe":3533,"pacific":3534,"attempt":3535,"wood":3536,"democratic":3537,"piece":3538,"prime":3539,"##ca":3540,"rural":3541,"mile":3542,"touch":3543,"appears":3544,"township":3545,"1964":3546,"1966":3547,"soldiers":3548,"##men":3549,"##ized":3550,"1965":3551,"pennsylvania":3552,"closer":3553,"fighting":3554,"claimed":3555,"score":3556,"jones":3557,"physical":3558,"editor":3559,"##ous":3560,"filled":3561,"genus":3562,"specific":3563,"sitting":3564,"super":3565,"mom":3566,"##va":3567,"therefore":3568,"supported":3569,"status":3570,"fear":3571,"cases":3572,"store":3573,"meaning":3574,"wales":3575,"minor":3576,"spain":3577,"tower":3578,"focus":3579,"vice":3580,"frank":3581,"follow":3582,"parish":3583,"separate":3584,"golden":3585,"horse":3586,"fifth":3587,"remaining":3588,"branch":3589,"32":3590,"presented":3591,"stared":3592,"##id":3593,"uses":3594,"secret":3595,"forms":3596,"##co":3597,"baseball":3598,"exactly":3599,"##ck":3600,"choice":3601,"note":3602,"discovered":3603,"travel":3604,"composed":3605,"truth":3606,"russia":3607,"ball":3608,"color":3609,"kiss":3610,"dad":3611,"wind":3612,"continue":3613,"ring":3614,"referred":3615,"numbers":3616,"digital":3617,"greater":3618,"##ns":3619,"metres":3620,"slightly":3621,"direct":3622,"increase":3623,"1960":3624,"responsible":3625,"crew":3626,"rule":3627,"trees":3628,"troops":3629,"##no":3630,"broke":3631,"goes":3632,"individuals":3633,"hundred":3634,"weight":3635,"creek":3636,"sleep":3637,"memory":3638,"defense":3639,"provides":3640,"ordered":3641,"code":3642,"value":3643,"jewish":3644,"windows":3645,"1944":3646,"safe":3647,"judge":3648,"whatever":3649,"corps":3650,"realized":3651,"growing":3652,"pre":3653,"##ga":3654,"cities":3655,"alexander":3656,"gaze":3657,"lies":3658,"spread":3659,"scott":3660,"letter":3661,"showed":3662,"situation":3663,"mayor":3664,"transport":3665,"watching":3666,"workers":3667,"extended":3668,"##li":3669,"expression":3670,"normal":3671,"##ment":3672,"chart":3673,"multiple":3674,"border":3675,"##ba":3676,"host":3677,"##ner":3678,"daily":3679,"mrs":3680,"walls":3681,"piano":3682,"##ko":3683,"heat":3684,"cannot":3685,"##ate":3686,"earned":3687,"products":3688,"drama":3689,"era":3690,"authority":3691,"seasons":3692,"join":3693,"grade":3694,"##io":3695,"sign":3696,"difficult":3697,"machine":3698,"1963":3699,"territory":3700,"mainly":3701,"##wood":3702,"stations":3703,"squadron":3704,"1962":3705,"stepped":3706,"iron":3707,"19th":3708,"##led":3709,"serve":3710,"appear":3711,"sky":3712,"speak":3713,"broken":3714,"charge":3715,"knowledge":3716,"kilometres":3717,"removed":3718,"ships":3719,"article":3720,"campus":3721,"simple":3722,"##ty":3723,"pushed":3724,"britain":3725,"##ve":3726,"leaves":3727,"recently":3728,"cd":3729,"soft":3730,"boston":3731,"latter":3732,"easy":3733,"acquired":3734,"poland":3735,"##sa":3736,"quality":3737,"officers":3738,"presence":3739,"planned":3740,"nations":3741,"mass":3742,"broadcast":3743,"jean":3744,"share":3745,"image":3746,"influence":3747,"wild":3748,"offer":3749,"emperor":3750,"electric":3751,"reading":3752,"headed":3753,"ability":3754,"promoted":3755,"yellow":3756,"ministry":3757,"1942":3758,"throat":3759,"smaller":3760,"politician":3761,"##by":3762,"latin":3763,"spoke":3764,"cars":3765,"williams":3766,"males":3767,"lack":3768,"pop":3769,"80":3770,"##ier":3771,"acting":3772,"seeing":3773,"consists":3774,"##ti":3775,"estate":3776,"1961":3777,"pressure":3778,"johnson":3779,"newspaper":3780,"jr":3781,"chris":3782,"olympics":3783,"online":3784,"conditions":3785,"beat":3786,"elements":3787,"walking":3788,"vote":3789,"##field":3790,"needs":3791,"carolina":3792,"text":3793,"featuring":3794,"global":3795,"block":3796,"shirt":3797,"levels":3798,"francisco":3799,"purpose":3800,"females":3801,"et":3802,"dutch":3803,"duke":3804,"ahead":3805,"gas":3806,"twice":3807,"safety":3808,"serious":3809,"turning":3810,"highly":3811,"lieutenant":3812,"firm":3813,"maria":3814,"amount":3815,"mixed":3816,"daniel":3817,"proposed":3818,"perfect":3819,"agreement":3820,"affairs":3821,"3rd":3822,"seconds":3823,"contemporary":3824,"paid":3825,"1943":3826,"prison":3827,"save":3828,"kitchen":3829,"label":3830,"administrative":3831,"intended":3832,"constructed":3833,"academic":3834,"nice":3835,"teacher":3836,"races":3837,"1956":3838,"formerly":3839,"corporation":3840,"ben":3841,"nation":3842,"issued":3843,"shut":3844,"1958":3845,"drums":3846,"housing":3847,"victoria":3848,"seems":3849,"opera":3850,"1959":3851,"graduated":3852,"function":3853,"von":3854,"mentioned":3855,"picked":3856,"build":3857,"recognized":3858,"shortly":3859,"protection":3860,"picture":3861,"notable":3862,"exchange":3863,"elections":3864,"1980s":3865,"loved":3866,"percent":3867,"racing":3868,"fish":3869,"elizabeth":3870,"garden":3871,"volume":3872,"hockey":3873,"1941":3874,"beside":3875,"settled":3876,"##ford":3877,"1940":3878,"competed":3879,"replied":3880,"drew":3881,"1948":3882,"actress":3883,"marine":3884,"scotland":3885,"steel":3886,"glanced":3887,"farm":3888,"steve":3889,"1957":3890,"risk":3891,"tonight":3892,"positive":3893,"magic":3894,"singles":3895,"effects":3896,"gray":3897,"screen":3898,"dog":3899,"##ja":3900,"residents":3901,"bus":3902,"sides":3903,"none":3904,"secondary":3905,"literature":3906,"polish":3907,"destroyed":3908,"flying":3909,"founder":3910,"households":3911,"1939":3912,"lay":3913,"reserve":3914,"usa":3915,"gallery":3916,"##ler":3917,"1946":3918,"industrial":3919,"younger":3920,"approach":3921,"appearances":3922,"urban":3923,"ones":3924,"1950":3925,"finish":3926,"avenue":3927,"powerful":3928,"fully":3929,"growth":3930,"page":3931,"honor":3932,"jersey":3933,"projects":3934,"advanced":3935,"revealed":3936,"basic":3937,"90":3938,"infantry":3939,"pair":3940,"equipment":3941,"visit":3942,"33":3943,"evening":3944,"search":3945,"grant":3946,"effort":3947,"solo":3948,"treatment":3949,"buried":3950,"republican":3951,"primarily":3952,"bottom":3953,"owner":3954,"1970s":3955,"israel":3956,"gives":3957,"jim":3958,"dream":3959,"bob":3960,"remain":3961,"spot":3962,"70":3963,"notes":3964,"produce":3965,"champions":3966,"contact":3967,"ed":3968,"soul":3969,"accepted":3970,"ways":3971,"del":3972,"##ally":3973,"losing":3974,"split":3975,"price":3976,"capacity":3977,"basis":3978,"trial":3979,"questions":3980,"##ina":3981,"1955":3982,"20th":3983,"guess":3984,"officially":3985,"memorial":3986,"naval":3987,"initial":3988,"##ization":3989,"whispered":3990,"median":3991,"engineer":3992,"##ful":3993,"sydney":3994,"##go":3995,"columbia":3996,"strength":3997,"300":3998,"1952":3999,"tears":4000,"senate":4001,"00":4002,"card":4003,"asian":4004,"agent":4005,"1947":4006,"software":4007,"44":4008,"draw":4009,"warm":4010,"supposed":4011,"com":4012,"pro":4013,"##il":4014,"transferred":4015,"leaned":4016,"##at":4017,"candidate":4018,"escape":4019,"mountains":4020,"asia":4021,"potential":4022,"activity":4023,"entertainment":4024,"seem":4025,"traffic":4026,"jackson":4027,"murder":4028,"36":4029,"slow":4030,"product":4031,"orchestra":4032,"haven":4033,"agency":4034,"bbc":4035,"taught":4036,"website":4037,"comedy":4038,"unable":4039,"storm":4040,"planning":4041,"albums":4042,"rugby":4043,"environment":4044,"scientific":4045,"grabbed":4046,"protect":4047,"##hi":4048,"boat":4049,"typically":4050,"1954":4051,"1953":4052,"damage":4053,"principal":4054,"divided":4055,"dedicated":4056,"mount":4057,"ohio":4058,"##berg":4059,"pick":4060,"fought":4061,"driver":4062,"##der":4063,"empty":4064,"shoulders":4065,"sort":4066,"thank":4067,"berlin":4068,"prominent":4069,"account":4070,"freedom":4071,"necessary":4072,"efforts":4073,"alex":4074,"headquarters":4075,"follows":4076,"alongside":4077,"des":4078,"simon":4079,"andrew":4080,"suggested":4081,"operating":4082,"learning":4083,"steps":4084,"1949":4085,"sweet":4086,"technical":4087,"begin":4088,"easily":4089,"34":4090,"teeth":4091,"speaking":4092,"settlement":4093,"scale":4094,"##sh":4095,"renamed":4096,"ray":4097,"max":4098,"enemy":4099,"semi":4100,"joint":4101,"compared":4102,"##rd":4103,"scottish":4104,"leadership":4105,"analysis":4106,"offers":4107,"georgia":4108,"pieces":4109,"captured":4110,"animal":4111,"deputy":4112,"guest":4113,"organized":4114,"##lin":4115,"tony":4116,"combined":4117,"method":4118,"challenge":4119,"1960s":4120,"huge":4121,"wants":4122,"battalion":4123,"sons":4124,"rise":4125,"crime":4126,"types":4127,"facilities":4128,"telling":4129,"path":4130,"1951":4131,"platform":4132,"sit":4133,"1990s":4134,"##lo":4135,"tells":4136,"assigned":4137,"rich":4138,"pull":4139,"##ot":4140,"commonly":4141,"alive":4142,"##za":4143,"letters":4144,"concept":4145,"conducted":4146,"wearing":4147,"happen":4148,"bought":4149,"becomes":4150,"holy":4151,"gets":4152,"ocean":4153,"defeat":4154,"languages":4155,"purchased":4156,"coffee":4157,"occurred":4158,"titled":4159,"##q":4160,"declared":4161,"applied":4162,"sciences":4163,"concert":4164,"sounds":4165,"jazz":4166,"brain":4167,"##me":4168,"painting":4169,"fleet":4170,"tax":4171,"nick":4172,"##ius":4173,"michigan":4174,"count":4175,"animals":4176,"leaders":4177,"episodes":4178,"##line":4179,"content":4180,"##den":4181,"birth":4182,"##it":4183,"clubs":4184,"64":4185,"palace":4186,"critical":4187,"refused":4188,"fair":4189,"leg":4190,"laughed":4191,"returning":4192,"surrounding":4193,"participated":4194,"formation":4195,"lifted":4196,"pointed":4197,"connected":4198,"rome":4199,"medicine":4200,"laid":4201,"taylor":4202,"santa":4203,"powers":4204,"adam":4205,"tall":4206,"shared":4207,"focused":4208,"knowing":4209,"yards":4210,"entrance":4211,"falls":4212,"##wa":4213,"calling":4214,"##ad":4215,"sources":4216,"chosen":4217,"beneath":4218,"resources":4219,"yard":4220,"##ite":4221,"nominated":4222,"silence":4223,"zone":4224,"defined":4225,"##que":4226,"gained":4227,"thirty":4228,"38":4229,"bodies":4230,"moon":4231,"##ard":4232,"adopted":4233,"christmas":4234,"widely":4235,"register":4236,"apart":4237,"iran":4238,"premier":4239,"serves":4240,"du":4241,"unknown":4242,"parties":4243,"##les":4244,"generation":4245,"##ff":4246,"continues":4247,"quick":4248,"fields":4249,"brigade":4250,"quiet":4251,"teaching":4252,"clothes":4253,"impact":4254,"weapons":4255,"partner":4256,"flat":4257,"theater":4258,"supreme":4259,"1938":4260,"37":4261,"relations":4262,"##tor":4263,"plants":4264,"suffered":4265,"1936":4266,"wilson":4267,"kids":4268,"begins":4269,"##age":4270,"1918":4271,"seats":4272,"armed":4273,"internet":4274,"models":4275,"worth":4276,"laws":4277,"400":4278,"communities":4279,"classes":4280,"background":4281,"knows":4282,"thanks":4283,"quarter":4284,"reaching":4285,"humans":4286,"carry":4287,"killing":4288,"format":4289,"kong":4290,"hong":4291,"setting":4292,"75":4293,"architecture":4294,"disease":4295,"railroad":4296,"inc":4297,"possibly":4298,"wish":4299,"arthur":4300,"thoughts":4301,"harry":4302,"doors":4303,"density":4304,"##di":4305,"crowd":4306,"illinois":4307,"stomach":4308,"tone":4309,"unique":4310,"reports":4311,"anyway":4312,"##ir":4313,"liberal":4314,"der":4315,"vehicle":4316,"thick":4317,"dry":4318,"drug":4319,"faced":4320,"largely":4321,"facility":4322,"theme":4323,"holds":4324,"creation":4325,"strange":4326,"colonel":4327,"##mi":4328,"revolution":4329,"bell":4330,"politics":4331,"turns":4332,"silent":4333,"rail":4334,"relief":4335,"independence":4336,"combat":4337,"shape":4338,"write":4339,"determined":4340,"sales":4341,"learned":4342,"4th":4343,"finger":4344,"oxford":4345,"providing":4346,"1937":4347,"heritage":4348,"fiction":4349,"situated":4350,"designated":4351,"allowing":4352,"distribution":4353,"hosted":4354,"##est":4355,"sight":4356,"interview":4357,"estimated":4358,"reduced":4359,"##ria":4360,"toronto":4361,"footballer":4362,"keeping":4363,"guys":4364,"damn":4365,"claim":4366,"motion":4367,"sport":4368,"sixth":4369,"stayed":4370,"##ze":4371,"en":4372,"rear":4373,"receive":4374,"handed":4375,"twelve":4376,"dress":4377,"audience":4378,"granted":4379,"brazil":4380,"##well":4381,"spirit":4382,"##ated":4383,"noticed":4384,"etc":4385,"olympic":4386,"representative":4387,"eric":4388,"tight":4389,"trouble":4390,"reviews":4391,"drink":4392,"vampire":4393,"missing":4394,"roles":4395,"ranked":4396,"newly":4397,"household":4398,"finals":4399,"wave":4400,"critics":4401,"##ee":4402,"phase":4403,"massachusetts":4404,"pilot":4405,"unlike":4406,"philadelphia":4407,"bright":4408,"guns":4409,"crown":4410,"organizations":4411,"roof":4412,"42":4413,"respectively":4414,"clearly":4415,"tongue":4416,"marked":4417,"circle":4418,"fox":4419,"korea":4420,"bronze":4421,"brian":4422,"expanded":4423,"sexual":4424,"supply":4425,"yourself":4426,"inspired":4427,"labour":4428,"fc":4429,"##ah":4430,"reference":4431,"vision":4432,"draft":4433,"connection":4434,"brand":4435,"reasons":4436,"1935":4437,"classic":4438,"driving":4439,"trip":4440,"jesus":4441,"cells":4442,"entry":4443,"1920":4444,"neither":4445,"trail":4446,"claims":4447,"atlantic":4448,"orders":4449,"labor":4450,"nose":4451,"afraid":4452,"identified":4453,"intelligence":4454,"calls":4455,"cancer":4456,"attacked":4457,"passing":4458,"stephen":4459,"positions":4460,"imperial":4461,"grey":4462,"jason":4463,"39":4464,"sunday":4465,"48":4466,"swedish":4467,"avoid":4468,"extra":4469,"uncle":4470,"message":4471,"covers":4472,"allows":4473,"surprise":4474,"materials":4475,"fame":4476,"hunter":4477,"##ji":4478,"1930":4479,"citizens":4480,"figures":4481,"davis":4482,"environmental":4483,"confirmed":4484,"shit":4485,"titles":4486,"di":4487,"performing":4488,"difference":4489,"acts":4490,"attacks":4491,"##ov":4492,"existing":4493,"votes":4494,"opportunity":4495,"nor":4496,"shop":4497,"entirely":4498,"trains":4499,"opposite":4500,"pakistan":4501,"##pa":4502,"develop":4503,"resulted":4504,"representatives":4505,"actions":4506,"reality":4507,"pressed":4508,"##ish":4509,"barely":4510,"wine":4511,"conversation":4512,"faculty":4513,"northwest":4514,"ends":4515,"documentary":4516,"nuclear":4517,"stock":4518,"grace":4519,"sets":4520,"eat":4521,"alternative":4522,"##ps":4523,"bag":4524,"resulting":4525,"creating":4526,"surprised":4527,"cemetery":4528,"1919":4529,"drop":4530,"finding":4531,"sarah":4532,"cricket":4533,"streets":4534,"tradition":4535,"ride":4536,"1933":4537,"exhibition":4538,"target":4539,"ear":4540,"explained":4541,"rain":4542,"composer":4543,"injury":4544,"apartment":4545,"municipal":4546,"educational":4547,"occupied":4548,"netherlands":4549,"clean":4550,"billion":4551,"constitution":4552,"learn":4553,"1914":4554,"maximum":4555,"classical":4556,"francis":4557,"lose":4558,"opposition":4559,"jose":4560,"ontario":4561,"bear":4562,"core":4563,"hills":4564,"rolled":4565,"ending":4566,"drawn":4567,"permanent":4568,"fun":4569,"##tes":4570,"##lla":4571,"lewis":4572,"sites":4573,"chamber":4574,"ryan":4575,"##way":4576,"scoring":4577,"height":4578,"1934":4579,"##house":4580,"lyrics":4581,"staring":4582,"55":4583,"officials":4584,"1917":4585,"snow":4586,"oldest":4587,"##tic":4588,"orange":4589,"##ger":4590,"qualified":4591,"interior":4592,"apparently":4593,"succeeded":4594,"thousand":4595,"dinner":4596,"lights":4597,"existence":4598,"fans":4599,"heavily":4600,"41":4601,"greatest":4602,"conservative":4603,"send":4604,"bowl":4605,"plus":4606,"enter":4607,"catch":4608,"##un":4609,"economy":4610,"duty":4611,"1929":4612,"speech":4613,"authorities":4614,"princess":4615,"performances":4616,"versions":4617,"shall":4618,"graduate":4619,"pictures":4620,"effective":4621,"remembered":4622,"poetry":4623,"desk":4624,"crossed":4625,"starring":4626,"starts":4627,"passenger":4628,"sharp":4629,"##ant":4630,"acres":4631,"ass":4632,"weather":4633,"falling":4634,"rank":4635,"fund":4636,"supporting":4637,"check":4638,"adult":4639,"publishing":4640,"heads":4641,"cm":4642,"southeast":4643,"lane":4644,"##burg":4645,"application":4646,"bc":4647,"##ura":4648,"les":4649,"condition":4650,"transfer":4651,"prevent":4652,"display":4653,"ex":4654,"regions":4655,"earl":4656,"federation":4657,"cool":4658,"relatively":4659,"answered":4660,"besides":4661,"1928":4662,"obtained":4663,"portion":4664,"##town":4665,"mix":4666,"##ding":4667,"reaction":4668,"liked":4669,"dean":4670,"express":4671,"peak":4672,"1932":4673,"##tte":4674,"counter":4675,"religion":4676,"chain":4677,"rare":4678,"miller":4679,"convention":4680,"aid":4681,"lie":4682,"vehicles":4683,"mobile":4684,"perform":4685,"squad":4686,"wonder":4687,"lying":4688,"crazy":4689,"sword":4690,"##ping":4691,"attempted":4692,"centuries":4693,"weren":4694,"philosophy":4695,"category":4696,"##ize":4697,"anna":4698,"interested":4699,"47":4700,"sweden":4701,"wolf":4702,"frequently":4703,"abandoned":4704,"kg":4705,"literary":4706,"alliance":4707,"task":4708,"entitled":4709,"##ay":4710,"threw":4711,"promotion":4712,"factory":4713,"tiny":4714,"soccer":4715,"visited":4716,"matt":4717,"fm":4718,"achieved":4719,"52":4720,"defence":4721,"internal":4722,"persian":4723,"43":4724,"methods":4725,"##ging":4726,"arrested":4727,"otherwise":4728,"cambridge":4729,"programming":4730,"villages":4731,"elementary":4732,"districts":4733,"rooms":4734,"criminal":4735,"conflict":4736,"worry":4737,"trained":4738,"1931":4739,"attempts":4740,"waited":4741,"signal":4742,"bird":4743,"truck":4744,"subsequent":4745,"programme":4746,"##ol":4747,"ad":4748,"49":4749,"communist":4750,"details":4751,"faith":4752,"sector":4753,"patrick":4754,"carrying":4755,"laugh":4756,"##ss":4757,"controlled":4758,"korean":4759,"showing":4760,"origin":4761,"fuel":4762,"evil":4763,"1927":4764,"##ent":4765,"brief":4766,"identity":4767,"darkness":4768,"address":4769,"pool":4770,"missed":4771,"publication":4772,"web":4773,"planet":4774,"ian":4775,"anne":4776,"wings":4777,"invited":4778,"##tt":4779,"briefly":4780,"standards":4781,"kissed":4782,"##be":4783,"ideas":4784,"climate":4785,"causing":4786,"walter":4787,"worse":4788,"albert":4789,"articles":4790,"winners":4791,"desire":4792,"aged":4793,"northeast":4794,"dangerous":4795,"gate":4796,"doubt":4797,"1922":4798,"wooden":4799,"multi":4800,"##ky":4801,"poet":4802,"rising":4803,"funding":4804,"46":4805,"communications":4806,"communication":4807,"violence":4808,"copies":4809,"prepared":4810,"ford":4811,"investigation":4812,"skills":4813,"1924":4814,"pulling":4815,"electronic":4816,"##ak":4817,"##ial":4818,"##han":4819,"containing":4820,"ultimately":4821,"offices":4822,"singing":4823,"understanding":4824,"restaurant":4825,"tomorrow":4826,"fashion":4827,"christ":4828,"ward":4829,"da":4830,"pope":4831,"stands":4832,"5th":4833,"flow":4834,"studios":4835,"aired":4836,"commissioned":4837,"contained":4838,"exist":4839,"fresh":4840,"americans":4841,"##per":4842,"wrestling":4843,"approved":4844,"kid":4845,"employed":4846,"respect":4847,"suit":4848,"1925":4849,"angel":4850,"asking":4851,"increasing":4852,"frame":4853,"angry":4854,"selling":4855,"1950s":4856,"thin":4857,"finds":4858,"##nd":4859,"temperature":4860,"statement":4861,"ali":4862,"explain":4863,"inhabitants":4864,"towns":4865,"extensive":4866,"narrow":4867,"51":4868,"jane":4869,"flowers":4870,"images":4871,"promise":4872,"somewhere":4873,"object":4874,"fly":4875,"closely":4876,"##ls":4877,"1912":4878,"bureau":4879,"cape":4880,"1926":4881,"weekly":4882,"presidential":4883,"legislative":4884,"1921":4885,"##ai":4886,"##au":4887,"launch":4888,"founding":4889,"##ny":4890,"978":4891,"##ring":4892,"artillery":4893,"strike":4894,"un":4895,"institutions":4896,"roll":4897,"writers":4898,"landing":4899,"chose":4900,"kevin":4901,"anymore":4902,"pp":4903,"##ut":4904,"attorney":4905,"fit":4906,"dan":4907,"billboard":4908,"receiving":4909,"agricultural":4910,"breaking":4911,"sought":4912,"dave":4913,"admitted":4914,"lands":4915,"mexican":4916,"##bury":4917,"charlie":4918,"specifically":4919,"hole":4920,"iv":4921,"howard":4922,"credit":4923,"moscow":4924,"roads":4925,"accident":4926,"1923":4927,"proved":4928,"wear":4929,"struck":4930,"hey":4931,"guards":4932,"stuff":4933,"slid":4934,"expansion":4935,"1915":4936,"cat":4937,"anthony":4938,"##kin":4939,"melbourne":4940,"opposed":4941,"sub":4942,"southwest":4943,"architect":4944,"failure":4945,"plane":4946,"1916":4947,"##ron":4948,"map":4949,"camera":4950,"tank":4951,"listen":4952,"regarding":4953,"wet":4954,"introduction":4955,"metropolitan":4956,"link":4957,"ep":4958,"fighter":4959,"inch":4960,"grown":4961,"gene":4962,"anger":4963,"fixed":4964,"buy":4965,"dvd":4966,"khan":4967,"domestic":4968,"worldwide":4969,"chapel":4970,"mill":4971,"functions":4972,"examples":4973,"##head":4974,"developing":4975,"1910":4976,"turkey":4977,"hits":4978,"pocket":4979,"antonio":4980,"papers":4981,"grow":4982,"unless":4983,"circuit":4984,"18th":4985,"concerned":4986,"attached":4987,"journalist":4988,"selection":4989,"journey":4990,"converted":4991,"provincial":4992,"painted":4993,"hearing":4994,"aren":4995,"bands":4996,"negative":4997,"aside":4998,"wondered":4999,"knight":5000,"lap":5001,"survey":5002,"ma":5003,"##ow":5004,"noise":5005,"billy":5006,"##ium":5007,"shooting":5008,"guide":5009,"bedroom":5010,"priest":5011,"resistance":5012,"motor":5013,"homes":5014,"sounded":5015,"giant":5016,"##mer":5017,"150":5018,"scenes":5019,"equal":5020,"comic":5021,"patients":5022,"hidden":5023,"solid":5024,"actual":5025,"bringing":5026,"afternoon":5027,"touched":5028,"funds":5029,"wedding":5030,"consisted":5031,"marie":5032,"canal":5033,"sr":5034,"kim":5035,"treaty":5036,"turkish":5037,"recognition":5038,"residence":5039,"cathedral":5040,"broad":5041,"knees":5042,"incident":5043,"shaped":5044,"fired":5045,"norwegian":5046,"handle":5047,"cheek":5048,"contest":5049,"represent":5050,"##pe":5051,"representing":5052,"beauty":5053,"##sen":5054,"birds":5055,"advantage":5056,"emergency":5057,"wrapped":5058,"drawing":5059,"notice":5060,"pink":5061,"broadcasting":5062,"##ong":5063,"somehow":5064,"bachelor":5065,"seventh":5066,"collected":5067,"registered":5068,"establishment":5069,"alan":5070,"assumed":5071,"chemical":5072,"personnel":5073,"roger":5074,"retirement":5075,"jeff":5076,"portuguese":5077,"wore":5078,"tied":5079,"device":5080,"threat":5081,"progress":5082,"advance":5083,"##ised":5084,"banks":5085,"hired":5086,"manchester":5087,"nfl":5088,"teachers":5089,"structures":5090,"forever":5091,"##bo":5092,"tennis":5093,"helping":5094,"saturday":5095,"sale":5096,"applications":5097,"junction":5098,"hip":5099,"incorporated":5100,"neighborhood":5101,"dressed":5102,"ceremony":5103,"##ds":5104,"influenced":5105,"hers":5106,"visual":5107,"stairs":5108,"decades":5109,"inner":5110,"kansas":5111,"hung":5112,"hoped":5113,"gain":5114,"scheduled":5115,"downtown":5116,"engaged":5117,"austria":5118,"clock":5119,"norway":5120,"certainly":5121,"pale":5122,"protected":5123,"1913":5124,"victor":5125,"employees":5126,"plate":5127,"putting":5128,"surrounded":5129,"##ists":5130,"finishing":5131,"blues":5132,"tropical":5133,"##ries":5134,"minnesota":5135,"consider":5136,"philippines":5137,"accept":5138,"54":5139,"retrieved":5140,"1900":5141,"concern":5142,"anderson":5143,"properties":5144,"institution":5145,"gordon":5146,"successfully":5147,"vietnam":5148,"##dy":5149,"backing":5150,"outstanding":5151,"muslim":5152,"crossing":5153,"folk":5154,"producing":5155,"usual":5156,"demand":5157,"occurs":5158,"observed":5159,"lawyer":5160,"educated":5161,"##ana":5162,"kelly":5163,"string":5164,"pleasure":5165,"budget":5166,"items":5167,"quietly":5168,"colorado":5169,"philip":5170,"typical":5171,"##worth":5172,"derived":5173,"600":5174,"survived":5175,"asks":5176,"mental":5177,"##ide":5178,"56":5179,"jake":5180,"jews":5181,"distinguished":5182,"ltd":5183,"1911":5184,"sri":5185,"extremely":5186,"53":5187,"athletic":5188,"loud":5189,"thousands":5190,"worried":5191,"shadow":5192,"transportation":5193,"horses":5194,"weapon":5195,"arena":5196,"importance":5197,"users":5198,"tim":5199,"objects":5200,"contributed":5201,"dragon":5202,"douglas":5203,"aware":5204,"senator":5205,"johnny":5206,"jordan":5207,"sisters":5208,"engines":5209,"flag":5210,"investment":5211,"samuel":5212,"shock":5213,"capable":5214,"clark":5215,"row":5216,"wheel":5217,"refers":5218,"session":5219,"familiar":5220,"biggest":5221,"wins":5222,"hate":5223,"maintained":5224,"drove":5225,"hamilton":5226,"request":5227,"expressed":5228,"injured":5229,"underground":5230,"churches":5231,"walker":5232,"wars":5233,"tunnel":5234,"passes":5235,"stupid":5236,"agriculture":5237,"softly":5238,"cabinet":5239,"regarded":5240,"joining":5241,"indiana":5242,"##ea":5243,"##ms":5244,"push":5245,"dates":5246,"spend":5247,"behavior":5248,"woods":5249,"protein":5250,"gently":5251,"chase":5252,"morgan":5253,"mention":5254,"burning":5255,"wake":5256,"combination":5257,"occur":5258,"mirror":5259,"leads":5260,"jimmy":5261,"indeed":5262,"impossible":5263,"singapore":5264,"paintings":5265,"covering":5266,"##nes":5267,"soldier":5268,"locations":5269,"attendance":5270,"sell":5271,"historian":5272,"wisconsin":5273,"invasion":5274,"argued":5275,"painter":5276,"diego":5277,"changing":5278,"egypt":5279,"##don":5280,"experienced":5281,"inches":5282,"##ku":5283,"missouri":5284,"vol":5285,"grounds":5286,"spoken":5287,"switzerland":5288,"##gan":5289,"reform":5290,"rolling":5291,"ha":5292,"forget":5293,"massive":5294,"resigned":5295,"burned":5296,"allen":5297,"tennessee":5298,"locked":5299,"values":5300,"improved":5301,"##mo":5302,"wounded":5303,"universe":5304,"sick":5305,"dating":5306,"facing":5307,"pack":5308,"purchase":5309,"user":5310,"##pur":5311,"moments":5312,"##ul":5313,"merged":5314,"anniversary":5315,"1908":5316,"coal":5317,"brick":5318,"understood":5319,"causes":5320,"dynasty":5321,"queensland":5322,"establish":5323,"stores":5324,"crisis":5325,"promote":5326,"hoping":5327,"views":5328,"cards":5329,"referee":5330,"extension":5331,"##si":5332,"raise":5333,"arizona":5334,"improve":5335,"colonial":5336,"formal":5337,"charged":5338,"##rt":5339,"palm":5340,"lucky":5341,"hide":5342,"rescue":5343,"faces":5344,"95":5345,"feelings":5346,"candidates":5347,"juan":5348,"##ell":5349,"goods":5350,"6th":5351,"courses":5352,"weekend":5353,"59":5354,"luke":5355,"cash":5356,"fallen":5357,"##om":5358,"delivered":5359,"affected":5360,"installed":5361,"carefully":5362,"tries":5363,"swiss":5364,"hollywood":5365,"costs":5366,"lincoln":5367,"responsibility":5368,"##he":5369,"shore":5370,"file":5371,"proper":5372,"normally":5373,"maryland":5374,"assistance":5375,"jump":5376,"constant":5377,"offering":5378,"friendly":5379,"waters":5380,"persons":5381,"realize":5382,"contain":5383,"trophy":5384,"800":5385,"partnership":5386,"factor":5387,"58":5388,"musicians":5389,"cry":5390,"bound":5391,"oregon":5392,"indicated":5393,"hero":5394,"houston":5395,"medium":5396,"##ure":5397,"consisting":5398,"somewhat":5399,"##ara":5400,"57":5401,"cycle":5402,"##che":5403,"beer":5404,"moore":5405,"frederick":5406,"gotten":5407,"eleven":5408,"worst":5409,"weak":5410,"approached":5411,"arranged":5412,"chin":5413,"loan":5414,"universal":5415,"bond":5416,"fifteen":5417,"pattern":5418,"disappeared":5419,"##ney":5420,"translated":5421,"##zed":5422,"lip":5423,"arab":5424,"capture":5425,"interests":5426,"insurance":5427,"##chi":5428,"shifted":5429,"cave":5430,"prix":5431,"warning":5432,"sections":5433,"courts":5434,"coat":5435,"plot":5436,"smell":5437,"feed":5438,"golf":5439,"favorite":5440,"maintain":5441,"knife":5442,"vs":5443,"voted":5444,"degrees":5445,"finance":5446,"quebec":5447,"opinion":5448,"translation":5449,"manner":5450,"ruled":5451,"operate":5452,"productions":5453,"choose":5454,"musician":5455,"discovery":5456,"confused":5457,"tired":5458,"separated":5459,"stream":5460,"techniques":5461,"committed":5462,"attend":5463,"ranking":5464,"kings":5465,"throw":5466,"passengers":5467,"measure":5468,"horror":5469,"fan":5470,"mining":5471,"sand":5472,"danger":5473,"salt":5474,"calm":5475,"decade":5476,"dam":5477,"require":5478,"runner":5479,"##ik":5480,"rush":5481,"associate":5482,"greece":5483,"##ker":5484,"rivers":5485,"consecutive":5486,"matthew":5487,"##ski":5488,"sighed":5489,"sq":5490,"documents":5491,"steam":5492,"edited":5493,"closing":5494,"tie":5495,"accused":5496,"1905":5497,"##ini":5498,"islamic":5499,"distributed":5500,"directors":5501,"organisation":5502,"bruce":5503,"7th":5504,"breathing":5505,"mad":5506,"lit":5507,"arrival":5508,"concrete":5509,"taste":5510,"08":5511,"composition":5512,"shaking":5513,"faster":5514,"amateur":5515,"adjacent":5516,"stating":5517,"1906":5518,"twin":5519,"flew":5520,"##ran":5521,"tokyo":5522,"publications":5523,"##tone":5524,"obviously":5525,"ridge":5526,"storage":5527,"1907":5528,"carl":5529,"pages":5530,"concluded":5531,"desert":5532,"driven":5533,"universities":5534,"ages":5535,"terminal":5536,"sequence":5537,"borough":5538,"250":5539,"constituency":5540,"creative":5541,"cousin":5542,"economics":5543,"dreams":5544,"margaret":5545,"notably":5546,"reduce":5547,"montreal":5548,"mode":5549,"17th":5550,"ears":5551,"saved":5552,"jan":5553,"vocal":5554,"##ica":5555,"1909":5556,"andy":5557,"##jo":5558,"riding":5559,"roughly":5560,"threatened":5561,"##ise":5562,"meters":5563,"meanwhile":5564,"landed":5565,"compete":5566,"repeated":5567,"grass":5568,"czech":5569,"regularly":5570,"charges":5571,"tea":5572,"sudden":5573,"appeal":5574,"##ung":5575,"solution":5576,"describes":5577,"pierre":5578,"classification":5579,"glad":5580,"parking":5581,"##ning":5582,"belt":5583,"physics":5584,"99":5585,"rachel":5586,"add":5587,"hungarian":5588,"participate":5589,"expedition":5590,"damaged":5591,"gift":5592,"childhood":5593,"85":5594,"fifty":5595,"##red":5596,"mathematics":5597,"jumped":5598,"letting":5599,"defensive":5600,"mph":5601,"##ux":5602,"##gh":5603,"testing":5604,"##hip":5605,"hundreds":5606,"shoot":5607,"owners":5608,"matters":5609,"smoke":5610,"israeli":5611,"kentucky":5612,"dancing":5613,"mounted":5614,"grandfather":5615,"emma":5616,"designs":5617,"profit":5618,"argentina":5619,"##gs":5620,"truly":5621,"li":5622,"lawrence":5623,"cole":5624,"begun":5625,"detroit":5626,"willing":5627,"branches":5628,"smiling":5629,"decide":5630,"miami":5631,"enjoyed":5632,"recordings":5633,"##dale":5634,"poverty":5635,"ethnic":5636,"gay":5637,"##bi":5638,"gary":5639,"arabic":5640,"09":5641,"accompanied":5642,"##one":5643,"##ons":5644,"fishing":5645,"determine":5646,"residential":5647,"acid":5648,"##ary":5649,"alice":5650,"returns":5651,"starred":5652,"mail":5653,"##ang":5654,"jonathan":5655,"strategy":5656,"##ue":5657,"net":5658,"forty":5659,"cook":5660,"businesses":5661,"equivalent":5662,"commonwealth":5663,"distinct":5664,"ill":5665,"##cy":5666,"seriously":5667,"##ors":5668,"##ped":5669,"shift":5670,"harris":5671,"replace":5672,"rio":5673,"imagine":5674,"formula":5675,"ensure":5676,"##ber":5677,"additionally":5678,"scheme":5679,"conservation":5680,"occasionally":5681,"purposes":5682,"feels":5683,"favor":5684,"##and":5685,"##ore":5686,"1930s":5687,"contrast":5688,"hanging":5689,"hunt":5690,"movies":5691,"1904":5692,"instruments":5693,"victims":5694,"danish":5695,"christopher":5696,"busy":5697,"demon":5698,"sugar":5699,"earliest":5700,"colony":5701,"studying":5702,"balance":5703,"duties":5704,"##ks":5705,"belgium":5706,"slipped":5707,"carter":5708,"05":5709,"visible":5710,"stages":5711,"iraq":5712,"fifa":5713,"##im":5714,"commune":5715,"forming":5716,"zero":5717,"07":5718,"continuing":5719,"talked":5720,"counties":5721,"legend":5722,"bathroom":5723,"option":5724,"tail":5725,"clay":5726,"daughters":5727,"afterwards":5728,"severe":5729,"jaw":5730,"visitors":5731,"##ded":5732,"devices":5733,"aviation":5734,"russell":5735,"kate":5736,"##vi":5737,"entering":5738,"subjects":5739,"##ino":5740,"temporary":5741,"swimming":5742,"forth":5743,"smooth":5744,"ghost":5745,"audio":5746,"bush":5747,"operates":5748,"rocks":5749,"movements":5750,"signs":5751,"eddie":5752,"##tz":5753,"ann":5754,"voices":5755,"honorary":5756,"06":5757,"memories":5758,"dallas":5759,"pure":5760,"measures":5761,"racial":5762,"promised":5763,"66":5764,"harvard":5765,"ceo":5766,"16th":5767,"parliamentary":5768,"indicate":5769,"benefit":5770,"flesh":5771,"dublin":5772,"louisiana":5773,"1902":5774,"1901":5775,"patient":5776,"sleeping":5777,"1903":5778,"membership":5779,"coastal":5780,"medieval":5781,"wanting":5782,"element":5783,"scholars":5784,"rice":5785,"62":5786,"limit":5787,"survive":5788,"makeup":5789,"rating":5790,"definitely":5791,"collaboration":5792,"obvious":5793,"##tan":5794,"boss":5795,"ms":5796,"baron":5797,"birthday":5798,"linked":5799,"soil":5800,"diocese":5801,"##lan":5802,"ncaa":5803,"##mann":5804,"offensive":5805,"shell":5806,"shouldn":5807,"waist":5808,"##tus":5809,"plain":5810,"ross":5811,"organ":5812,"resolution":5813,"manufacturing":5814,"adding":5815,"relative":5816,"kennedy":5817,"98":5818,"whilst":5819,"moth":5820,"marketing":5821,"gardens":5822,"crash":5823,"72":5824,"heading":5825,"partners":5826,"credited":5827,"carlos":5828,"moves":5829,"cable":5830,"##zi":5831,"marshall":5832,"##out":5833,"depending":5834,"bottle":5835,"represents":5836,"rejected":5837,"responded":5838,"existed":5839,"04":5840,"jobs":5841,"denmark":5842,"lock":5843,"##ating":5844,"treated":5845,"graham":5846,"routes":5847,"talent":5848,"commissioner":5849,"drugs":5850,"secure":5851,"tests":5852,"reign":5853,"restored":5854,"photography":5855,"##gi":5856,"contributions":5857,"oklahoma":5858,"designer":5859,"disc":5860,"grin":5861,"seattle":5862,"robin":5863,"paused":5864,"atlanta":5865,"unusual":5866,"##gate":5867,"praised":5868,"las":5869,"laughing":5870,"satellite":5871,"hungary":5872,"visiting":5873,"##sky":5874,"interesting":5875,"factors":5876,"deck":5877,"poems":5878,"norman":5879,"##water":5880,"stuck":5881,"speaker":5882,"rifle":5883,"domain":5884,"premiered":5885,"##her":5886,"dc":5887,"comics":5888,"actors":5889,"01":5890,"reputation":5891,"eliminated":5892,"8th":5893,"ceiling":5894,"prisoners":5895,"script":5896,"##nce":5897,"leather":5898,"austin":5899,"mississippi":5900,"rapidly":5901,"admiral":5902,"parallel":5903,"charlotte":5904,"guilty":5905,"tools":5906,"gender":5907,"divisions":5908,"fruit":5909,"##bs":5910,"laboratory":5911,"nelson":5912,"fantasy":5913,"marry":5914,"rapid":5915,"aunt":5916,"tribe":5917,"requirements":5918,"aspects":5919,"suicide":5920,"amongst":5921,"adams":5922,"bone":5923,"ukraine":5924,"abc":5925,"kick":5926,"sees":5927,"edinburgh":5928,"clothing":5929,"column":5930,"rough":5931,"gods":5932,"hunting":5933,"broadway":5934,"gathered":5935,"concerns":5936,"##ek":5937,"spending":5938,"ty":5939,"12th":5940,"snapped":5941,"requires":5942,"solar":5943,"bones":5944,"cavalry":5945,"##tta":5946,"iowa":5947,"drinking":5948,"waste":5949,"index":5950,"franklin":5951,"charity":5952,"thompson":5953,"stewart":5954,"tip":5955,"flash":5956,"landscape":5957,"friday":5958,"enjoy":5959,"singh":5960,"poem":5961,"listening":5962,"##back":5963,"eighth":5964,"fred":5965,"differences":5966,"adapted":5967,"bomb":5968,"ukrainian":5969,"surgery":5970,"corporate":5971,"masters":5972,"anywhere":5973,"##more":5974,"waves":5975,"odd":5976,"sean":5977,"portugal":5978,"orleans":5979,"dick":5980,"debate":5981,"kent":5982,"eating":5983,"puerto":5984,"cleared":5985,"96":5986,"expect":5987,"cinema":5988,"97":5989,"guitarist":5990,"blocks":5991,"electrical":5992,"agree":5993,"involving":5994,"depth":5995,"dying":5996,"panel":5997,"struggle":5998,"##ged":5999,"peninsula":6000,"adults":6001,"novels":6002,"emerged":6003,"vienna":6004,"metro":6005,"debuted":6006,"shoes":6007,"tamil":6008,"songwriter":6009,"meets":6010,"prove":6011,"beating":6012,"instance":6013,"heaven":6014,"scared":6015,"sending":6016,"marks":6017,"artistic":6018,"passage":6019,"superior":6020,"03":6021,"significantly":6022,"shopping":6023,"##tive":6024,"retained":6025,"##izing":6026,"malaysia":6027,"technique":6028,"cheeks":6029,"##ola":6030,"warren":6031,"maintenance":6032,"destroy":6033,"extreme":6034,"allied":6035,"120":6036,"appearing":6037,"##yn":6038,"fill":6039,"advice":6040,"alabama":6041,"qualifying":6042,"policies":6043,"cleveland":6044,"hat":6045,"battery":6046,"smart":6047,"authors":6048,"10th":6049,"soundtrack":6050,"acted":6051,"dated":6052,"lb":6053,"glance":6054,"equipped":6055,"coalition":6056,"funny":6057,"outer":6058,"ambassador":6059,"roy":6060,"possibility":6061,"couples":6062,"campbell":6063,"dna":6064,"loose":6065,"ethan":6066,"supplies":6067,"1898":6068,"gonna":6069,"88":6070,"monster":6071,"##res":6072,"shake":6073,"agents":6074,"frequency":6075,"springs":6076,"dogs":6077,"practices":6078,"61":6079,"gang":6080,"plastic":6081,"easier":6082,"suggests":6083,"gulf":6084,"blade":6085,"exposed":6086,"colors":6087,"industries":6088,"markets":6089,"pan":6090,"nervous":6091,"electoral":6092,"charts":6093,"legislation":6094,"ownership":6095,"##idae":6096,"mac":6097,"appointment":6098,"shield":6099,"copy":6100,"assault":6101,"socialist":6102,"abbey":6103,"monument":6104,"license":6105,"throne":6106,"employment":6107,"jay":6108,"93":6109,"replacement":6110,"charter":6111,"cloud":6112,"powered":6113,"suffering":6114,"accounts":6115,"oak":6116,"connecticut":6117,"strongly":6118,"wright":6119,"colour":6120,"crystal":6121,"13th":6122,"context":6123,"welsh":6124,"networks":6125,"voiced":6126,"gabriel":6127,"jerry":6128,"##cing":6129,"forehead":6130,"mp":6131,"##ens":6132,"manage":6133,"schedule":6134,"totally":6135,"remix":6136,"##ii":6137,"forests":6138,"occupation":6139,"print":6140,"nicholas":6141,"brazilian":6142,"strategic":6143,"vampires":6144,"engineers":6145,"76":6146,"roots":6147,"seek":6148,"correct":6149,"instrumental":6150,"und":6151,"alfred":6152,"backed":6153,"hop":6154,"##des":6155,"stanley":6156,"robinson":6157,"traveled":6158,"wayne":6159,"welcome":6160,"austrian":6161,"achieve":6162,"67":6163,"exit":6164,"rates":6165,"1899":6166,"strip":6167,"whereas":6168,"##cs":6169,"sing":6170,"deeply":6171,"adventure":6172,"bobby":6173,"rick":6174,"jamie":6175,"careful":6176,"components":6177,"cap":6178,"useful":6179,"personality":6180,"knee":6181,"##shi":6182,"pushing":6183,"hosts":6184,"02":6185,"protest":6186,"ca":6187,"ottoman":6188,"symphony":6189,"##sis":6190,"63":6191,"boundary":6192,"1890":6193,"processes":6194,"considering":6195,"considerable":6196,"tons":6197,"##work":6198,"##ft":6199,"##nia":6200,"cooper":6201,"trading":6202,"dear":6203,"conduct":6204,"91":6205,"illegal":6206,"apple":6207,"revolutionary":6208,"holiday":6209,"definition":6210,"harder":6211,"##van":6212,"jacob":6213,"circumstances":6214,"destruction":6215,"##lle":6216,"popularity":6217,"grip":6218,"classified":6219,"liverpool":6220,"donald":6221,"baltimore":6222,"flows":6223,"seeking":6224,"honour":6225,"approval":6226,"92":6227,"mechanical":6228,"till":6229,"happening":6230,"statue":6231,"critic":6232,"increasingly":6233,"immediate":6234,"describe":6235,"commerce":6236,"stare":6237,"##ster":6238,"indonesia":6239,"meat":6240,"rounds":6241,"boats":6242,"baker":6243,"orthodox":6244,"depression":6245,"formally":6246,"worn":6247,"naked":6248,"claire":6249,"muttered":6250,"sentence":6251,"11th":6252,"emily":6253,"document":6254,"77":6255,"criticism":6256,"wished":6257,"vessel":6258,"spiritual":6259,"bent":6260,"virgin":6261,"parker":6262,"minimum":6263,"murray":6264,"lunch":6265,"danny":6266,"printed":6267,"compilation":6268,"keyboards":6269,"false":6270,"blow":6271,"belonged":6272,"68":6273,"raising":6274,"78":6275,"cutting":6276,"##board":6277,"pittsburgh":6278,"##up":6279,"9th":6280,"shadows":6281,"81":6282,"hated":6283,"indigenous":6284,"jon":6285,"15th":6286,"barry":6287,"scholar":6288,"ah":6289,"##zer":6290,"oliver":6291,"##gy":6292,"stick":6293,"susan":6294,"meetings":6295,"attracted":6296,"spell":6297,"romantic":6298,"##ver":6299,"ye":6300,"1895":6301,"photo":6302,"demanded":6303,"customers":6304,"##ac":6305,"1896":6306,"logan":6307,"revival":6308,"keys":6309,"modified":6310,"commanded":6311,"jeans":6312,"##ious":6313,"upset":6314,"raw":6315,"phil":6316,"detective":6317,"hiding":6318,"resident":6319,"vincent":6320,"##bly":6321,"experiences":6322,"diamond":6323,"defeating":6324,"coverage":6325,"lucas":6326,"external":6327,"parks":6328,"franchise":6329,"helen":6330,"bible":6331,"successor":6332,"percussion":6333,"celebrated":6334,"il":6335,"lift":6336,"profile":6337,"clan":6338,"romania":6339,"##ied":6340,"mills":6341,"##su":6342,"nobody":6343,"achievement":6344,"shrugged":6345,"fault":6346,"1897":6347,"rhythm":6348,"initiative":6349,"breakfast":6350,"carbon":6351,"700":6352,"69":6353,"lasted":6354,"violent":6355,"74":6356,"wound":6357,"ken":6358,"killer":6359,"gradually":6360,"filmed":6361,"°c":6362,"dollars":6363,"processing":6364,"94":6365,"remove":6366,"criticized":6367,"guests":6368,"sang":6369,"chemistry":6370,"##vin":6371,"legislature":6372,"disney":6373,"##bridge":6374,"uniform":6375,"escaped":6376,"integrated":6377,"proposal":6378,"purple":6379,"denied":6380,"liquid":6381,"karl":6382,"influential":6383,"morris":6384,"nights":6385,"stones":6386,"intense":6387,"experimental":6388,"twisted":6389,"71":6390,"84":6391,"##ld":6392,"pace":6393,"nazi":6394,"mitchell":6395,"ny":6396,"blind":6397,"reporter":6398,"newspapers":6399,"14th":6400,"centers":6401,"burn":6402,"basin":6403,"forgotten":6404,"surviving":6405,"filed":6406,"collections":6407,"monastery":6408,"losses":6409,"manual":6410,"couch":6411,"description":6412,"appropriate":6413,"merely":6414,"tag":6415,"missions":6416,"sebastian":6417,"restoration":6418,"replacing":6419,"triple":6420,"73":6421,"elder":6422,"julia":6423,"warriors":6424,"benjamin":6425,"julian":6426,"convinced":6427,"stronger":6428,"amazing":6429,"declined":6430,"versus":6431,"merchant":6432,"happens":6433,"output":6434,"finland":6435,"bare":6436,"barbara":6437,"absence":6438,"ignored":6439,"dawn":6440,"injuries":6441,"##port":6442,"producers":6443,"##ram":6444,"82":6445,"luis":6446,"##ities":6447,"kw":6448,"admit":6449,"expensive":6450,"electricity":6451,"nba":6452,"exception":6453,"symbol":6454,"##ving":6455,"ladies":6456,"shower":6457,"sheriff":6458,"characteristics":6459,"##je":6460,"aimed":6461,"button":6462,"ratio":6463,"effectively":6464,"summit":6465,"angle":6466,"jury":6467,"bears":6468,"foster":6469,"vessels":6470,"pants":6471,"executed":6472,"evans":6473,"dozen":6474,"advertising":6475,"kicked":6476,"patrol":6477,"1889":6478,"competitions":6479,"lifetime":6480,"principles":6481,"athletics":6482,"##logy":6483,"birmingham":6484,"sponsored":6485,"89":6486,"rob":6487,"nomination":6488,"1893":6489,"acoustic":6490,"##sm":6491,"creature":6492,"longest":6493,"##tra":6494,"credits":6495,"harbor":6496,"dust":6497,"josh":6498,"##so":6499,"territories":6500,"milk":6501,"infrastructure":6502,"completion":6503,"thailand":6504,"indians":6505,"leon":6506,"archbishop":6507,"##sy":6508,"assist":6509,"pitch":6510,"blake":6511,"arrangement":6512,"girlfriend":6513,"serbian":6514,"operational":6515,"hence":6516,"sad":6517,"scent":6518,"fur":6519,"dj":6520,"sessions":6521,"hp":6522,"refer":6523,"rarely":6524,"##ora":6525,"exists":6526,"1892":6527,"##ten":6528,"scientists":6529,"dirty":6530,"penalty":6531,"burst":6532,"portrait":6533,"seed":6534,"79":6535,"pole":6536,"limits":6537,"rival":6538,"1894":6539,"stable":6540,"alpha":6541,"grave":6542,"constitutional":6543,"alcohol":6544,"arrest":6545,"flower":6546,"mystery":6547,"devil":6548,"architectural":6549,"relationships":6550,"greatly":6551,"habitat":6552,"##istic":6553,"larry":6554,"progressive":6555,"remote":6556,"cotton":6557,"##ics":6558,"##ok":6559,"preserved":6560,"reaches":6561,"##ming":6562,"cited":6563,"86":6564,"vast":6565,"scholarship":6566,"decisions":6567,"cbs":6568,"joy":6569,"teach":6570,"1885":6571,"editions":6572,"knocked":6573,"eve":6574,"searching":6575,"partly":6576,"participation":6577,"gap":6578,"animated":6579,"fate":6580,"excellent":6581,"##ett":6582,"na":6583,"87":6584,"alternate":6585,"saints":6586,"youngest":6587,"##ily":6588,"climbed":6589,"##ita":6590,"##tors":6591,"suggest":6592,"##ct":6593,"discussion":6594,"staying":6595,"choir":6596,"lakes":6597,"jacket":6598,"revenue":6599,"nevertheless":6600,"peaked":6601,"instrument":6602,"wondering":6603,"annually":6604,"managing":6605,"neil":6606,"1891":6607,"signing":6608,"terry":6609,"##ice":6610,"apply":6611,"clinical":6612,"brooklyn":6613,"aim":6614,"catherine":6615,"fuck":6616,"farmers":6617,"figured":6618,"ninth":6619,"pride":6620,"hugh":6621,"evolution":6622,"ordinary":6623,"involvement":6624,"comfortable":6625,"shouted":6626,"tech":6627,"encouraged":6628,"taiwan":6629,"representation":6630,"sharing":6631,"##lia":6632,"##em":6633,"panic":6634,"exact":6635,"cargo":6636,"competing":6637,"fat":6638,"cried":6639,"83":6640,"1920s":6641,"occasions":6642,"pa":6643,"cabin":6644,"borders":6645,"utah":6646,"marcus":6647,"##isation":6648,"badly":6649,"muscles":6650,"##ance":6651,"victorian":6652,"transition":6653,"warner":6654,"bet":6655,"permission":6656,"##rin":6657,"slave":6658,"terrible":6659,"similarly":6660,"shares":6661,"seth":6662,"uefa":6663,"possession":6664,"medals":6665,"benefits":6666,"colleges":6667,"lowered":6668,"perfectly":6669,"mall":6670,"transit":6671,"##ye":6672,"##kar":6673,"publisher":6674,"##ened":6675,"harrison":6676,"deaths":6677,"elevation":6678,"##ae":6679,"asleep":6680,"machines":6681,"sigh":6682,"ash":6683,"hardly":6684,"argument":6685,"occasion":6686,"parent":6687,"leo":6688,"decline":6689,"1888":6690,"contribution":6691,"##ua":6692,"concentration":6693,"1000":6694,"opportunities":6695,"hispanic":6696,"guardian":6697,"extent":6698,"emotions":6699,"hips":6700,"mason":6701,"volumes":6702,"bloody":6703,"controversy":6704,"diameter":6705,"steady":6706,"mistake":6707,"phoenix":6708,"identify":6709,"violin":6710,"##sk":6711,"departure":6712,"richmond":6713,"spin":6714,"funeral":6715,"enemies":6716,"1864":6717,"gear":6718,"literally":6719,"connor":6720,"random":6721,"sergeant":6722,"grab":6723,"confusion":6724,"1865":6725,"transmission":6726,"informed":6727,"op":6728,"leaning":6729,"sacred":6730,"suspended":6731,"thinks":6732,"gates":6733,"portland":6734,"luck":6735,"agencies":6736,"yours":6737,"hull":6738,"expert":6739,"muscle":6740,"layer":6741,"practical":6742,"sculpture":6743,"jerusalem":6744,"latest":6745,"lloyd":6746,"statistics":6747,"deeper":6748,"recommended":6749,"warrior":6750,"arkansas":6751,"mess":6752,"supports":6753,"greg":6754,"eagle":6755,"1880":6756,"recovered":6757,"rated":6758,"concerts":6759,"rushed":6760,"##ano":6761,"stops":6762,"eggs":6763,"files":6764,"premiere":6765,"keith":6766,"##vo":6767,"delhi":6768,"turner":6769,"pit":6770,"affair":6771,"belief":6772,"paint":6773,"##zing":6774,"mate":6775,"##ach":6776,"##ev":6777,"victim":6778,"##ology":6779,"withdrew":6780,"bonus":6781,"styles":6782,"fled":6783,"##ud":6784,"glasgow":6785,"technologies":6786,"funded":6787,"nbc":6788,"adaptation":6789,"##ata":6790,"portrayed":6791,"cooperation":6792,"supporters":6793,"judges":6794,"bernard":6795,"justin":6796,"hallway":6797,"ralph":6798,"##ick":6799,"graduating":6800,"controversial":6801,"distant":6802,"continental":6803,"spider":6804,"bite":6805,"##ho":6806,"recognize":6807,"intention":6808,"mixing":6809,"##ese":6810,"egyptian":6811,"bow":6812,"tourism":6813,"suppose":6814,"claiming":6815,"tiger":6816,"dominated":6817,"participants":6818,"vi":6819,"##ru":6820,"nurse":6821,"partially":6822,"tape":6823,"##rum":6824,"psychology":6825,"##rn":6826,"essential":6827,"touring":6828,"duo":6829,"voting":6830,"civilian":6831,"emotional":6832,"channels":6833,"##king":6834,"apparent":6835,"hebrew":6836,"1887":6837,"tommy":6838,"carrier":6839,"intersection":6840,"beast":6841,"hudson":6842,"##gar":6843,"##zo":6844,"lab":6845,"nova":6846,"bench":6847,"discuss":6848,"costa":6849,"##ered":6850,"detailed":6851,"behalf":6852,"drivers":6853,"unfortunately":6854,"obtain":6855,"##lis":6856,"rocky":6857,"##dae":6858,"siege":6859,"friendship":6860,"honey":6861,"##rian":6862,"1861":6863,"amy":6864,"hang":6865,"posted":6866,"governments":6867,"collins":6868,"respond":6869,"wildlife":6870,"preferred":6871,"operator":6872,"##po":6873,"laura":6874,"pregnant":6875,"videos":6876,"dennis":6877,"suspected":6878,"boots":6879,"instantly":6880,"weird":6881,"automatic":6882,"businessman":6883,"alleged":6884,"placing":6885,"throwing":6886,"ph":6887,"mood":6888,"1862":6889,"perry":6890,"venue":6891,"jet":6892,"remainder":6893,"##lli":6894,"##ci":6895,"passion":6896,"biological":6897,"boyfriend":6898,"1863":6899,"dirt":6900,"buffalo":6901,"ron":6902,"segment":6903,"fa":6904,"abuse":6905,"##era":6906,"genre":6907,"thrown":6908,"stroke":6909,"colored":6910,"stress":6911,"exercise":6912,"displayed":6913,"##gen":6914,"struggled":6915,"##tti":6916,"abroad":6917,"dramatic":6918,"wonderful":6919,"thereafter":6920,"madrid":6921,"component":6922,"widespread":6923,"##sed":6924,"tale":6925,"citizen":6926,"todd":6927,"monday":6928,"1886":6929,"vancouver":6930,"overseas":6931,"forcing":6932,"crying":6933,"descent":6934,"##ris":6935,"discussed":6936,"substantial":6937,"ranks":6938,"regime":6939,"1870":6940,"provinces":6941,"switch":6942,"drum":6943,"zane":6944,"ted":6945,"tribes":6946,"proof":6947,"lp":6948,"cream":6949,"researchers":6950,"volunteer":6951,"manor":6952,"silk":6953,"milan":6954,"donated":6955,"allies":6956,"venture":6957,"principle":6958,"delivery":6959,"enterprise":6960,"##ves":6961,"##ans":6962,"bars":6963,"traditionally":6964,"witch":6965,"reminded":6966,"copper":6967,"##uk":6968,"pete":6969,"inter":6970,"links":6971,"colin":6972,"grinned":6973,"elsewhere":6974,"competitive":6975,"frequent":6976,"##oy":6977,"scream":6978,"##hu":6979,"tension":6980,"texts":6981,"submarine":6982,"finnish":6983,"defending":6984,"defend":6985,"pat":6986,"detail":6987,"1884":6988,"affiliated":6989,"stuart":6990,"themes":6991,"villa":6992,"periods":6993,"tool":6994,"belgian":6995,"ruling":6996,"crimes":6997,"answers":6998,"folded":6999,"licensed":7000,"resort":7001,"demolished":7002,"hans":7003,"lucy":7004,"1881":7005,"lion":7006,"traded":7007,"photographs":7008,"writes":7009,"craig":7010,"##fa":7011,"trials":7012,"generated":7013,"beth":7014,"noble":7015,"debt":7016,"percentage":7017,"yorkshire":7018,"erected":7019,"ss":7020,"viewed":7021,"grades":7022,"confidence":7023,"ceased":7024,"islam":7025,"telephone":7026,"retail":7027,"##ible":7028,"chile":7029,"m²":7030,"roberts":7031,"sixteen":7032,"##ich":7033,"commented":7034,"hampshire":7035,"innocent":7036,"dual":7037,"pounds":7038,"checked":7039,"regulations":7040,"afghanistan":7041,"sung":7042,"rico":7043,"liberty":7044,"assets":7045,"bigger":7046,"options":7047,"angels":7048,"relegated":7049,"tribute":7050,"wells":7051,"attending":7052,"leaf":7053,"##yan":7054,"butler":7055,"romanian":7056,"forum":7057,"monthly":7058,"lisa":7059,"patterns":7060,"gmina":7061,"##tory":7062,"madison":7063,"hurricane":7064,"rev":7065,"##ians":7066,"bristol":7067,"##ula":7068,"elite":7069,"valuable":7070,"disaster":7071,"democracy":7072,"awareness":7073,"germans":7074,"freyja":7075,"##ins":7076,"loop":7077,"absolutely":7078,"paying":7079,"populations":7080,"maine":7081,"sole":7082,"prayer":7083,"spencer":7084,"releases":7085,"doorway":7086,"bull":7087,"##ani":7088,"lover":7089,"midnight":7090,"conclusion":7091,"##sson":7092,"thirteen":7093,"lily":7094,"mediterranean":7095,"##lt":7096,"nhl":7097,"proud":7098,"sample":7099,"##hill":7100,"drummer":7101,"guinea":7102,"##ova":7103,"murphy":7104,"climb":7105,"##ston":7106,"instant":7107,"attributed":7108,"horn":7109,"ain":7110,"railways":7111,"steven":7112,"##ao":7113,"autumn":7114,"ferry":7115,"opponent":7116,"root":7117,"traveling":7118,"secured":7119,"corridor":7120,"stretched":7121,"tales":7122,"sheet":7123,"trinity":7124,"cattle":7125,"helps":7126,"indicates":7127,"manhattan":7128,"murdered":7129,"fitted":7130,"1882":7131,"gentle":7132,"grandmother":7133,"mines":7134,"shocked":7135,"vegas":7136,"produces":7137,"##light":7138,"caribbean":7139,"##ou":7140,"belong":7141,"continuous":7142,"desperate":7143,"drunk":7144,"historically":7145,"trio":7146,"waved":7147,"raf":7148,"dealing":7149,"nathan":7150,"bat":7151,"murmured":7152,"interrupted":7153,"residing":7154,"scientist":7155,"pioneer":7156,"harold":7157,"aaron":7158,"##net":7159,"delta":7160,"attempting":7161,"minority":7162,"mini":7163,"believes":7164,"chorus":7165,"tend":7166,"lots":7167,"eyed":7168,"indoor":7169,"load":7170,"shots":7171,"updated":7172,"jail":7173,"##llo":7174,"concerning":7175,"connecting":7176,"wealth":7177,"##ved":7178,"slaves":7179,"arrive":7180,"rangers":7181,"sufficient":7182,"rebuilt":7183,"##wick":7184,"cardinal":7185,"flood":7186,"muhammad":7187,"whenever":7188,"relation":7189,"runners":7190,"moral":7191,"repair":7192,"viewers":7193,"arriving":7194,"revenge":7195,"punk":7196,"assisted":7197,"bath":7198,"fairly":7199,"breathe":7200,"lists":7201,"innings":7202,"illustrated":7203,"whisper":7204,"nearest":7205,"voters":7206,"clinton":7207,"ties":7208,"ultimate":7209,"screamed":7210,"beijing":7211,"lions":7212,"andre":7213,"fictional":7214,"gathering":7215,"comfort":7216,"radar":7217,"suitable":7218,"dismissed":7219,"hms":7220,"ban":7221,"pine":7222,"wrist":7223,"atmosphere":7224,"voivodeship":7225,"bid":7226,"timber":7227,"##ned":7228,"##nan":7229,"giants":7230,"##ane":7231,"cameron":7232,"recovery":7233,"uss":7234,"identical":7235,"categories":7236,"switched":7237,"serbia":7238,"laughter":7239,"noah":7240,"ensemble":7241,"therapy":7242,"peoples":7243,"touching":7244,"##off":7245,"locally":7246,"pearl":7247,"platforms":7248,"everywhere":7249,"ballet":7250,"tables":7251,"lanka":7252,"herbert":7253,"outdoor":7254,"toured":7255,"derek":7256,"1883":7257,"spaces":7258,"contested":7259,"swept":7260,"1878":7261,"exclusive":7262,"slight":7263,"connections":7264,"##dra":7265,"winds":7266,"prisoner":7267,"collective":7268,"bangladesh":7269,"tube":7270,"publicly":7271,"wealthy":7272,"thai":7273,"##ys":7274,"isolated":7275,"select":7276,"##ric":7277,"insisted":7278,"pen":7279,"fortune":7280,"ticket":7281,"spotted":7282,"reportedly":7283,"animation":7284,"enforcement":7285,"tanks":7286,"110":7287,"decides":7288,"wider":7289,"lowest":7290,"owen":7291,"##time":7292,"nod":7293,"hitting":7294,"##hn":7295,"gregory":7296,"furthermore":7297,"magazines":7298,"fighters":7299,"solutions":7300,"##ery":7301,"pointing":7302,"requested":7303,"peru":7304,"reed":7305,"chancellor":7306,"knights":7307,"mask":7308,"worker":7309,"eldest":7310,"flames":7311,"reduction":7312,"1860":7313,"volunteers":7314,"##tis":7315,"reporting":7316,"##hl":7317,"wire":7318,"advisory":7319,"endemic":7320,"origins":7321,"settlers":7322,"pursue":7323,"knock":7324,"consumer":7325,"1876":7326,"eu":7327,"compound":7328,"creatures":7329,"mansion":7330,"sentenced":7331,"ivan":7332,"deployed":7333,"guitars":7334,"frowned":7335,"involves":7336,"mechanism":7337,"kilometers":7338,"perspective":7339,"shops":7340,"maps":7341,"terminus":7342,"duncan":7343,"alien":7344,"fist":7345,"bridges":7346,"##pers":7347,"heroes":7348,"fed":7349,"derby":7350,"swallowed":7351,"##ros":7352,"patent":7353,"sara":7354,"illness":7355,"characterized":7356,"adventures":7357,"slide":7358,"hawaii":7359,"jurisdiction":7360,"##op":7361,"organised":7362,"##side":7363,"adelaide":7364,"walks":7365,"biology":7366,"se":7367,"##ties":7368,"rogers":7369,"swing":7370,"tightly":7371,"boundaries":7372,"##rie":7373,"prepare":7374,"implementation":7375,"stolen":7376,"##sha":7377,"certified":7378,"colombia":7379,"edwards":7380,"garage":7381,"##mm":7382,"recalled":7383,"##ball":7384,"rage":7385,"harm":7386,"nigeria":7387,"breast":7388,"##ren":7389,"furniture":7390,"pupils":7391,"settle":7392,"##lus":7393,"cuba":7394,"balls":7395,"client":7396,"alaska":7397,"21st":7398,"linear":7399,"thrust":7400,"celebration":7401,"latino":7402,"genetic":7403,"terror":7404,"##cia":7405,"##ening":7406,"lightning":7407,"fee":7408,"witness":7409,"lodge":7410,"establishing":7411,"skull":7412,"##ique":7413,"earning":7414,"hood":7415,"##ei":7416,"rebellion":7417,"wang":7418,"sporting":7419,"warned":7420,"missile":7421,"devoted":7422,"activist":7423,"porch":7424,"worship":7425,"fourteen":7426,"package":7427,"1871":7428,"decorated":7429,"##shire":7430,"housed":7431,"##ock":7432,"chess":7433,"sailed":7434,"doctors":7435,"oscar":7436,"joan":7437,"treat":7438,"garcia":7439,"harbour":7440,"jeremy":7441,"##ire":7442,"traditions":7443,"dominant":7444,"jacques":7445,"##gon":7446,"##wan":7447,"relocated":7448,"1879":7449,"amendment":7450,"sized":7451,"companion":7452,"simultaneously":7453,"volleyball":7454,"spun":7455,"acre":7456,"increases":7457,"stopping":7458,"loves":7459,"belongs":7460,"affect":7461,"drafted":7462,"tossed":7463,"scout":7464,"battles":7465,"1875":7466,"filming":7467,"shoved":7468,"munich":7469,"tenure":7470,"vertical":7471,"romance":7472,"pc":7473,"##cher":7474,"argue":7475,"##ical":7476,"craft":7477,"ranging":7478,"www":7479,"opens":7480,"honest":7481,"tyler":7482,"yesterday":7483,"virtual":7484,"##let":7485,"muslims":7486,"reveal":7487,"snake":7488,"immigrants":7489,"radical":7490,"screaming":7491,"speakers":7492,"firing":7493,"saving":7494,"belonging":7495,"ease":7496,"lighting":7497,"prefecture":7498,"blame":7499,"farmer":7500,"hungry":7501,"grows":7502,"rubbed":7503,"beam":7504,"sur":7505,"subsidiary":7506,"##cha":7507,"armenian":7508,"sao":7509,"dropping":7510,"conventional":7511,"##fer":7512,"microsoft":7513,"reply":7514,"qualify":7515,"spots":7516,"1867":7517,"sweat":7518,"festivals":7519,"##ken":7520,"immigration":7521,"physician":7522,"discover":7523,"exposure":7524,"sandy":7525,"explanation":7526,"isaac":7527,"implemented":7528,"##fish":7529,"hart":7530,"initiated":7531,"connect":7532,"stakes":7533,"presents":7534,"heights":7535,"householder":7536,"pleased":7537,"tourist":7538,"regardless":7539,"slip":7540,"closest":7541,"##ction":7542,"surely":7543,"sultan":7544,"brings":7545,"riley":7546,"preparation":7547,"aboard":7548,"slammed":7549,"baptist":7550,"experiment":7551,"ongoing":7552,"interstate":7553,"organic":7554,"playoffs":7555,"##ika":7556,"1877":7557,"130":7558,"##tar":7559,"hindu":7560,"error":7561,"tours":7562,"tier":7563,"plenty":7564,"arrangements":7565,"talks":7566,"trapped":7567,"excited":7568,"sank":7569,"ho":7570,"athens":7571,"1872":7572,"denver":7573,"welfare":7574,"suburb":7575,"athletes":7576,"trick":7577,"diverse":7578,"belly":7579,"exclusively":7580,"yelled":7581,"1868":7582,"##med":7583,"conversion":7584,"##ette":7585,"1874":7586,"internationally":7587,"computers":7588,"conductor":7589,"abilities":7590,"sensitive":7591,"hello":7592,"dispute":7593,"measured":7594,"globe":7595,"rocket":7596,"prices":7597,"amsterdam":7598,"flights":7599,"tigers":7600,"inn":7601,"municipalities":7602,"emotion":7603,"references":7604,"3d":7605,"##mus":7606,"explains":7607,"airlines":7608,"manufactured":7609,"pm":7610,"archaeological":7611,"1873":7612,"interpretation":7613,"devon":7614,"comment":7615,"##ites":7616,"settlements":7617,"kissing":7618,"absolute":7619,"improvement":7620,"suite":7621,"impressed":7622,"barcelona":7623,"sullivan":7624,"jefferson":7625,"towers":7626,"jesse":7627,"julie":7628,"##tin":7629,"##lu":7630,"grandson":7631,"hi":7632,"gauge":7633,"regard":7634,"rings":7635,"interviews":7636,"trace":7637,"raymond":7638,"thumb":7639,"departments":7640,"burns":7641,"serial":7642,"bulgarian":7643,"scores":7644,"demonstrated":7645,"##ix":7646,"1866":7647,"kyle":7648,"alberta":7649,"underneath":7650,"romanized":7651,"##ward":7652,"relieved":7653,"acquisition":7654,"phrase":7655,"cliff":7656,"reveals":7657,"han":7658,"cuts":7659,"merger":7660,"custom":7661,"##dar":7662,"nee":7663,"gilbert":7664,"graduation":7665,"##nts":7666,"assessment":7667,"cafe":7668,"difficulty":7669,"demands":7670,"swung":7671,"democrat":7672,"jennifer":7673,"commons":7674,"1940s":7675,"grove":7676,"##yo":7677,"completing":7678,"focuses":7679,"sum":7680,"substitute":7681,"bearing":7682,"stretch":7683,"reception":7684,"##py":7685,"reflected":7686,"essentially":7687,"destination":7688,"pairs":7689,"##ched":7690,"survival":7691,"resource":7692,"##bach":7693,"promoting":7694,"doubles":7695,"messages":7696,"tear":7697,"##down":7698,"##fully":7699,"parade":7700,"florence":7701,"harvey":7702,"incumbent":7703,"partial":7704,"framework":7705,"900":7706,"pedro":7707,"frozen":7708,"procedure":7709,"olivia":7710,"controls":7711,"##mic":7712,"shelter":7713,"personally":7714,"temperatures":7715,"##od":7716,"brisbane":7717,"tested":7718,"sits":7719,"marble":7720,"comprehensive":7721,"oxygen":7722,"leonard":7723,"##kov":7724,"inaugural":7725,"iranian":7726,"referring":7727,"quarters":7728,"attitude":7729,"##ivity":7730,"mainstream":7731,"lined":7732,"mars":7733,"dakota":7734,"norfolk":7735,"unsuccessful":7736,"##°":7737,"explosion":7738,"helicopter":7739,"congressional":7740,"##sing":7741,"inspector":7742,"bitch":7743,"seal":7744,"departed":7745,"divine":7746,"##ters":7747,"coaching":7748,"examination":7749,"punishment":7750,"manufacturer":7751,"sink":7752,"columns":7753,"unincorporated":7754,"signals":7755,"nevada":7756,"squeezed":7757,"dylan":7758,"dining":7759,"photos":7760,"martial":7761,"manuel":7762,"eighteen":7763,"elevator":7764,"brushed":7765,"plates":7766,"ministers":7767,"ivy":7768,"congregation":7769,"##len":7770,"slept":7771,"specialized":7772,"taxes":7773,"curve":7774,"restricted":7775,"negotiations":7776,"likes":7777,"statistical":7778,"arnold":7779,"inspiration":7780,"execution":7781,"bold":7782,"intermediate":7783,"significance":7784,"margin":7785,"ruler":7786,"wheels":7787,"gothic":7788,"intellectual":7789,"dependent":7790,"listened":7791,"eligible":7792,"buses":7793,"widow":7794,"syria":7795,"earn":7796,"cincinnati":7797,"collapsed":7798,"recipient":7799,"secrets":7800,"accessible":7801,"philippine":7802,"maritime":7803,"goddess":7804,"clerk":7805,"surrender":7806,"breaks":7807,"playoff":7808,"database":7809,"##ified":7810,"##lon":7811,"ideal":7812,"beetle":7813,"aspect":7814,"soap":7815,"regulation":7816,"strings":7817,"expand":7818,"anglo":7819,"shorter":7820,"crosses":7821,"retreat":7822,"tough":7823,"coins":7824,"wallace":7825,"directions":7826,"pressing":7827,"##oon":7828,"shipping":7829,"locomotives":7830,"comparison":7831,"topics":7832,"nephew":7833,"##mes":7834,"distinction":7835,"honors":7836,"travelled":7837,"sierra":7838,"ibn":7839,"##over":7840,"fortress":7841,"sa":7842,"recognised":7843,"carved":7844,"1869":7845,"clients":7846,"##dan":7847,"intent":7848,"##mar":7849,"coaches":7850,"describing":7851,"bread":7852,"##ington":7853,"beaten":7854,"northwestern":7855,"##ona":7856,"merit":7857,"youtube":7858,"collapse":7859,"challenges":7860,"em":7861,"historians":7862,"objective":7863,"submitted":7864,"virus":7865,"attacking":7866,"drake":7867,"assume":7868,"##ere":7869,"diseases":7870,"marc":7871,"stem":7872,"leeds":7873,"##cus":7874,"##ab":7875,"farming":7876,"glasses":7877,"##lock":7878,"visits":7879,"nowhere":7880,"fellowship":7881,"relevant":7882,"carries":7883,"restaurants":7884,"experiments":7885,"101":7886,"constantly":7887,"bases":7888,"targets":7889,"shah":7890,"tenth":7891,"opponents":7892,"verse":7893,"territorial":7894,"##ira":7895,"writings":7896,"corruption":7897,"##hs":7898,"instruction":7899,"inherited":7900,"reverse":7901,"emphasis":7902,"##vic":7903,"employee":7904,"arch":7905,"keeps":7906,"rabbi":7907,"watson":7908,"payment":7909,"uh":7910,"##ala":7911,"nancy":7912,"##tre":7913,"venice":7914,"fastest":7915,"sexy":7916,"banned":7917,"adrian":7918,"properly":7919,"ruth":7920,"touchdown":7921,"dollar":7922,"boards":7923,"metre":7924,"circles":7925,"edges":7926,"favour":7927,"comments":7928,"ok":7929,"travels":7930,"liberation":7931,"scattered":7932,"firmly":7933,"##ular":7934,"holland":7935,"permitted":7936,"diesel":7937,"kenya":7938,"den":7939,"originated":7940,"##ral":7941,"demons":7942,"resumed":7943,"dragged":7944,"rider":7945,"##rus":7946,"servant":7947,"blinked":7948,"extend":7949,"torn":7950,"##ias":7951,"##sey":7952,"input":7953,"meal":7954,"everybody":7955,"cylinder":7956,"kinds":7957,"camps":7958,"##fe":7959,"bullet":7960,"logic":7961,"##wn":7962,"croatian":7963,"evolved":7964,"healthy":7965,"fool":7966,"chocolate":7967,"wise":7968,"preserve":7969,"pradesh":7970,"##ess":7971,"respective":7972,"1850":7973,"##ew":7974,"chicken":7975,"artificial":7976,"gross":7977,"corresponding":7978,"convicted":7979,"cage":7980,"caroline":7981,"dialogue":7982,"##dor":7983,"narrative":7984,"stranger":7985,"mario":7986,"br":7987,"christianity":7988,"failing":7989,"trent":7990,"commanding":7991,"buddhist":7992,"1848":7993,"maurice":7994,"focusing":7995,"yale":7996,"bike":7997,"altitude":7998,"##ering":7999,"mouse":8000,"revised":8001,"##sley":8002,"veteran":8003,"##ig":8004,"pulls":8005,"theology":8006,"crashed":8007,"campaigns":8008,"legion":8009,"##ability":8010,"drag":8011,"excellence":8012,"customer":8013,"cancelled":8014,"intensity":8015,"excuse":8016,"##lar":8017,"liga":8018,"participating":8019,"contributing":8020,"printing":8021,"##burn":8022,"variable":8023,"##rk":8024,"curious":8025,"bin":8026,"legacy":8027,"renaissance":8028,"##my":8029,"symptoms":8030,"binding":8031,"vocalist":8032,"dancer":8033,"##nie":8034,"grammar":8035,"gospel":8036,"democrats":8037,"ya":8038,"enters":8039,"sc":8040,"diplomatic":8041,"hitler":8042,"##ser":8043,"clouds":8044,"mathematical":8045,"quit":8046,"defended":8047,"oriented":8048,"##heim":8049,"fundamental":8050,"hardware":8051,"impressive":8052,"equally":8053,"convince":8054,"confederate":8055,"guilt":8056,"chuck":8057,"sliding":8058,"##ware":8059,"magnetic":8060,"narrowed":8061,"petersburg":8062,"bulgaria":8063,"otto":8064,"phd":8065,"skill":8066,"##ama":8067,"reader":8068,"hopes":8069,"pitcher":8070,"reservoir":8071,"hearts":8072,"automatically":8073,"expecting":8074,"mysterious":8075,"bennett":8076,"extensively":8077,"imagined":8078,"seeds":8079,"monitor":8080,"fix":8081,"##ative":8082,"journalism":8083,"struggling":8084,"signature":8085,"ranch":8086,"encounter":8087,"photographer":8088,"observation":8089,"protests":8090,"##pin":8091,"influences":8092,"##hr":8093,"calendar":8094,"##all":8095,"cruz":8096,"croatia":8097,"locomotive":8098,"hughes":8099,"naturally":8100,"shakespeare":8101,"basement":8102,"hook":8103,"uncredited":8104,"faded":8105,"theories":8106,"approaches":8107,"dare":8108,"phillips":8109,"filling":8110,"fury":8111,"obama":8112,"##ain":8113,"efficient":8114,"arc":8115,"deliver":8116,"min":8117,"raid":8118,"breeding":8119,"inducted":8120,"leagues":8121,"efficiency":8122,"axis":8123,"montana":8124,"eagles":8125,"##ked":8126,"supplied":8127,"instructions":8128,"karen":8129,"picking":8130,"indicating":8131,"trap":8132,"anchor":8133,"practically":8134,"christians":8135,"tomb":8136,"vary":8137,"occasional":8138,"electronics":8139,"lords":8140,"readers":8141,"newcastle":8142,"faint":8143,"innovation":8144,"collect":8145,"situations":8146,"engagement":8147,"160":8148,"claude":8149,"mixture":8150,"##feld":8151,"peer":8152,"tissue":8153,"logo":8154,"lean":8155,"##ration":8156,"°f":8157,"floors":8158,"##ven":8159,"architects":8160,"reducing":8161,"##our":8162,"##ments":8163,"rope":8164,"1859":8165,"ottawa":8166,"##har":8167,"samples":8168,"banking":8169,"declaration":8170,"proteins":8171,"resignation":8172,"francois":8173,"saudi":8174,"advocate":8175,"exhibited":8176,"armor":8177,"twins":8178,"divorce":8179,"##ras":8180,"abraham":8181,"reviewed":8182,"jo":8183,"temporarily":8184,"matrix":8185,"physically":8186,"pulse":8187,"curled":8188,"##ena":8189,"difficulties":8190,"bengal":8191,"usage":8192,"##ban":8193,"annie":8194,"riders":8195,"certificate":8196,"##pi":8197,"holes":8198,"warsaw":8199,"distinctive":8200,"jessica":8201,"##mon":8202,"mutual":8203,"1857":8204,"customs":8205,"circular":8206,"eugene":8207,"removal":8208,"loaded":8209,"mere":8210,"vulnerable":8211,"depicted":8212,"generations":8213,"dame":8214,"heir":8215,"enormous":8216,"lightly":8217,"climbing":8218,"pitched":8219,"lessons":8220,"pilots":8221,"nepal":8222,"ram":8223,"google":8224,"preparing":8225,"brad":8226,"louise":8227,"renowned":8228,"##₂":8229,"liam":8230,"##ably":8231,"plaza":8232,"shaw":8233,"sophie":8234,"brilliant":8235,"bills":8236,"##bar":8237,"##nik":8238,"fucking":8239,"mainland":8240,"server":8241,"pleasant":8242,"seized":8243,"veterans":8244,"jerked":8245,"fail":8246,"beta":8247,"brush":8248,"radiation":8249,"stored":8250,"warmth":8251,"southeastern":8252,"nate":8253,"sin":8254,"raced":8255,"berkeley":8256,"joke":8257,"athlete":8258,"designation":8259,"trunk":8260,"##low":8261,"roland":8262,"qualification":8263,"archives":8264,"heels":8265,"artwork":8266,"receives":8267,"judicial":8268,"reserves":8269,"##bed":8270,"woke":8271,"installation":8272,"abu":8273,"floating":8274,"fake":8275,"lesser":8276,"excitement":8277,"interface":8278,"concentrated":8279,"addressed":8280,"characteristic":8281,"amanda":8282,"saxophone":8283,"monk":8284,"auto":8285,"##bus":8286,"releasing":8287,"egg":8288,"dies":8289,"interaction":8290,"defender":8291,"ce":8292,"outbreak":8293,"glory":8294,"loving":8295,"##bert":8296,"sequel":8297,"consciousness":8298,"http":8299,"awake":8300,"ski":8301,"enrolled":8302,"##ress":8303,"handling":8304,"rookie":8305,"brow":8306,"somebody":8307,"biography":8308,"warfare":8309,"amounts":8310,"contracts":8311,"presentation":8312,"fabric":8313,"dissolved":8314,"challenged":8315,"meter":8316,"psychological":8317,"lt":8318,"elevated":8319,"rally":8320,"accurate":8321,"##tha":8322,"hospitals":8323,"undergraduate":8324,"specialist":8325,"venezuela":8326,"exhibit":8327,"shed":8328,"nursing":8329,"protestant":8330,"fluid":8331,"structural":8332,"footage":8333,"jared":8334,"consistent":8335,"prey":8336,"##ska":8337,"succession":8338,"reflect":8339,"exile":8340,"lebanon":8341,"wiped":8342,"suspect":8343,"shanghai":8344,"resting":8345,"integration":8346,"preservation":8347,"marvel":8348,"variant":8349,"pirates":8350,"sheep":8351,"rounded":8352,"capita":8353,"sailing":8354,"colonies":8355,"manuscript":8356,"deemed":8357,"variations":8358,"clarke":8359,"functional":8360,"emerging":8361,"boxing":8362,"relaxed":8363,"curse":8364,"azerbaijan":8365,"heavyweight":8366,"nickname":8367,"editorial":8368,"rang":8369,"grid":8370,"tightened":8371,"earthquake":8372,"flashed":8373,"miguel":8374,"rushing":8375,"##ches":8376,"improvements":8377,"boxes":8378,"brooks":8379,"180":8380,"consumption":8381,"molecular":8382,"felix":8383,"societies":8384,"repeatedly":8385,"variation":8386,"aids":8387,"civic":8388,"graphics":8389,"professionals":8390,"realm":8391,"autonomous":8392,"receiver":8393,"delayed":8394,"workshop":8395,"militia":8396,"chairs":8397,"trump":8398,"canyon":8399,"##point":8400,"harsh":8401,"extending":8402,"lovely":8403,"happiness":8404,"##jan":8405,"stake":8406,"eyebrows":8407,"embassy":8408,"wellington":8409,"hannah":8410,"##ella":8411,"sony":8412,"corners":8413,"bishops":8414,"swear":8415,"cloth":8416,"contents":8417,"xi":8418,"namely":8419,"commenced":8420,"1854":8421,"stanford":8422,"nashville":8423,"courage":8424,"graphic":8425,"commitment":8426,"garrison":8427,"##bin":8428,"hamlet":8429,"clearing":8430,"rebels":8431,"attraction":8432,"literacy":8433,"cooking":8434,"ruins":8435,"temples":8436,"jenny":8437,"humanity":8438,"celebrate":8439,"hasn":8440,"freight":8441,"sixty":8442,"rebel":8443,"bastard":8444,"##art":8445,"newton":8446,"##ada":8447,"deer":8448,"##ges":8449,"##ching":8450,"smiles":8451,"delaware":8452,"singers":8453,"##ets":8454,"approaching":8455,"assists":8456,"flame":8457,"##ph":8458,"boulevard":8459,"barrel":8460,"planted":8461,"##ome":8462,"pursuit":8463,"##sia":8464,"consequences":8465,"posts":8466,"shallow":8467,"invitation":8468,"rode":8469,"depot":8470,"ernest":8471,"kane":8472,"rod":8473,"concepts":8474,"preston":8475,"topic":8476,"chambers":8477,"striking":8478,"blast":8479,"arrives":8480,"descendants":8481,"montgomery":8482,"ranges":8483,"worlds":8484,"##lay":8485,"##ari":8486,"span":8487,"chaos":8488,"praise":8489,"##ag":8490,"fewer":8491,"1855":8492,"sanctuary":8493,"mud":8494,"fbi":8495,"##ions":8496,"programmes":8497,"maintaining":8498,"unity":8499,"harper":8500,"bore":8501,"handsome":8502,"closure":8503,"tournaments":8504,"thunder":8505,"nebraska":8506,"linda":8507,"facade":8508,"puts":8509,"satisfied":8510,"argentine":8511,"dale":8512,"cork":8513,"dome":8514,"panama":8515,"##yl":8516,"1858":8517,"tasks":8518,"experts":8519,"##ates":8520,"feeding":8521,"equation":8522,"##las":8523,"##ida":8524,"##tu":8525,"engage":8526,"bryan":8527,"##ax":8528,"um":8529,"quartet":8530,"melody":8531,"disbanded":8532,"sheffield":8533,"blocked":8534,"gasped":8535,"delay":8536,"kisses":8537,"maggie":8538,"connects":8539,"##non":8540,"sts":8541,"poured":8542,"creator":8543,"publishers":8544,"##we":8545,"guided":8546,"ellis":8547,"extinct":8548,"hug":8549,"gaining":8550,"##ord":8551,"complicated":8552,"##bility":8553,"poll":8554,"clenched":8555,"investigate":8556,"##use":8557,"thereby":8558,"quantum":8559,"spine":8560,"cdp":8561,"humor":8562,"kills":8563,"administered":8564,"semifinals":8565,"##du":8566,"encountered":8567,"ignore":8568,"##bu":8569,"commentary":8570,"##maker":8571,"bother":8572,"roosevelt":8573,"140":8574,"plains":8575,"halfway":8576,"flowing":8577,"cultures":8578,"crack":8579,"imprisoned":8580,"neighboring":8581,"airline":8582,"##ses":8583,"##view":8584,"##mate":8585,"##ec":8586,"gather":8587,"wolves":8588,"marathon":8589,"transformed":8590,"##ill":8591,"cruise":8592,"organisations":8593,"carol":8594,"punch":8595,"exhibitions":8596,"numbered":8597,"alarm":8598,"ratings":8599,"daddy":8600,"silently":8601,"##stein":8602,"queens":8603,"colours":8604,"impression":8605,"guidance":8606,"liu":8607,"tactical":8608,"##rat":8609,"marshal":8610,"della":8611,"arrow":8612,"##ings":8613,"rested":8614,"feared":8615,"tender":8616,"owns":8617,"bitter":8618,"advisor":8619,"escort":8620,"##ides":8621,"spare":8622,"farms":8623,"grants":8624,"##ene":8625,"dragons":8626,"encourage":8627,"colleagues":8628,"cameras":8629,"##und":8630,"sucked":8631,"pile":8632,"spirits":8633,"prague":8634,"statements":8635,"suspension":8636,"landmark":8637,"fence":8638,"torture":8639,"recreation":8640,"bags":8641,"permanently":8642,"survivors":8643,"pond":8644,"spy":8645,"predecessor":8646,"bombing":8647,"coup":8648,"##og":8649,"protecting":8650,"transformation":8651,"glow":8652,"##lands":8653,"##book":8654,"dug":8655,"priests":8656,"andrea":8657,"feat":8658,"barn":8659,"jumping":8660,"##chen":8661,"##ologist":8662,"##con":8663,"casualties":8664,"stern":8665,"auckland":8666,"pipe":8667,"serie":8668,"revealing":8669,"ba":8670,"##bel":8671,"trevor":8672,"mercy":8673,"spectrum":8674,"yang":8675,"consist":8676,"governing":8677,"collaborated":8678,"possessed":8679,"epic":8680,"comprises":8681,"blew":8682,"shane":8683,"##ack":8684,"lopez":8685,"honored":8686,"magical":8687,"sacrifice":8688,"judgment":8689,"perceived":8690,"hammer":8691,"mtv":8692,"baronet":8693,"tune":8694,"das":8695,"missionary":8696,"sheets":8697,"350":8698,"neutral":8699,"oral":8700,"threatening":8701,"attractive":8702,"shade":8703,"aims":8704,"seminary":8705,"##master":8706,"estates":8707,"1856":8708,"michel":8709,"wounds":8710,"refugees":8711,"manufacturers":8712,"##nic":8713,"mercury":8714,"syndrome":8715,"porter":8716,"##iya":8717,"##din":8718,"hamburg":8719,"identification":8720,"upstairs":8721,"purse":8722,"widened":8723,"pause":8724,"cared":8725,"breathed":8726,"affiliate":8727,"santiago":8728,"prevented":8729,"celtic":8730,"fisher":8731,"125":8732,"recruited":8733,"byzantine":8734,"reconstruction":8735,"farther":8736,"##mp":8737,"diet":8738,"sake":8739,"au":8740,"spite":8741,"sensation":8742,"##ert":8743,"blank":8744,"separation":8745,"105":8746,"##hon":8747,"vladimir":8748,"armies":8749,"anime":8750,"##lie":8751,"accommodate":8752,"orbit":8753,"cult":8754,"sofia":8755,"archive":8756,"##ify":8757,"##box":8758,"founders":8759,"sustained":8760,"disorder":8761,"honours":8762,"northeastern":8763,"mia":8764,"crops":8765,"violet":8766,"threats":8767,"blanket":8768,"fires":8769,"canton":8770,"followers":8771,"southwestern":8772,"prototype":8773,"voyage":8774,"assignment":8775,"altered":8776,"moderate":8777,"protocol":8778,"pistol":8779,"##eo":8780,"questioned":8781,"brass":8782,"lifting":8783,"1852":8784,"math":8785,"authored":8786,"##ual":8787,"doug":8788,"dimensional":8789,"dynamic":8790,"##san":8791,"1851":8792,"pronounced":8793,"grateful":8794,"quest":8795,"uncomfortable":8796,"boom":8797,"presidency":8798,"stevens":8799,"relating":8800,"politicians":8801,"chen":8802,"barrier":8803,"quinn":8804,"diana":8805,"mosque":8806,"tribal":8807,"cheese":8808,"palmer":8809,"portions":8810,"sometime":8811,"chester":8812,"treasure":8813,"wu":8814,"bend":8815,"download":8816,"millions":8817,"reforms":8818,"registration":8819,"##osa":8820,"consequently":8821,"monitoring":8822,"ate":8823,"preliminary":8824,"brandon":8825,"invented":8826,"ps":8827,"eaten":8828,"exterior":8829,"intervention":8830,"ports":8831,"documented":8832,"log":8833,"displays":8834,"lecture":8835,"sally":8836,"favourite":8837,"##itz":8838,"vermont":8839,"lo":8840,"invisible":8841,"isle":8842,"breed":8843,"##ator":8844,"journalists":8845,"relay":8846,"speaks":8847,"backward":8848,"explore":8849,"midfielder":8850,"actively":8851,"stefan":8852,"procedures":8853,"cannon":8854,"blond":8855,"kenneth":8856,"centered":8857,"servants":8858,"chains":8859,"libraries":8860,"malcolm":8861,"essex":8862,"henri":8863,"slavery":8864,"##hal":8865,"facts":8866,"fairy":8867,"coached":8868,"cassie":8869,"cats":8870,"washed":8871,"cop":8872,"##fi":8873,"announcement":8874,"item":8875,"2000s":8876,"vinyl":8877,"activated":8878,"marco":8879,"frontier":8880,"growled":8881,"curriculum":8882,"##das":8883,"loyal":8884,"accomplished":8885,"leslie":8886,"ritual":8887,"kenny":8888,"##00":8889,"vii":8890,"napoleon":8891,"hollow":8892,"hybrid":8893,"jungle":8894,"stationed":8895,"friedrich":8896,"counted":8897,"##ulated":8898,"platinum":8899,"theatrical":8900,"seated":8901,"col":8902,"rubber":8903,"glen":8904,"1840":8905,"diversity":8906,"healing":8907,"extends":8908,"id":8909,"provisions":8910,"administrator":8911,"columbus":8912,"##oe":8913,"tributary":8914,"te":8915,"assured":8916,"org":8917,"##uous":8918,"prestigious":8919,"examined":8920,"lectures":8921,"grammy":8922,"ronald":8923,"associations":8924,"bailey":8925,"allan":8926,"essays":8927,"flute":8928,"believing":8929,"consultant":8930,"proceedings":8931,"travelling":8932,"1853":8933,"kit":8934,"kerala":8935,"yugoslavia":8936,"buddy":8937,"methodist":8938,"##ith":8939,"burial":8940,"centres":8941,"batman":8942,"##nda":8943,"discontinued":8944,"bo":8945,"dock":8946,"stockholm":8947,"lungs":8948,"severely":8949,"##nk":8950,"citing":8951,"manga":8952,"##ugh":8953,"steal":8954,"mumbai":8955,"iraqi":8956,"robot":8957,"celebrity":8958,"bride":8959,"broadcasts":8960,"abolished":8961,"pot":8962,"joel":8963,"overhead":8964,"franz":8965,"packed":8966,"reconnaissance":8967,"johann":8968,"acknowledged":8969,"introduce":8970,"handled":8971,"doctorate":8972,"developments":8973,"drinks":8974,"alley":8975,"palestine":8976,"##nis":8977,"##aki":8978,"proceeded":8979,"recover":8980,"bradley":8981,"grain":8982,"patch":8983,"afford":8984,"infection":8985,"nationalist":8986,"legendary":8987,"##ath":8988,"interchange":8989,"virtually":8990,"gen":8991,"gravity":8992,"exploration":8993,"amber":8994,"vital":8995,"wishes":8996,"powell":8997,"doctrine":8998,"elbow":8999,"screenplay":9000,"##bird":9001,"contribute":9002,"indonesian":9003,"pet":9004,"creates":9005,"##com":9006,"enzyme":9007,"kylie":9008,"discipline":9009,"drops":9010,"manila":9011,"hunger":9012,"##ien":9013,"layers":9014,"suffer":9015,"fever":9016,"bits":9017,"monica":9018,"keyboard":9019,"manages":9020,"##hood":9021,"searched":9022,"appeals":9023,"##bad":9024,"testament":9025,"grande":9026,"reid":9027,"##war":9028,"beliefs":9029,"congo":9030,"##ification":9031,"##dia":9032,"si":9033,"requiring":9034,"##via":9035,"casey":9036,"1849":9037,"regret":9038,"streak":9039,"rape":9040,"depends":9041,"syrian":9042,"sprint":9043,"pound":9044,"tourists":9045,"upcoming":9046,"pub":9047,"##xi":9048,"tense":9049,"##els":9050,"practiced":9051,"echo":9052,"nationwide":9053,"guild":9054,"motorcycle":9055,"liz":9056,"##zar":9057,"chiefs":9058,"desired":9059,"elena":9060,"bye":9061,"precious":9062,"absorbed":9063,"relatives":9064,"booth":9065,"pianist":9066,"##mal":9067,"citizenship":9068,"exhausted":9069,"wilhelm":9070,"##ceae":9071,"##hed":9072,"noting":9073,"quarterback":9074,"urge":9075,"hectares":9076,"##gue":9077,"ace":9078,"holly":9079,"##tal":9080,"blonde":9081,"davies":9082,"parked":9083,"sustainable":9084,"stepping":9085,"twentieth":9086,"airfield":9087,"galaxy":9088,"nest":9089,"chip":9090,"##nell":9091,"tan":9092,"shaft":9093,"paulo":9094,"requirement":9095,"##zy":9096,"paradise":9097,"tobacco":9098,"trans":9099,"renewed":9100,"vietnamese":9101,"##cker":9102,"##ju":9103,"suggesting":9104,"catching":9105,"holmes":9106,"enjoying":9107,"md":9108,"trips":9109,"colt":9110,"holder":9111,"butterfly":9112,"nerve":9113,"reformed":9114,"cherry":9115,"bowling":9116,"trailer":9117,"carriage":9118,"goodbye":9119,"appreciate":9120,"toy":9121,"joshua":9122,"interactive":9123,"enabled":9124,"involve":9125,"##kan":9126,"collar":9127,"determination":9128,"bunch":9129,"facebook":9130,"recall":9131,"shorts":9132,"superintendent":9133,"episcopal":9134,"frustration":9135,"giovanni":9136,"nineteenth":9137,"laser":9138,"privately":9139,"array":9140,"circulation":9141,"##ovic":9142,"armstrong":9143,"deals":9144,"painful":9145,"permit":9146,"discrimination":9147,"##wi":9148,"aires":9149,"retiring":9150,"cottage":9151,"ni":9152,"##sta":9153,"horizon":9154,"ellen":9155,"jamaica":9156,"ripped":9157,"fernando":9158,"chapters":9159,"playstation":9160,"patron":9161,"lecturer":9162,"navigation":9163,"behaviour":9164,"genes":9165,"georgian":9166,"export":9167,"solomon":9168,"rivals":9169,"swift":9170,"seventeen":9171,"rodriguez":9172,"princeton":9173,"independently":9174,"sox":9175,"1847":9176,"arguing":9177,"entity":9178,"casting":9179,"hank":9180,"criteria":9181,"oakland":9182,"geographic":9183,"milwaukee":9184,"reflection":9185,"expanding":9186,"conquest":9187,"dubbed":9188,"##tv":9189,"halt":9190,"brave":9191,"brunswick":9192,"doi":9193,"arched":9194,"curtis":9195,"divorced":9196,"predominantly":9197,"somerset":9198,"streams":9199,"ugly":9200,"zoo":9201,"horrible":9202,"curved":9203,"buenos":9204,"fierce":9205,"dictionary":9206,"vector":9207,"theological":9208,"unions":9209,"handful":9210,"stability":9211,"chan":9212,"punjab":9213,"segments":9214,"##lly":9215,"altar":9216,"ignoring":9217,"gesture":9218,"monsters":9219,"pastor":9220,"##stone":9221,"thighs":9222,"unexpected":9223,"operators":9224,"abruptly":9225,"coin":9226,"compiled":9227,"associates":9228,"improving":9229,"migration":9230,"pin":9231,"##ose":9232,"compact":9233,"collegiate":9234,"reserved":9235,"##urs":9236,"quarterfinals":9237,"roster":9238,"restore":9239,"assembled":9240,"hurry":9241,"oval":9242,"##cies":9243,"1846":9244,"flags":9245,"martha":9246,"##del":9247,"victories":9248,"sharply":9249,"##rated":9250,"argues":9251,"deadly":9252,"neo":9253,"drawings":9254,"symbols":9255,"performer":9256,"##iel":9257,"griffin":9258,"restrictions":9259,"editing":9260,"andrews":9261,"java":9262,"journals":9263,"arabia":9264,"compositions":9265,"dee":9266,"pierce":9267,"removing":9268,"hindi":9269,"casino":9270,"runway":9271,"civilians":9272,"minds":9273,"nasa":9274,"hotels":9275,"##zation":9276,"refuge":9277,"rent":9278,"retain":9279,"potentially":9280,"conferences":9281,"suburban":9282,"conducting":9283,"##tto":9284,"##tions":9285,"##tle":9286,"descended":9287,"massacre":9288,"##cal":9289,"ammunition":9290,"terrain":9291,"fork":9292,"souls":9293,"counts":9294,"chelsea":9295,"durham":9296,"drives":9297,"cab":9298,"##bank":9299,"perth":9300,"realizing":9301,"palestinian":9302,"finn":9303,"simpson":9304,"##dal":9305,"betty":9306,"##ule":9307,"moreover":9308,"particles":9309,"cardinals":9310,"tent":9311,"evaluation":9312,"extraordinary":9313,"##oid":9314,"inscription":9315,"##works":9316,"wednesday":9317,"chloe":9318,"maintains":9319,"panels":9320,"ashley":9321,"trucks":9322,"##nation":9323,"cluster":9324,"sunlight":9325,"strikes":9326,"zhang":9327,"##wing":9328,"dialect":9329,"canon":9330,"##ap":9331,"tucked":9332,"##ws":9333,"collecting":9334,"##mas":9335,"##can":9336,"##sville":9337,"maker":9338,"quoted":9339,"evan":9340,"franco":9341,"aria":9342,"buying":9343,"cleaning":9344,"eva":9345,"closet":9346,"provision":9347,"apollo":9348,"clinic":9349,"rat":9350,"##ez":9351,"necessarily":9352,"ac":9353,"##gle":9354,"##ising":9355,"venues":9356,"flipped":9357,"cent":9358,"spreading":9359,"trustees":9360,"checking":9361,"authorized":9362,"##sco":9363,"disappointed":9364,"##ado":9365,"notion":9366,"duration":9367,"trumpet":9368,"hesitated":9369,"topped":9370,"brussels":9371,"rolls":9372,"theoretical":9373,"hint":9374,"define":9375,"aggressive":9376,"repeat":9377,"wash":9378,"peaceful":9379,"optical":9380,"width":9381,"allegedly":9382,"mcdonald":9383,"strict":9384,"copyright":9385,"##illa":9386,"investors":9387,"mar":9388,"jam":9389,"witnesses":9390,"sounding":9391,"miranda":9392,"michelle":9393,"privacy":9394,"hugo":9395,"harmony":9396,"##pp":9397,"valid":9398,"lynn":9399,"glared":9400,"nina":9401,"102":9402,"headquartered":9403,"diving":9404,"boarding":9405,"gibson":9406,"##ncy":9407,"albanian":9408,"marsh":9409,"routine":9410,"dealt":9411,"enhanced":9412,"er":9413,"intelligent":9414,"substance":9415,"targeted":9416,"enlisted":9417,"discovers":9418,"spinning":9419,"observations":9420,"pissed":9421,"smoking":9422,"rebecca":9423,"capitol":9424,"visa":9425,"varied":9426,"costume":9427,"seemingly":9428,"indies":9429,"compensation":9430,"surgeon":9431,"thursday":9432,"arsenal":9433,"westminster":9434,"suburbs":9435,"rid":9436,"anglican":9437,"##ridge":9438,"knots":9439,"foods":9440,"alumni":9441,"lighter":9442,"fraser":9443,"whoever":9444,"portal":9445,"scandal":9446,"##ray":9447,"gavin":9448,"advised":9449,"instructor":9450,"flooding":9451,"terrorist":9452,"##ale":9453,"teenage":9454,"interim":9455,"senses":9456,"duck":9457,"teen":9458,"thesis":9459,"abby":9460,"eager":9461,"overcome":9462,"##ile":9463,"newport":9464,"glenn":9465,"rises":9466,"shame":9467,"##cc":9468,"prompted":9469,"priority":9470,"forgot":9471,"bomber":9472,"nicolas":9473,"protective":9474,"360":9475,"cartoon":9476,"katherine":9477,"breeze":9478,"lonely":9479,"trusted":9480,"henderson":9481,"richardson":9482,"relax":9483,"banner":9484,"candy":9485,"palms":9486,"remarkable":9487,"##rio":9488,"legends":9489,"cricketer":9490,"essay":9491,"ordained":9492,"edmund":9493,"rifles":9494,"trigger":9495,"##uri":9496,"##away":9497,"sail":9498,"alert":9499,"1830":9500,"audiences":9501,"penn":9502,"sussex":9503,"siblings":9504,"pursued":9505,"indianapolis":9506,"resist":9507,"rosa":9508,"consequence":9509,"succeed":9510,"avoided":9511,"1845":9512,"##ulation":9513,"inland":9514,"##tie":9515,"##nna":9516,"counsel":9517,"profession":9518,"chronicle":9519,"hurried":9520,"##una":9521,"eyebrow":9522,"eventual":9523,"bleeding":9524,"innovative":9525,"cure":9526,"##dom":9527,"committees":9528,"accounting":9529,"con":9530,"scope":9531,"hardy":9532,"heather":9533,"tenor":9534,"gut":9535,"herald":9536,"codes":9537,"tore":9538,"scales":9539,"wagon":9540,"##oo":9541,"luxury":9542,"tin":9543,"prefer":9544,"fountain":9545,"triangle":9546,"bonds":9547,"darling":9548,"convoy":9549,"dried":9550,"traced":9551,"beings":9552,"troy":9553,"accidentally":9554,"slam":9555,"findings":9556,"smelled":9557,"joey":9558,"lawyers":9559,"outcome":9560,"steep":9561,"bosnia":9562,"configuration":9563,"shifting":9564,"toll":9565,"brook":9566,"performers":9567,"lobby":9568,"philosophical":9569,"construct":9570,"shrine":9571,"aggregate":9572,"boot":9573,"cox":9574,"phenomenon":9575,"savage":9576,"insane":9577,"solely":9578,"reynolds":9579,"lifestyle":9580,"##ima":9581,"nationally":9582,"holdings":9583,"consideration":9584,"enable":9585,"edgar":9586,"mo":9587,"mama":9588,"##tein":9589,"fights":9590,"relegation":9591,"chances":9592,"atomic":9593,"hub":9594,"conjunction":9595,"awkward":9596,"reactions":9597,"currency":9598,"finale":9599,"kumar":9600,"underwent":9601,"steering":9602,"elaborate":9603,"gifts":9604,"comprising":9605,"melissa":9606,"veins":9607,"reasonable":9608,"sunshine":9609,"chi":9610,"solve":9611,"trails":9612,"inhabited":9613,"elimination":9614,"ethics":9615,"huh":9616,"ana":9617,"molly":9618,"consent":9619,"apartments":9620,"layout":9621,"marines":9622,"##ces":9623,"hunters":9624,"bulk":9625,"##oma":9626,"hometown":9627,"##wall":9628,"##mont":9629,"cracked":9630,"reads":9631,"neighbouring":9632,"withdrawn":9633,"admission":9634,"wingspan":9635,"damned":9636,"anthology":9637,"lancashire":9638,"brands":9639,"batting":9640,"forgive":9641,"cuban":9642,"awful":9643,"##lyn":9644,"104":9645,"dimensions":9646,"imagination":9647,"##ade":9648,"dante":9649,"##ship":9650,"tracking":9651,"desperately":9652,"goalkeeper":9653,"##yne":9654,"groaned":9655,"workshops":9656,"confident":9657,"burton":9658,"gerald":9659,"milton":9660,"circus":9661,"uncertain":9662,"slope":9663,"copenhagen":9664,"sophia":9665,"fog":9666,"philosopher":9667,"portraits":9668,"accent":9669,"cycling":9670,"varying":9671,"gripped":9672,"larvae":9673,"garrett":9674,"specified":9675,"scotia":9676,"mature":9677,"luther":9678,"kurt":9679,"rap":9680,"##kes":9681,"aerial":9682,"750":9683,"ferdinand":9684,"heated":9685,"es":9686,"transported":9687,"##shan":9688,"safely":9689,"nonetheless":9690,"##orn":9691,"##gal":9692,"motors":9693,"demanding":9694,"##sburg":9695,"startled":9696,"##brook":9697,"ally":9698,"generate":9699,"caps":9700,"ghana":9701,"stained":9702,"demo":9703,"mentions":9704,"beds":9705,"ap":9706,"afterward":9707,"diary":9708,"##bling":9709,"utility":9710,"##iro":9711,"richards":9712,"1837":9713,"conspiracy":9714,"conscious":9715,"shining":9716,"footsteps":9717,"observer":9718,"cyprus":9719,"urged":9720,"loyalty":9721,"developer":9722,"probability":9723,"olive":9724,"upgraded":9725,"gym":9726,"miracle":9727,"insects":9728,"graves":9729,"1844":9730,"ourselves":9731,"hydrogen":9732,"amazon":9733,"katie":9734,"tickets":9735,"poets":9736,"##pm":9737,"planes":9738,"##pan":9739,"prevention":9740,"witnessed":9741,"dense":9742,"jin":9743,"randy":9744,"tang":9745,"warehouse":9746,"monroe":9747,"bang":9748,"archived":9749,"elderly":9750,"investigations":9751,"alec":9752,"granite":9753,"mineral":9754,"conflicts":9755,"controlling":9756,"aboriginal":9757,"carlo":9758,"##zu":9759,"mechanics":9760,"stan":9761,"stark":9762,"rhode":9763,"skirt":9764,"est":9765,"##berry":9766,"bombs":9767,"respected":9768,"##horn":9769,"imposed":9770,"limestone":9771,"deny":9772,"nominee":9773,"memphis":9774,"grabbing":9775,"disabled":9776,"##als":9777,"amusement":9778,"aa":9779,"frankfurt":9780,"corn":9781,"referendum":9782,"varies":9783,"slowed":9784,"disk":9785,"firms":9786,"unconscious":9787,"incredible":9788,"clue":9789,"sue":9790,"##zhou":9791,"twist":9792,"##cio":9793,"joins":9794,"idaho":9795,"chad":9796,"developers":9797,"computing":9798,"destroyer":9799,"103":9800,"mortal":9801,"tucker":9802,"kingston":9803,"choices":9804,"yu":9805,"carson":9806,"1800":9807,"os":9808,"whitney":9809,"geneva":9810,"pretend":9811,"dimension":9812,"staged":9813,"plateau":9814,"maya":9815,"##une":9816,"freestyle":9817,"##bc":9818,"rovers":9819,"hiv":9820,"##ids":9821,"tristan":9822,"classroom":9823,"prospect":9824,"##hus":9825,"honestly":9826,"diploma":9827,"lied":9828,"thermal":9829,"auxiliary":9830,"feast":9831,"unlikely":9832,"iata":9833,"##tel":9834,"morocco":9835,"pounding":9836,"treasury":9837,"lithuania":9838,"considerably":9839,"1841":9840,"dish":9841,"1812":9842,"geological":9843,"matching":9844,"stumbled":9845,"destroying":9846,"marched":9847,"brien":9848,"advances":9849,"cake":9850,"nicole":9851,"belle":9852,"settling":9853,"measuring":9854,"directing":9855,"##mie":9856,"tuesday":9857,"bassist":9858,"capabilities":9859,"stunned":9860,"fraud":9861,"torpedo":9862,"##list":9863,"##phone":9864,"anton":9865,"wisdom":9866,"surveillance":9867,"ruined":9868,"##ulate":9869,"lawsuit":9870,"healthcare":9871,"theorem":9872,"halls":9873,"trend":9874,"aka":9875,"horizontal":9876,"dozens":9877,"acquire":9878,"lasting":9879,"swim":9880,"hawk":9881,"gorgeous":9882,"fees":9883,"vicinity":9884,"decrease":9885,"adoption":9886,"tactics":9887,"##ography":9888,"pakistani":9889,"##ole":9890,"draws":9891,"##hall":9892,"willie":9893,"burke":9894,"heath":9895,"algorithm":9896,"integral":9897,"powder":9898,"elliott":9899,"brigadier":9900,"jackie":9901,"tate":9902,"varieties":9903,"darker":9904,"##cho":9905,"lately":9906,"cigarette":9907,"specimens":9908,"adds":9909,"##ree":9910,"##ensis":9911,"##inger":9912,"exploded":9913,"finalist":9914,"cia":9915,"murders":9916,"wilderness":9917,"arguments":9918,"nicknamed":9919,"acceptance":9920,"onwards":9921,"manufacture":9922,"robertson":9923,"jets":9924,"tampa":9925,"enterprises":9926,"blog":9927,"loudly":9928,"composers":9929,"nominations":9930,"1838":9931,"ai":9932,"malta":9933,"inquiry":9934,"automobile":9935,"hosting":9936,"viii":9937,"rays":9938,"tilted":9939,"grief":9940,"museums":9941,"strategies":9942,"furious":9943,"euro":9944,"equality":9945,"cohen":9946,"poison":9947,"surrey":9948,"wireless":9949,"governed":9950,"ridiculous":9951,"moses":9952,"##esh":9953,"##room":9954,"vanished":9955,"##ito":9956,"barnes":9957,"attract":9958,"morrison":9959,"istanbul":9960,"##iness":9961,"absent":9962,"rotation":9963,"petition":9964,"janet":9965,"##logical":9966,"satisfaction":9967,"custody":9968,"deliberately":9969,"observatory":9970,"comedian":9971,"surfaces":9972,"pinyin":9973,"novelist":9974,"strictly":9975,"canterbury":9976,"oslo":9977,"monks":9978,"embrace":9979,"ibm":9980,"jealous":9981,"photograph":9982,"continent":9983,"dorothy":9984,"marina":9985,"doc":9986,"excess":9987,"holden":9988,"allegations":9989,"explaining":9990,"stack":9991,"avoiding":9992,"lance":9993,"storyline":9994,"majesty":9995,"poorly":9996,"spike":9997,"dos":9998,"bradford":9999,"raven":10000,"travis":10001,"classics":10002,"proven":10003,"voltage":10004,"pillow":10005,"fists":10006,"butt":10007,"1842":10008,"interpreted":10009,"##car":10010,"1839":10011,"gage":10012,"telegraph":10013,"lens":10014,"promising":10015,"expelled":10016,"casual":10017,"collector":10018,"zones":10019,"##min":10020,"silly":10021,"nintendo":10022,"##kh":10023,"##bra":10024,"downstairs":10025,"chef":10026,"suspicious":10027,"afl":10028,"flies":10029,"vacant":10030,"uganda":10031,"pregnancy":10032,"condemned":10033,"lutheran":10034,"estimates":10035,"cheap":10036,"decree":10037,"saxon":10038,"proximity":10039,"stripped":10040,"idiot":10041,"deposits":10042,"contrary":10043,"presenter":10044,"magnus":10045,"glacier":10046,"im":10047,"offense":10048,"edwin":10049,"##ori":10050,"upright":10051,"##long":10052,"bolt":10053,"##ois":10054,"toss":10055,"geographical":10056,"##izes":10057,"environments":10058,"delicate":10059,"marking":10060,"abstract":10061,"xavier":10062,"nails":10063,"windsor":10064,"plantation":10065,"occurring":10066,"equity":10067,"saskatchewan":10068,"fears":10069,"drifted":10070,"sequences":10071,"vegetation":10072,"revolt":10073,"##stic":10074,"1843":10075,"sooner":10076,"fusion":10077,"opposing":10078,"nato":10079,"skating":10080,"1836":10081,"secretly":10082,"ruin":10083,"lease":10084,"##oc":10085,"edit":10086,"##nne":10087,"flora":10088,"anxiety":10089,"ruby":10090,"##ological":10091,"##mia":10092,"tel":10093,"bout":10094,"taxi":10095,"emmy":10096,"frost":10097,"rainbow":10098,"compounds":10099,"foundations":10100,"rainfall":10101,"assassination":10102,"nightmare":10103,"dominican":10104,"##win":10105,"achievements":10106,"deserve":10107,"orlando":10108,"intact":10109,"armenia":10110,"##nte":10111,"calgary":10112,"valentine":10113,"106":10114,"marion":10115,"proclaimed":10116,"theodore":10117,"bells":10118,"courtyard":10119,"thigh":10120,"gonzalez":10121,"console":10122,"troop":10123,"minimal":10124,"monte":10125,"everyday":10126,"##ence":10127,"##if":10128,"supporter":10129,"terrorism":10130,"buck":10131,"openly":10132,"presbyterian":10133,"activists":10134,"carpet":10135,"##iers":10136,"rubbing":10137,"uprising":10138,"##yi":10139,"cute":10140,"conceived":10141,"legally":10142,"##cht":10143,"millennium":10144,"cello":10145,"velocity":10146,"ji":10147,"rescued":10148,"cardiff":10149,"1835":10150,"rex":10151,"concentrate":10152,"senators":10153,"beard":10154,"rendered":10155,"glowing":10156,"battalions":10157,"scouts":10158,"competitors":10159,"sculptor":10160,"catalogue":10161,"arctic":10162,"ion":10163,"raja":10164,"bicycle":10165,"wow":10166,"glancing":10167,"lawn":10168,"##woman":10169,"gentleman":10170,"lighthouse":10171,"publish":10172,"predicted":10173,"calculated":10174,"##val":10175,"variants":10176,"##gne":10177,"strain":10178,"##ui":10179,"winston":10180,"deceased":10181,"##nus":10182,"touchdowns":10183,"brady":10184,"caleb":10185,"sinking":10186,"echoed":10187,"crush":10188,"hon":10189,"blessed":10190,"protagonist":10191,"hayes":10192,"endangered":10193,"magnitude":10194,"editors":10195,"##tine":10196,"estimate":10197,"responsibilities":10198,"##mel":10199,"backup":10200,"laying":10201,"consumed":10202,"sealed":10203,"zurich":10204,"lovers":10205,"frustrated":10206,"##eau":10207,"ahmed":10208,"kicking":10209,"mit":10210,"treasurer":10211,"1832":10212,"biblical":10213,"refuse":10214,"terrified":10215,"pump":10216,"agrees":10217,"genuine":10218,"imprisonment":10219,"refuses":10220,"plymouth":10221,"##hen":10222,"lou":10223,"##nen":10224,"tara":10225,"trembling":10226,"antarctic":10227,"ton":10228,"learns":10229,"##tas":10230,"crap":10231,"crucial":10232,"faction":10233,"atop":10234,"##borough":10235,"wrap":10236,"lancaster":10237,"odds":10238,"hopkins":10239,"erik":10240,"lyon":10241,"##eon":10242,"bros":10243,"##ode":10244,"snap":10245,"locality":10246,"tips":10247,"empress":10248,"crowned":10249,"cal":10250,"acclaimed":10251,"chuckled":10252,"##ory":10253,"clara":10254,"sends":10255,"mild":10256,"towel":10257,"##fl":10258,"##day":10259,"##а":10260,"wishing":10261,"assuming":10262,"interviewed":10263,"##bal":10264,"##die":10265,"interactions":10266,"eden":10267,"cups":10268,"helena":10269,"##lf":10270,"indie":10271,"beck":10272,"##fire":10273,"batteries":10274,"filipino":10275,"wizard":10276,"parted":10277,"##lam":10278,"traces":10279,"##born":10280,"rows":10281,"idol":10282,"albany":10283,"delegates":10284,"##ees":10285,"##sar":10286,"discussions":10287,"##ex":10288,"notre":10289,"instructed":10290,"belgrade":10291,"highways":10292,"suggestion":10293,"lauren":10294,"possess":10295,"orientation":10296,"alexandria":10297,"abdul":10298,"beats":10299,"salary":10300,"reunion":10301,"ludwig":10302,"alright":10303,"wagner":10304,"intimate":10305,"pockets":10306,"slovenia":10307,"hugged":10308,"brighton":10309,"merchants":10310,"cruel":10311,"stole":10312,"trek":10313,"slopes":10314,"repairs":10315,"enrollment":10316,"politically":10317,"underlying":10318,"promotional":10319,"counting":10320,"boeing":10321,"##bb":10322,"isabella":10323,"naming":10324,"##и":10325,"keen":10326,"bacteria":10327,"listing":10328,"separately":10329,"belfast":10330,"ussr":10331,"450":10332,"lithuanian":10333,"anybody":10334,"ribs":10335,"sphere":10336,"martinez":10337,"cock":10338,"embarrassed":10339,"proposals":10340,"fragments":10341,"nationals":10342,"##fs":10343,"##wski":10344,"premises":10345,"fin":10346,"1500":10347,"alpine":10348,"matched":10349,"freely":10350,"bounded":10351,"jace":10352,"sleeve":10353,"##af":10354,"gaming":10355,"pier":10356,"populated":10357,"evident":10358,"##like":10359,"frances":10360,"flooded":10361,"##dle":10362,"frightened":10363,"pour":10364,"trainer":10365,"framed":10366,"visitor":10367,"challenging":10368,"pig":10369,"wickets":10370,"##fold":10371,"infected":10372,"email":10373,"##pes":10374,"arose":10375,"##aw":10376,"reward":10377,"ecuador":10378,"oblast":10379,"vale":10380,"ch":10381,"shuttle":10382,"##usa":10383,"bach":10384,"rankings":10385,"forbidden":10386,"cornwall":10387,"accordance":10388,"salem":10389,"consumers":10390,"bruno":10391,"fantastic":10392,"toes":10393,"machinery":10394,"resolved":10395,"julius":10396,"remembering":10397,"propaganda":10398,"iceland":10399,"bombardment":10400,"tide":10401,"contacts":10402,"wives":10403,"##rah":10404,"concerto":10405,"macdonald":10406,"albania":10407,"implement":10408,"daisy":10409,"tapped":10410,"sudan":10411,"helmet":10412,"angela":10413,"mistress":10414,"##lic":10415,"crop":10416,"sunk":10417,"finest":10418,"##craft":10419,"hostile":10420,"##ute":10421,"##tsu":10422,"boxer":10423,"fr":10424,"paths":10425,"adjusted":10426,"habit":10427,"ballot":10428,"supervision":10429,"soprano":10430,"##zen":10431,"bullets":10432,"wicked":10433,"sunset":10434,"regiments":10435,"disappear":10436,"lamp":10437,"performs":10438,"app":10439,"##gia":10440,"##oa":10441,"rabbit":10442,"digging":10443,"incidents":10444,"entries":10445,"##cion":10446,"dishes":10447,"##oi":10448,"introducing":10449,"##ati":10450,"##fied":10451,"freshman":10452,"slot":10453,"jill":10454,"tackles":10455,"baroque":10456,"backs":10457,"##iest":10458,"lone":10459,"sponsor":10460,"destiny":10461,"altogether":10462,"convert":10463,"##aro":10464,"consensus":10465,"shapes":10466,"demonstration":10467,"basically":10468,"feminist":10469,"auction":10470,"artifacts":10471,"##bing":10472,"strongest":10473,"twitter":10474,"halifax":10475,"2019":10476,"allmusic":10477,"mighty":10478,"smallest":10479,"precise":10480,"alexandra":10481,"viola":10482,"##los":10483,"##ille":10484,"manuscripts":10485,"##illo":10486,"dancers":10487,"ari":10488,"managers":10489,"monuments":10490,"blades":10491,"barracks":10492,"springfield":10493,"maiden":10494,"consolidated":10495,"electron":10496,"##end":10497,"berry":10498,"airing":10499,"wheat":10500,"nobel":10501,"inclusion":10502,"blair":10503,"payments":10504,"geography":10505,"bee":10506,"cc":10507,"eleanor":10508,"react":10509,"##hurst":10510,"afc":10511,"manitoba":10512,"##yu":10513,"su":10514,"lineup":10515,"fitness":10516,"recreational":10517,"investments":10518,"airborne":10519,"disappointment":10520,"##dis":10521,"edmonton":10522,"viewing":10523,"##row":10524,"renovation":10525,"##cast":10526,"infant":10527,"bankruptcy":10528,"roses":10529,"aftermath":10530,"pavilion":10531,"##yer":10532,"carpenter":10533,"withdrawal":10534,"ladder":10535,"##hy":10536,"discussing":10537,"popped":10538,"reliable":10539,"agreements":10540,"rochester":10541,"##abad":10542,"curves":10543,"bombers":10544,"220":10545,"rao":10546,"reverend":10547,"decreased":10548,"choosing":10549,"107":10550,"stiff":10551,"consulting":10552,"naples":10553,"crawford":10554,"tracy":10555,"ka":10556,"ribbon":10557,"cops":10558,"##lee":10559,"crushed":10560,"deciding":10561,"unified":10562,"teenager":10563,"accepting":10564,"flagship":10565,"explorer":10566,"poles":10567,"sanchez":10568,"inspection":10569,"revived":10570,"skilled":10571,"induced":10572,"exchanged":10573,"flee":10574,"locals":10575,"tragedy":10576,"swallow":10577,"loading":10578,"hanna":10579,"demonstrate":10580,"##ela":10581,"salvador":10582,"flown":10583,"contestants":10584,"civilization":10585,"##ines":10586,"wanna":10587,"rhodes":10588,"fletcher":10589,"hector":10590,"knocking":10591,"considers":10592,"##ough":10593,"nash":10594,"mechanisms":10595,"sensed":10596,"mentally":10597,"walt":10598,"unclear":10599,"##eus":10600,"renovated":10601,"madame":10602,"##cks":10603,"crews":10604,"governmental":10605,"##hin":10606,"undertaken":10607,"monkey":10608,"##ben":10609,"##ato":10610,"fatal":10611,"armored":10612,"copa":10613,"caves":10614,"governance":10615,"grasp":10616,"perception":10617,"certification":10618,"froze":10619,"damp":10620,"tugged":10621,"wyoming":10622,"##rg":10623,"##ero":10624,"newman":10625,"##lor":10626,"nerves":10627,"curiosity":10628,"graph":10629,"115":10630,"##ami":10631,"withdraw":10632,"tunnels":10633,"dull":10634,"meredith":10635,"moss":10636,"exhibits":10637,"neighbors":10638,"communicate":10639,"accuracy":10640,"explored":10641,"raiders":10642,"republicans":10643,"secular":10644,"kat":10645,"superman":10646,"penny":10647,"criticised":10648,"##tch":10649,"freed":10650,"update":10651,"conviction":10652,"wade":10653,"ham":10654,"likewise":10655,"delegation":10656,"gotta":10657,"doll":10658,"promises":10659,"technological":10660,"myth":10661,"nationality":10662,"resolve":10663,"convent":10664,"##mark":10665,"sharon":10666,"dig":10667,"sip":10668,"coordinator":10669,"entrepreneur":10670,"fold":10671,"##dine":10672,"capability":10673,"councillor":10674,"synonym":10675,"blown":10676,"swan":10677,"cursed":10678,"1815":10679,"jonas":10680,"haired":10681,"sofa":10682,"canvas":10683,"keeper":10684,"rivalry":10685,"##hart":10686,"rapper":10687,"speedway":10688,"swords":10689,"postal":10690,"maxwell":10691,"estonia":10692,"potter":10693,"recurring":10694,"##nn":10695,"##ave":10696,"errors":10697,"##oni":10698,"cognitive":10699,"1834":10700,"##²":10701,"claws":10702,"nadu":10703,"roberto":10704,"bce":10705,"wrestler":10706,"ellie":10707,"##ations":10708,"infinite":10709,"ink":10710,"##tia":10711,"presumably":10712,"finite":10713,"staircase":10714,"108":10715,"noel":10716,"patricia":10717,"nacional":10718,"##cation":10719,"chill":10720,"eternal":10721,"tu":10722,"preventing":10723,"prussia":10724,"fossil":10725,"limbs":10726,"##logist":10727,"ernst":10728,"frog":10729,"perez":10730,"rene":10731,"##ace":10732,"pizza":10733,"prussian":10734,"##ios":10735,"##vy":10736,"molecules":10737,"regulatory":10738,"answering":10739,"opinions":10740,"sworn":10741,"lengths":10742,"supposedly":10743,"hypothesis":10744,"upward":10745,"habitats":10746,"seating":10747,"ancestors":10748,"drank":10749,"yield":10750,"hd":10751,"synthesis":10752,"researcher":10753,"modest":10754,"##var":10755,"mothers":10756,"peered":10757,"voluntary":10758,"homeland":10759,"##the":10760,"acclaim":10761,"##igan":10762,"static":10763,"valve":10764,"luxembourg":10765,"alto":10766,"carroll":10767,"fe":10768,"receptor":10769,"norton":10770,"ambulance":10771,"##tian":10772,"johnston":10773,"catholics":10774,"depicting":10775,"jointly":10776,"elephant":10777,"gloria":10778,"mentor":10779,"badge":10780,"ahmad":10781,"distinguish":10782,"remarked":10783,"councils":10784,"precisely":10785,"allison":10786,"advancing":10787,"detection":10788,"crowded":10789,"##10":10790,"cooperative":10791,"ankle":10792,"mercedes":10793,"dagger":10794,"surrendered":10795,"pollution":10796,"commit":10797,"subway":10798,"jeffrey":10799,"lesson":10800,"sculptures":10801,"provider":10802,"##fication":10803,"membrane":10804,"timothy":10805,"rectangular":10806,"fiscal":10807,"heating":10808,"teammate":10809,"basket":10810,"particle":10811,"anonymous":10812,"deployment":10813,"##ple":10814,"missiles":10815,"courthouse":10816,"proportion":10817,"shoe":10818,"sec":10819,"##ller":10820,"complaints":10821,"forbes":10822,"blacks":10823,"abandon":10824,"remind":10825,"sizes":10826,"overwhelming":10827,"autobiography":10828,"natalie":10829,"##awa":10830,"risks":10831,"contestant":10832,"countryside":10833,"babies":10834,"scorer":10835,"invaded":10836,"enclosed":10837,"proceed":10838,"hurling":10839,"disorders":10840,"##cu":10841,"reflecting":10842,"continuously":10843,"cruiser":10844,"graduates":10845,"freeway":10846,"investigated":10847,"ore":10848,"deserved":10849,"maid":10850,"blocking":10851,"phillip":10852,"jorge":10853,"shakes":10854,"dove":10855,"mann":10856,"variables":10857,"lacked":10858,"burden":10859,"accompanying":10860,"que":10861,"consistently":10862,"organizing":10863,"provisional":10864,"complained":10865,"endless":10866,"##rm":10867,"tubes":10868,"juice":10869,"georges":10870,"krishna":10871,"mick":10872,"labels":10873,"thriller":10874,"##uch":10875,"laps":10876,"arcade":10877,"sage":10878,"snail":10879,"##table":10880,"shannon":10881,"fi":10882,"laurence":10883,"seoul":10884,"vacation":10885,"presenting":10886,"hire":10887,"churchill":10888,"surprisingly":10889,"prohibited":10890,"savannah":10891,"technically":10892,"##oli":10893,"170":10894,"##lessly":10895,"testimony":10896,"suited":10897,"speeds":10898,"toys":10899,"romans":10900,"mlb":10901,"flowering":10902,"measurement":10903,"talented":10904,"kay":10905,"settings":10906,"charleston":10907,"expectations":10908,"shattered":10909,"achieving":10910,"triumph":10911,"ceremonies":10912,"portsmouth":10913,"lanes":10914,"mandatory":10915,"loser":10916,"stretching":10917,"cologne":10918,"realizes":10919,"seventy":10920,"cornell":10921,"careers":10922,"webb":10923,"##ulating":10924,"americas":10925,"budapest":10926,"ava":10927,"suspicion":10928,"##ison":10929,"yo":10930,"conrad":10931,"##hai":10932,"sterling":10933,"jessie":10934,"rector":10935,"##az":10936,"1831":10937,"transform":10938,"organize":10939,"loans":10940,"christine":10941,"volcanic":10942,"warrant":10943,"slender":10944,"summers":10945,"subfamily":10946,"newer":10947,"danced":10948,"dynamics":10949,"rhine":10950,"proceeds":10951,"heinrich":10952,"gastropod":10953,"commands":10954,"sings":10955,"facilitate":10956,"easter":10957,"ra":10958,"positioned":10959,"responses":10960,"expense":10961,"fruits":10962,"yanked":10963,"imported":10964,"25th":10965,"velvet":10966,"vic":10967,"primitive":10968,"tribune":10969,"baldwin":10970,"neighbourhood":10971,"donna":10972,"rip":10973,"hay":10974,"pr":10975,"##uro":10976,"1814":10977,"espn":10978,"welcomed":10979,"##aria":10980,"qualifier":10981,"glare":10982,"highland":10983,"timing":10984,"##cted":10985,"shells":10986,"eased":10987,"geometry":10988,"louder":10989,"exciting":10990,"slovakia":10991,"##sion":10992,"##iz":10993,"##lot":10994,"savings":10995,"prairie":10996,"##ques":10997,"marching":10998,"rafael":10999,"tonnes":11000,"##lled":11001,"curtain":11002,"preceding":11003,"shy":11004,"heal":11005,"greene":11006,"worthy":11007,"##pot":11008,"detachment":11009,"bury":11010,"sherman":11011,"##eck":11012,"reinforced":11013,"seeks":11014,"bottles":11015,"contracted":11016,"duchess":11017,"outfit":11018,"walsh":11019,"##sc":11020,"mickey":11021,"##ase":11022,"geoffrey":11023,"archer":11024,"squeeze":11025,"dawson":11026,"eliminate":11027,"invention":11028,"##enberg":11029,"neal":11030,"##eth":11031,"stance":11032,"dealer":11033,"coral":11034,"maple":11035,"retire":11036,"polo":11037,"simplified":11038,"##ht":11039,"1833":11040,"hid":11041,"watts":11042,"backwards":11043,"jules":11044,"##oke":11045,"genesis":11046,"mt":11047,"frames":11048,"rebounds":11049,"burma":11050,"woodland":11051,"moist":11052,"santos":11053,"whispers":11054,"drained":11055,"subspecies":11056,"##aa":11057,"streaming":11058,"ulster":11059,"burnt":11060,"correspondence":11061,"maternal":11062,"gerard":11063,"denis":11064,"stealing":11065,"##load":11066,"genius":11067,"duchy":11068,"##oria":11069,"inaugurated":11070,"momentum":11071,"suits":11072,"placement":11073,"sovereign":11074,"clause":11075,"thames":11076,"##hara":11077,"confederation":11078,"reservation":11079,"sketch":11080,"yankees":11081,"lets":11082,"rotten":11083,"charm":11084,"hal":11085,"verses":11086,"ultra":11087,"commercially":11088,"dot":11089,"salon":11090,"citation":11091,"adopt":11092,"winnipeg":11093,"mist":11094,"allocated":11095,"cairo":11096,"##boy":11097,"jenkins":11098,"interference":11099,"objectives":11100,"##wind":11101,"1820":11102,"portfolio":11103,"armoured":11104,"sectors":11105,"##eh":11106,"initiatives":11107,"##world":11108,"integrity":11109,"exercises":11110,"robe":11111,"tap":11112,"ab":11113,"gazed":11114,"##tones":11115,"distracted":11116,"rulers":11117,"111":11118,"favorable":11119,"jerome":11120,"tended":11121,"cart":11122,"factories":11123,"##eri":11124,"diplomat":11125,"valued":11126,"gravel":11127,"charitable":11128,"##try":11129,"calvin":11130,"exploring":11131,"chang":11132,"shepherd":11133,"terrace":11134,"pdf":11135,"pupil":11136,"##ural":11137,"reflects":11138,"ups":11139,"##rch":11140,"governors":11141,"shelf":11142,"depths":11143,"##nberg":11144,"trailed":11145,"crest":11146,"tackle":11147,"##nian":11148,"##ats":11149,"hatred":11150,"##kai":11151,"clare":11152,"makers":11153,"ethiopia":11154,"longtime":11155,"detected":11156,"embedded":11157,"lacking":11158,"slapped":11159,"rely":11160,"thomson":11161,"anticipation":11162,"iso":11163,"morton":11164,"successive":11165,"agnes":11166,"screenwriter":11167,"straightened":11168,"philippe":11169,"playwright":11170,"haunted":11171,"licence":11172,"iris":11173,"intentions":11174,"sutton":11175,"112":11176,"logical":11177,"correctly":11178,"##weight":11179,"branded":11180,"licked":11181,"tipped":11182,"silva":11183,"ricky":11184,"narrator":11185,"requests":11186,"##ents":11187,"greeted":11188,"supernatural":11189,"cow":11190,"##wald":11191,"lung":11192,"refusing":11193,"employer":11194,"strait":11195,"gaelic":11196,"liner":11197,"##piece":11198,"zoe":11199,"sabha":11200,"##mba":11201,"driveway":11202,"harvest":11203,"prints":11204,"bates":11205,"reluctantly":11206,"threshold":11207,"algebra":11208,"ira":11209,"wherever":11210,"coupled":11211,"240":11212,"assumption":11213,"picks":11214,"##air":11215,"designers":11216,"raids":11217,"gentlemen":11218,"##ean":11219,"roller":11220,"blowing":11221,"leipzig":11222,"locks":11223,"screw":11224,"dressing":11225,"strand":11226,"##lings":11227,"scar":11228,"dwarf":11229,"depicts":11230,"##nu":11231,"nods":11232,"##mine":11233,"differ":11234,"boris":11235,"##eur":11236,"yuan":11237,"flip":11238,"##gie":11239,"mob":11240,"invested":11241,"questioning":11242,"applying":11243,"##ture":11244,"shout":11245,"##sel":11246,"gameplay":11247,"blamed":11248,"illustrations":11249,"bothered":11250,"weakness":11251,"rehabilitation":11252,"##of":11253,"##zes":11254,"envelope":11255,"rumors":11256,"miners":11257,"leicester":11258,"subtle":11259,"kerry":11260,"##ico":11261,"ferguson":11262,"##fu":11263,"premiership":11264,"ne":11265,"##cat":11266,"bengali":11267,"prof":11268,"catches":11269,"remnants":11270,"dana":11271,"##rily":11272,"shouting":11273,"presidents":11274,"baltic":11275,"ought":11276,"ghosts":11277,"dances":11278,"sailors":11279,"shirley":11280,"fancy":11281,"dominic":11282,"##bie":11283,"madonna":11284,"##rick":11285,"bark":11286,"buttons":11287,"gymnasium":11288,"ashes":11289,"liver":11290,"toby":11291,"oath":11292,"providence":11293,"doyle":11294,"evangelical":11295,"nixon":11296,"cement":11297,"carnegie":11298,"embarked":11299,"hatch":11300,"surroundings":11301,"guarantee":11302,"needing":11303,"pirate":11304,"essence":11305,"##bee":11306,"filter":11307,"crane":11308,"hammond":11309,"projected":11310,"immune":11311,"percy":11312,"twelfth":11313,"##ult":11314,"regent":11315,"doctoral":11316,"damon":11317,"mikhail":11318,"##ichi":11319,"lu":11320,"critically":11321,"elect":11322,"realised":11323,"abortion":11324,"acute":11325,"screening":11326,"mythology":11327,"steadily":11328,"##fc":11329,"frown":11330,"nottingham":11331,"kirk":11332,"wa":11333,"minneapolis":11334,"##rra":11335,"module":11336,"algeria":11337,"mc":11338,"nautical":11339,"encounters":11340,"surprising":11341,"statues":11342,"availability":11343,"shirts":11344,"pie":11345,"alma":11346,"brows":11347,"munster":11348,"mack":11349,"soup":11350,"crater":11351,"tornado":11352,"sanskrit":11353,"cedar":11354,"explosive":11355,"bordered":11356,"dixon":11357,"planets":11358,"stamp":11359,"exam":11360,"happily":11361,"##bble":11362,"carriers":11363,"kidnapped":11364,"##vis":11365,"accommodation":11366,"emigrated":11367,"##met":11368,"knockout":11369,"correspondent":11370,"violation":11371,"profits":11372,"peaks":11373,"lang":11374,"specimen":11375,"agenda":11376,"ancestry":11377,"pottery":11378,"spelling":11379,"equations":11380,"obtaining":11381,"ki":11382,"linking":11383,"1825":11384,"debris":11385,"asylum":11386,"##20":11387,"buddhism":11388,"teddy":11389,"##ants":11390,"gazette":11391,"##nger":11392,"##sse":11393,"dental":11394,"eligibility":11395,"utc":11396,"fathers":11397,"averaged":11398,"zimbabwe":11399,"francesco":11400,"coloured":11401,"hissed":11402,"translator":11403,"lynch":11404,"mandate":11405,"humanities":11406,"mackenzie":11407,"uniforms":11408,"lin":11409,"##iana":11410,"##gio":11411,"asset":11412,"mhz":11413,"fitting":11414,"samantha":11415,"genera":11416,"wei":11417,"rim":11418,"beloved":11419,"shark":11420,"riot":11421,"entities":11422,"expressions":11423,"indo":11424,"carmen":11425,"slipping":11426,"owing":11427,"abbot":11428,"neighbor":11429,"sidney":11430,"##av":11431,"rats":11432,"recommendations":11433,"encouraging":11434,"squadrons":11435,"anticipated":11436,"commanders":11437,"conquered":11438,"##oto":11439,"donations":11440,"diagnosed":11441,"##mond":11442,"divide":11443,"##iva":11444,"guessed":11445,"decoration":11446,"vernon":11447,"auditorium":11448,"revelation":11449,"conversations":11450,"##kers":11451,"##power":11452,"herzegovina":11453,"dash":11454,"alike":11455,"protested":11456,"lateral":11457,"herman":11458,"accredited":11459,"mg":11460,"##gent":11461,"freeman":11462,"mel":11463,"fiji":11464,"crow":11465,"crimson":11466,"##rine":11467,"livestock":11468,"##pped":11469,"humanitarian":11470,"bored":11471,"oz":11472,"whip":11473,"##lene":11474,"##ali":11475,"legitimate":11476,"alter":11477,"grinning":11478,"spelled":11479,"anxious":11480,"oriental":11481,"wesley":11482,"##nin":11483,"##hole":11484,"carnival":11485,"controller":11486,"detect":11487,"##ssa":11488,"bowed":11489,"educator":11490,"kosovo":11491,"macedonia":11492,"##sin":11493,"occupy":11494,"mastering":11495,"stephanie":11496,"janeiro":11497,"para":11498,"unaware":11499,"nurses":11500,"noon":11501,"135":11502,"cam":11503,"hopefully":11504,"ranger":11505,"combine":11506,"sociology":11507,"polar":11508,"rica":11509,"##eer":11510,"neill":11511,"##sman":11512,"holocaust":11513,"##ip":11514,"doubled":11515,"lust":11516,"1828":11517,"109":11518,"decent":11519,"cooling":11520,"unveiled":11521,"##card":11522,"1829":11523,"nsw":11524,"homer":11525,"chapman":11526,"meyer":11527,"##gin":11528,"dive":11529,"mae":11530,"reagan":11531,"expertise":11532,"##gled":11533,"darwin":11534,"brooke":11535,"sided":11536,"prosecution":11537,"investigating":11538,"comprised":11539,"petroleum":11540,"genres":11541,"reluctant":11542,"differently":11543,"trilogy":11544,"johns":11545,"vegetables":11546,"corpse":11547,"highlighted":11548,"lounge":11549,"pension":11550,"unsuccessfully":11551,"elegant":11552,"aided":11553,"ivory":11554,"beatles":11555,"amelia":11556,"cain":11557,"dubai":11558,"sunny":11559,"immigrant":11560,"babe":11561,"click":11562,"##nder":11563,"underwater":11564,"pepper":11565,"combining":11566,"mumbled":11567,"atlas":11568,"horns":11569,"accessed":11570,"ballad":11571,"physicians":11572,"homeless":11573,"gestured":11574,"rpm":11575,"freak":11576,"louisville":11577,"corporations":11578,"patriots":11579,"prizes":11580,"rational":11581,"warn":11582,"modes":11583,"decorative":11584,"overnight":11585,"din":11586,"troubled":11587,"phantom":11588,"##ort":11589,"monarch":11590,"sheer":11591,"##dorf":11592,"generals":11593,"guidelines":11594,"organs":11595,"addresses":11596,"##zon":11597,"enhance":11598,"curling":11599,"parishes":11600,"cord":11601,"##kie":11602,"linux":11603,"caesar":11604,"deutsche":11605,"bavaria":11606,"##bia":11607,"coleman":11608,"cyclone":11609,"##eria":11610,"bacon":11611,"petty":11612,"##yama":11613,"##old":11614,"hampton":11615,"diagnosis":11616,"1824":11617,"throws":11618,"complexity":11619,"rita":11620,"disputed":11621,"##₃":11622,"pablo":11623,"##sch":11624,"marketed":11625,"trafficking":11626,"##ulus":11627,"examine":11628,"plague":11629,"formats":11630,"##oh":11631,"vault":11632,"faithful":11633,"##bourne":11634,"webster":11635,"##ox":11636,"highlights":11637,"##ient":11638,"##ann":11639,"phones":11640,"vacuum":11641,"sandwich":11642,"modeling":11643,"##gated":11644,"bolivia":11645,"clergy":11646,"qualities":11647,"isabel":11648,"##nas":11649,"##ars":11650,"wears":11651,"screams":11652,"reunited":11653,"annoyed":11654,"bra":11655,"##ancy":11656,"##rate":11657,"differential":11658,"transmitter":11659,"tattoo":11660,"container":11661,"poker":11662,"##och":11663,"excessive":11664,"resides":11665,"cowboys":11666,"##tum":11667,"augustus":11668,"trash":11669,"providers":11670,"statute":11671,"retreated":11672,"balcony":11673,"reversed":11674,"void":11675,"storey":11676,"preceded":11677,"masses":11678,"leap":11679,"laughs":11680,"neighborhoods":11681,"wards":11682,"schemes":11683,"falcon":11684,"santo":11685,"battlefield":11686,"pad":11687,"ronnie":11688,"thread":11689,"lesbian":11690,"venus":11691,"##dian":11692,"beg":11693,"sandstone":11694,"daylight":11695,"punched":11696,"gwen":11697,"analog":11698,"stroked":11699,"wwe":11700,"acceptable":11701,"measurements":11702,"dec":11703,"toxic":11704,"##kel":11705,"adequate":11706,"surgical":11707,"economist":11708,"parameters":11709,"varsity":11710,"##sberg":11711,"quantity":11712,"ella":11713,"##chy":11714,"##rton":11715,"countess":11716,"generating":11717,"precision":11718,"diamonds":11719,"expressway":11720,"ga":11721,"##ı":11722,"1821":11723,"uruguay":11724,"talents":11725,"galleries":11726,"expenses":11727,"scanned":11728,"colleague":11729,"outlets":11730,"ryder":11731,"lucien":11732,"##ila":11733,"paramount":11734,"##bon":11735,"syracuse":11736,"dim":11737,"fangs":11738,"gown":11739,"sweep":11740,"##sie":11741,"toyota":11742,"missionaries":11743,"websites":11744,"##nsis":11745,"sentences":11746,"adviser":11747,"val":11748,"trademark":11749,"spells":11750,"##plane":11751,"patience":11752,"starter":11753,"slim":11754,"##borg":11755,"toe":11756,"incredibly":11757,"shoots":11758,"elliot":11759,"nobility":11760,"##wyn":11761,"cowboy":11762,"endorsed":11763,"gardner":11764,"tendency":11765,"persuaded":11766,"organisms":11767,"emissions":11768,"kazakhstan":11769,"amused":11770,"boring":11771,"chips":11772,"themed":11773,"##hand":11774,"llc":11775,"constantinople":11776,"chasing":11777,"systematic":11778,"guatemala":11779,"borrowed":11780,"erin":11781,"carey":11782,"##hard":11783,"highlands":11784,"struggles":11785,"1810":11786,"##ifying":11787,"##ced":11788,"wong":11789,"exceptions":11790,"develops":11791,"enlarged":11792,"kindergarten":11793,"castro":11794,"##ern":11795,"##rina":11796,"leigh":11797,"zombie":11798,"juvenile":11799,"##most":11800,"consul":11801,"##nar":11802,"sailor":11803,"hyde":11804,"clarence":11805,"intensive":11806,"pinned":11807,"nasty":11808,"useless":11809,"jung":11810,"clayton":11811,"stuffed":11812,"exceptional":11813,"ix":11814,"apostolic":11815,"230":11816,"transactions":11817,"##dge":11818,"exempt":11819,"swinging":11820,"cove":11821,"religions":11822,"##ash":11823,"shields":11824,"dairy":11825,"bypass":11826,"190":11827,"pursuing":11828,"bug":11829,"joyce":11830,"bombay":11831,"chassis":11832,"southampton":11833,"chat":11834,"interact":11835,"redesignated":11836,"##pen":11837,"nascar":11838,"pray":11839,"salmon":11840,"rigid":11841,"regained":11842,"malaysian":11843,"grim":11844,"publicity":11845,"constituted":11846,"capturing":11847,"toilet":11848,"delegate":11849,"purely":11850,"tray":11851,"drift":11852,"loosely":11853,"striker":11854,"weakened":11855,"trinidad":11856,"mitch":11857,"itv":11858,"defines":11859,"transmitted":11860,"ming":11861,"scarlet":11862,"nodding":11863,"fitzgerald":11864,"fu":11865,"narrowly":11866,"sp":11867,"tooth":11868,"standings":11869,"virtue":11870,"##₁":11871,"##wara":11872,"##cting":11873,"chateau":11874,"gloves":11875,"lid":11876,"##nel":11877,"hurting":11878,"conservatory":11879,"##pel":11880,"sinclair":11881,"reopened":11882,"sympathy":11883,"nigerian":11884,"strode":11885,"advocated":11886,"optional":11887,"chronic":11888,"discharge":11889,"##rc":11890,"suck":11891,"compatible":11892,"laurel":11893,"stella":11894,"shi":11895,"fails":11896,"wage":11897,"dodge":11898,"128":11899,"informal":11900,"sorts":11901,"levi":11902,"buddha":11903,"villagers":11904,"##aka":11905,"chronicles":11906,"heavier":11907,"summoned":11908,"gateway":11909,"3000":11910,"eleventh":11911,"jewelry":11912,"translations":11913,"accordingly":11914,"seas":11915,"##ency":11916,"fiber":11917,"pyramid":11918,"cubic":11919,"dragging":11920,"##ista":11921,"caring":11922,"##ops":11923,"android":11924,"contacted":11925,"lunar":11926,"##dt":11927,"kai":11928,"lisbon":11929,"patted":11930,"1826":11931,"sacramento":11932,"theft":11933,"madagascar":11934,"subtropical":11935,"disputes":11936,"ta":11937,"holidays":11938,"piper":11939,"willow":11940,"mare":11941,"cane":11942,"itunes":11943,"newfoundland":11944,"benny":11945,"companions":11946,"dong":11947,"raj":11948,"observe":11949,"roar":11950,"charming":11951,"plaque":11952,"tibetan":11953,"fossils":11954,"enacted":11955,"manning":11956,"bubble":11957,"tina":11958,"tanzania":11959,"##eda":11960,"##hir":11961,"funk":11962,"swamp":11963,"deputies":11964,"cloak":11965,"ufc":11966,"scenario":11967,"par":11968,"scratch":11969,"metals":11970,"anthem":11971,"guru":11972,"engaging":11973,"specially":11974,"##boat":11975,"dialects":11976,"nineteen":11977,"cecil":11978,"duet":11979,"disability":11980,"messenger":11981,"unofficial":11982,"##lies":11983,"defunct":11984,"eds":11985,"moonlight":11986,"drainage":11987,"surname":11988,"puzzle":11989,"honda":11990,"switching":11991,"conservatives":11992,"mammals":11993,"knox":11994,"broadcaster":11995,"sidewalk":11996,"cope":11997,"##ried":11998,"benson":11999,"princes":12000,"peterson":12001,"##sal":12002,"bedford":12003,"sharks":12004,"eli":12005,"wreck":12006,"alberto":12007,"gasp":12008,"archaeology":12009,"lgbt":12010,"teaches":12011,"securities":12012,"madness":12013,"compromise":12014,"waving":12015,"coordination":12016,"davidson":12017,"visions":12018,"leased":12019,"possibilities":12020,"eighty":12021,"jun":12022,"fernandez":12023,"enthusiasm":12024,"assassin":12025,"sponsorship":12026,"reviewer":12027,"kingdoms":12028,"estonian":12029,"laboratories":12030,"##fy":12031,"##nal":12032,"applies":12033,"verb":12034,"celebrations":12035,"##zzo":12036,"rowing":12037,"lightweight":12038,"sadness":12039,"submit":12040,"mvp":12041,"balanced":12042,"dude":12043,"##vas":12044,"explicitly":12045,"metric":12046,"magnificent":12047,"mound":12048,"brett":12049,"mohammad":12050,"mistakes":12051,"irregular":12052,"##hing":12053,"##ass":12054,"sanders":12055,"betrayed":12056,"shipped":12057,"surge":12058,"##enburg":12059,"reporters":12060,"termed":12061,"georg":12062,"pity":12063,"verbal":12064,"bulls":12065,"abbreviated":12066,"enabling":12067,"appealed":12068,"##are":12069,"##atic":12070,"sicily":12071,"sting":12072,"heel":12073,"sweetheart":12074,"bart":12075,"spacecraft":12076,"brutal":12077,"monarchy":12078,"##tter":12079,"aberdeen":12080,"cameo":12081,"diane":12082,"##ub":12083,"survivor":12084,"clyde":12085,"##aries":12086,"complaint":12087,"##makers":12088,"clarinet":12089,"delicious":12090,"chilean":12091,"karnataka":12092,"coordinates":12093,"1818":12094,"panties":12095,"##rst":12096,"pretending":12097,"ar":12098,"dramatically":12099,"kiev":12100,"bella":12101,"tends":12102,"distances":12103,"113":12104,"catalog":12105,"launching":12106,"instances":12107,"telecommunications":12108,"portable":12109,"lindsay":12110,"vatican":12111,"##eim":12112,"angles":12113,"aliens":12114,"marker":12115,"stint":12116,"screens":12117,"bolton":12118,"##rne":12119,"judy":12120,"wool":12121,"benedict":12122,"plasma":12123,"europa":12124,"spark":12125,"imaging":12126,"filmmaker":12127,"swiftly":12128,"##een":12129,"contributor":12130,"##nor":12131,"opted":12132,"stamps":12133,"apologize":12134,"financing":12135,"butter":12136,"gideon":12137,"sophisticated":12138,"alignment":12139,"avery":12140,"chemicals":12141,"yearly":12142,"speculation":12143,"prominence":12144,"professionally":12145,"##ils":12146,"immortal":12147,"institutional":12148,"inception":12149,"wrists":12150,"identifying":12151,"tribunal":12152,"derives":12153,"gains":12154,"##wo":12155,"papal":12156,"preference":12157,"linguistic":12158,"vince":12159,"operative":12160,"brewery":12161,"##ont":12162,"unemployment":12163,"boyd":12164,"##ured":12165,"##outs":12166,"albeit":12167,"prophet":12168,"1813":12169,"bi":12170,"##rr":12171,"##face":12172,"##rad":12173,"quarterly":12174,"asteroid":12175,"cleaned":12176,"radius":12177,"temper":12178,"##llen":12179,"telugu":12180,"jerk":12181,"viscount":12182,"menu":12183,"##ote":12184,"glimpse":12185,"##aya":12186,"yacht":12187,"hawaiian":12188,"baden":12189,"##rl":12190,"laptop":12191,"readily":12192,"##gu":12193,"monetary":12194,"offshore":12195,"scots":12196,"watches":12197,"##yang":12198,"##arian":12199,"upgrade":12200,"needle":12201,"xbox":12202,"lea":12203,"encyclopedia":12204,"flank":12205,"fingertips":12206,"##pus":12207,"delight":12208,"teachings":12209,"confirm":12210,"roth":12211,"beaches":12212,"midway":12213,"winters":12214,"##iah":12215,"teasing":12216,"daytime":12217,"beverly":12218,"gambling":12219,"bonnie":12220,"##backs":12221,"regulated":12222,"clement":12223,"hermann":12224,"tricks":12225,"knot":12226,"##shing":12227,"##uring":12228,"##vre":12229,"detached":12230,"ecological":12231,"owed":12232,"specialty":12233,"byron":12234,"inventor":12235,"bats":12236,"stays":12237,"screened":12238,"unesco":12239,"midland":12240,"trim":12241,"affection":12242,"##ander":12243,"##rry":12244,"jess":12245,"thoroughly":12246,"feedback":12247,"##uma":12248,"chennai":12249,"strained":12250,"heartbeat":12251,"wrapping":12252,"overtime":12253,"pleaded":12254,"##sworth":12255,"mon":12256,"leisure":12257,"oclc":12258,"##tate":12259,"##ele":12260,"feathers":12261,"angelo":12262,"thirds":12263,"nuts":12264,"surveys":12265,"clever":12266,"gill":12267,"commentator":12268,"##dos":12269,"darren":12270,"rides":12271,"gibraltar":12272,"##nc":12273,"##mu":12274,"dissolution":12275,"dedication":12276,"shin":12277,"meals":12278,"saddle":12279,"elvis":12280,"reds":12281,"chaired":12282,"taller":12283,"appreciation":12284,"functioning":12285,"niece":12286,"favored":12287,"advocacy":12288,"robbie":12289,"criminals":12290,"suffolk":12291,"yugoslav":12292,"passport":12293,"constable":12294,"congressman":12295,"hastings":12296,"vera":12297,"##rov":12298,"consecrated":12299,"sparks":12300,"ecclesiastical":12301,"confined":12302,"##ovich":12303,"muller":12304,"floyd":12305,"nora":12306,"1822":12307,"paved":12308,"1827":12309,"cumberland":12310,"ned":12311,"saga":12312,"spiral":12313,"##flow":12314,"appreciated":12315,"yi":12316,"collaborative":12317,"treating":12318,"similarities":12319,"feminine":12320,"finishes":12321,"##ib":12322,"jade":12323,"import":12324,"##nse":12325,"##hot":12326,"champagne":12327,"mice":12328,"securing":12329,"celebrities":12330,"helsinki":12331,"attributes":12332,"##gos":12333,"cousins":12334,"phases":12335,"ache":12336,"lucia":12337,"gandhi":12338,"submission":12339,"vicar":12340,"spear":12341,"shine":12342,"tasmania":12343,"biting":12344,"detention":12345,"constitute":12346,"tighter":12347,"seasonal":12348,"##gus":12349,"terrestrial":12350,"matthews":12351,"##oka":12352,"effectiveness":12353,"parody":12354,"philharmonic":12355,"##onic":12356,"1816":12357,"strangers":12358,"encoded":12359,"consortium":12360,"guaranteed":12361,"regards":12362,"shifts":12363,"tortured":12364,"collision":12365,"supervisor":12366,"inform":12367,"broader":12368,"insight":12369,"theaters":12370,"armour":12371,"emeritus":12372,"blink":12373,"incorporates":12374,"mapping":12375,"##50":12376,"##ein":12377,"handball":12378,"flexible":12379,"##nta":12380,"substantially":12381,"generous":12382,"thief":12383,"##own":12384,"carr":12385,"loses":12386,"1793":12387,"prose":12388,"ucla":12389,"romeo":12390,"generic":12391,"metallic":12392,"realization":12393,"damages":12394,"mk":12395,"commissioners":12396,"zach":12397,"default":12398,"##ther":12399,"helicopters":12400,"lengthy":12401,"stems":12402,"spa":12403,"partnered":12404,"spectators":12405,"rogue":12406,"indication":12407,"penalties":12408,"teresa":12409,"1801":12410,"sen":12411,"##tric":12412,"dalton":12413,"##wich":12414,"irving":12415,"photographic":12416,"##vey":12417,"dell":12418,"deaf":12419,"peters":12420,"excluded":12421,"unsure":12422,"##vable":12423,"patterson":12424,"crawled":12425,"##zio":12426,"resided":12427,"whipped":12428,"latvia":12429,"slower":12430,"ecole":12431,"pipes":12432,"employers":12433,"maharashtra":12434,"comparable":12435,"va":12436,"textile":12437,"pageant":12438,"##gel":12439,"alphabet":12440,"binary":12441,"irrigation":12442,"chartered":12443,"choked":12444,"antoine":12445,"offs":12446,"waking":12447,"supplement":12448,"##wen":12449,"quantities":12450,"demolition":12451,"regain":12452,"locate":12453,"urdu":12454,"folks":12455,"alt":12456,"114":12457,"##mc":12458,"scary":12459,"andreas":12460,"whites":12461,"##ava":12462,"classrooms":12463,"mw":12464,"aesthetic":12465,"publishes":12466,"valleys":12467,"guides":12468,"cubs":12469,"johannes":12470,"bryant":12471,"conventions":12472,"affecting":12473,"##itt":12474,"drain":12475,"awesome":12476,"isolation":12477,"prosecutor":12478,"ambitious":12479,"apology":12480,"captive":12481,"downs":12482,"atmospheric":12483,"lorenzo":12484,"aisle":12485,"beef":12486,"foul":12487,"##onia":12488,"kidding":12489,"composite":12490,"disturbed":12491,"illusion":12492,"natives":12493,"##ffer":12494,"emi":12495,"rockets":12496,"riverside":12497,"wartime":12498,"painters":12499,"adolf":12500,"melted":12501,"##ail":12502,"uncertainty":12503,"simulation":12504,"hawks":12505,"progressed":12506,"meantime":12507,"builder":12508,"spray":12509,"breach":12510,"unhappy":12511,"regina":12512,"russians":12513,"##urg":12514,"determining":12515,"##tation":12516,"tram":12517,"1806":12518,"##quin":12519,"aging":12520,"##12":12521,"1823":12522,"garion":12523,"rented":12524,"mister":12525,"diaz":12526,"terminated":12527,"clip":12528,"1817":12529,"depend":12530,"nervously":12531,"disco":12532,"owe":12533,"defenders":12534,"shiva":12535,"notorious":12536,"disbelief":12537,"shiny":12538,"worcester":12539,"##gation":12540,"##yr":12541,"trailing":12542,"undertook":12543,"islander":12544,"belarus":12545,"limitations":12546,"watershed":12547,"fuller":12548,"overlooking":12549,"utilized":12550,"raphael":12551,"1819":12552,"synthetic":12553,"breakdown":12554,"klein":12555,"##nate":12556,"moaned":12557,"memoir":12558,"lamb":12559,"practicing":12560,"##erly":12561,"cellular":12562,"arrows":12563,"exotic":12564,"##graphy":12565,"witches":12566,"117":12567,"charted":12568,"rey":12569,"hut":12570,"hierarchy":12571,"subdivision":12572,"freshwater":12573,"giuseppe":12574,"aloud":12575,"reyes":12576,"qatar":12577,"marty":12578,"sideways":12579,"utterly":12580,"sexually":12581,"jude":12582,"prayers":12583,"mccarthy":12584,"softball":12585,"blend":12586,"damien":12587,"##gging":12588,"##metric":12589,"wholly":12590,"erupted":12591,"lebanese":12592,"negro":12593,"revenues":12594,"tasted":12595,"comparative":12596,"teamed":12597,"transaction":12598,"labeled":12599,"maori":12600,"sovereignty":12601,"parkway":12602,"trauma":12603,"gran":12604,"malay":12605,"121":12606,"advancement":12607,"descendant":12608,"2020":12609,"buzz":12610,"salvation":12611,"inventory":12612,"symbolic":12613,"##making":12614,"antarctica":12615,"mps":12616,"##gas":12617,"##bro":12618,"mohammed":12619,"myanmar":12620,"holt":12621,"submarines":12622,"tones":12623,"##lman":12624,"locker":12625,"patriarch":12626,"bangkok":12627,"emerson":12628,"remarks":12629,"predators":12630,"kin":12631,"afghan":12632,"confession":12633,"norwich":12634,"rental":12635,"emerge":12636,"advantages":12637,"##zel":12638,"rca":12639,"##hold":12640,"shortened":12641,"storms":12642,"aidan":12643,"##matic":12644,"autonomy":12645,"compliance":12646,"##quet":12647,"dudley":12648,"atp":12649,"##osis":12650,"1803":12651,"motto":12652,"documentation":12653,"summary":12654,"professors":12655,"spectacular":12656,"christina":12657,"archdiocese":12658,"flashing":12659,"innocence":12660,"remake":12661,"##dell":12662,"psychic":12663,"reef":12664,"scare":12665,"employ":12666,"rs":12667,"sticks":12668,"meg":12669,"gus":12670,"leans":12671,"##ude":12672,"accompany":12673,"bergen":12674,"tomas":12675,"##iko":12676,"doom":12677,"wages":12678,"pools":12679,"##nch":12680,"##bes":12681,"breasts":12682,"scholarly":12683,"alison":12684,"outline":12685,"brittany":12686,"breakthrough":12687,"willis":12688,"realistic":12689,"##cut":12690,"##boro":12691,"competitor":12692,"##stan":12693,"pike":12694,"picnic":12695,"icon":12696,"designing":12697,"commercials":12698,"washing":12699,"villain":12700,"skiing":12701,"micro":12702,"costumes":12703,"auburn":12704,"halted":12705,"executives":12706,"##hat":12707,"logistics":12708,"cycles":12709,"vowel":12710,"applicable":12711,"barrett":12712,"exclaimed":12713,"eurovision":12714,"eternity":12715,"ramon":12716,"##umi":12717,"##lls":12718,"modifications":12719,"sweeping":12720,"disgust":12721,"##uck":12722,"torch":12723,"aviv":12724,"ensuring":12725,"rude":12726,"dusty":12727,"sonic":12728,"donovan":12729,"outskirts":12730,"cu":12731,"pathway":12732,"##band":12733,"##gun":12734,"##lines":12735,"disciplines":12736,"acids":12737,"cadet":12738,"paired":12739,"##40":12740,"sketches":12741,"##sive":12742,"marriages":12743,"##⁺":12744,"folding":12745,"peers":12746,"slovak":12747,"implies":12748,"admired":12749,"##beck":12750,"1880s":12751,"leopold":12752,"instinct":12753,"attained":12754,"weston":12755,"megan":12756,"horace":12757,"##ination":12758,"dorsal":12759,"ingredients":12760,"evolutionary":12761,"##its":12762,"complications":12763,"deity":12764,"lethal":12765,"brushing":12766,"levy":12767,"deserted":12768,"institutes":12769,"posthumously":12770,"delivering":12771,"telescope":12772,"coronation":12773,"motivated":12774,"rapids":12775,"luc":12776,"flicked":12777,"pays":12778,"volcano":12779,"tanner":12780,"weighed":12781,"##nica":12782,"crowds":12783,"frankie":12784,"gifted":12785,"addressing":12786,"granddaughter":12787,"winding":12788,"##rna":12789,"constantine":12790,"gomez":12791,"##front":12792,"landscapes":12793,"rudolf":12794,"anthropology":12795,"slate":12796,"werewolf":12797,"##lio":12798,"astronomy":12799,"circa":12800,"rouge":12801,"dreaming":12802,"sack":12803,"knelt":12804,"drowned":12805,"naomi":12806,"prolific":12807,"tracked":12808,"freezing":12809,"herb":12810,"##dium":12811,"agony":12812,"randall":12813,"twisting":12814,"wendy":12815,"deposit":12816,"touches":12817,"vein":12818,"wheeler":12819,"##bbled":12820,"##bor":12821,"batted":12822,"retaining":12823,"tire":12824,"presently":12825,"compare":12826,"specification":12827,"daemon":12828,"nigel":12829,"##grave":12830,"merry":12831,"recommendation":12832,"czechoslovakia":12833,"sandra":12834,"ng":12835,"roma":12836,"##sts":12837,"lambert":12838,"inheritance":12839,"sheikh":12840,"winchester":12841,"cries":12842,"examining":12843,"##yle":12844,"comeback":12845,"cuisine":12846,"nave":12847,"##iv":12848,"ko":12849,"retrieve":12850,"tomatoes":12851,"barker":12852,"polished":12853,"defining":12854,"irene":12855,"lantern":12856,"personalities":12857,"begging":12858,"tract":12859,"swore":12860,"1809":12861,"175":12862,"##gic":12863,"omaha":12864,"brotherhood":12865,"##rley":12866,"haiti":12867,"##ots":12868,"exeter":12869,"##ete":12870,"##zia":12871,"steele":12872,"dumb":12873,"pearson":12874,"210":12875,"surveyed":12876,"elisabeth":12877,"trends":12878,"##ef":12879,"fritz":12880,"##rf":12881,"premium":12882,"bugs":12883,"fraction":12884,"calmly":12885,"viking":12886,"##birds":12887,"tug":12888,"inserted":12889,"unusually":12890,"##ield":12891,"confronted":12892,"distress":12893,"crashing":12894,"brent":12895,"turks":12896,"resign":12897,"##olo":12898,"cambodia":12899,"gabe":12900,"sauce":12901,"##kal":12902,"evelyn":12903,"116":12904,"extant":12905,"clusters":12906,"quarry":12907,"teenagers":12908,"luna":12909,"##lers":12910,"##ister":12911,"affiliation":12912,"drill":12913,"##ashi":12914,"panthers":12915,"scenic":12916,"libya":12917,"anita":12918,"strengthen":12919,"inscriptions":12920,"##cated":12921,"lace":12922,"sued":12923,"judith":12924,"riots":12925,"##uted":12926,"mint":12927,"##eta":12928,"preparations":12929,"midst":12930,"dub":12931,"challenger":12932,"##vich":12933,"mock":12934,"cf":12935,"displaced":12936,"wicket":12937,"breaths":12938,"enables":12939,"schmidt":12940,"analyst":12941,"##lum":12942,"ag":12943,"highlight":12944,"automotive":12945,"axe":12946,"josef":12947,"newark":12948,"sufficiently":12949,"resembles":12950,"50th":12951,"##pal":12952,"flushed":12953,"mum":12954,"traits":12955,"##ante":12956,"commodore":12957,"incomplete":12958,"warming":12959,"titular":12960,"ceremonial":12961,"ethical":12962,"118":12963,"celebrating":12964,"eighteenth":12965,"cao":12966,"lima":12967,"medalist":12968,"mobility":12969,"strips":12970,"snakes":12971,"##city":12972,"miniature":12973,"zagreb":12974,"barton":12975,"escapes":12976,"umbrella":12977,"automated":12978,"doubted":12979,"differs":12980,"cooled":12981,"georgetown":12982,"dresden":12983,"cooked":12984,"fade":12985,"wyatt":12986,"rna":12987,"jacobs":12988,"carlton":12989,"abundant":12990,"stereo":12991,"boost":12992,"madras":12993,"inning":12994,"##hia":12995,"spur":12996,"ip":12997,"malayalam":12998,"begged":12999,"osaka":13000,"groan":13001,"escaping":13002,"charging":13003,"dose":13004,"vista":13005,"##aj":13006,"bud":13007,"papa":13008,"communists":13009,"advocates":13010,"edged":13011,"tri":13012,"##cent":13013,"resemble":13014,"peaking":13015,"necklace":13016,"fried":13017,"montenegro":13018,"saxony":13019,"goose":13020,"glances":13021,"stuttgart":13022,"curator":13023,"recruit":13024,"grocery":13025,"sympathetic":13026,"##tting":13027,"##fort":13028,"127":13029,"lotus":13030,"randolph":13031,"ancestor":13032,"##rand":13033,"succeeding":13034,"jupiter":13035,"1798":13036,"macedonian":13037,"##heads":13038,"hiking":13039,"1808":13040,"handing":13041,"fischer":13042,"##itive":13043,"garbage":13044,"node":13045,"##pies":13046,"prone":13047,"singular":13048,"papua":13049,"inclined":13050,"attractions":13051,"italia":13052,"pouring":13053,"motioned":13054,"grandma":13055,"garnered":13056,"jacksonville":13057,"corp":13058,"ego":13059,"ringing":13060,"aluminum":13061,"##hausen":13062,"ordering":13063,"##foot":13064,"drawer":13065,"traders":13066,"synagogue":13067,"##play":13068,"##kawa":13069,"resistant":13070,"wandering":13071,"fragile":13072,"fiona":13073,"teased":13074,"var":13075,"hardcore":13076,"soaked":13077,"jubilee":13078,"decisive":13079,"exposition":13080,"mercer":13081,"poster":13082,"valencia":13083,"hale":13084,"kuwait":13085,"1811":13086,"##ises":13087,"##wr":13088,"##eed":13089,"tavern":13090,"gamma":13091,"122":13092,"johan":13093,"##uer":13094,"airways":13095,"amino":13096,"gil":13097,"##ury":13098,"vocational":13099,"domains":13100,"torres":13101,"##sp":13102,"generator":13103,"folklore":13104,"outcomes":13105,"##keeper":13106,"canberra":13107,"shooter":13108,"fl":13109,"beams":13110,"confrontation":13111,"##lling":13112,"##gram":13113,"feb":13114,"aligned":13115,"forestry":13116,"pipeline":13117,"jax":13118,"motorway":13119,"conception":13120,"decay":13121,"##tos":13122,"coffin":13123,"##cott":13124,"stalin":13125,"1805":13126,"escorted":13127,"minded":13128,"##nam":13129,"sitcom":13130,"purchasing":13131,"twilight":13132,"veronica":13133,"additions":13134,"passive":13135,"tensions":13136,"straw":13137,"123":13138,"frequencies":13139,"1804":13140,"refugee":13141,"cultivation":13142,"##iate":13143,"christie":13144,"clary":13145,"bulletin":13146,"crept":13147,"disposal":13148,"##rich":13149,"##zong":13150,"processor":13151,"crescent":13152,"##rol":13153,"bmw":13154,"emphasized":13155,"whale":13156,"nazis":13157,"aurora":13158,"##eng":13159,"dwelling":13160,"hauled":13161,"sponsors":13162,"toledo":13163,"mega":13164,"ideology":13165,"theatres":13166,"tessa":13167,"cerambycidae":13168,"saves":13169,"turtle":13170,"cone":13171,"suspects":13172,"kara":13173,"rusty":13174,"yelling":13175,"greeks":13176,"mozart":13177,"shades":13178,"cocked":13179,"participant":13180,"##tro":13181,"shire":13182,"spit":13183,"freeze":13184,"necessity":13185,"##cos":13186,"inmates":13187,"nielsen":13188,"councillors":13189,"loaned":13190,"uncommon":13191,"omar":13192,"peasants":13193,"botanical":13194,"offspring":13195,"daniels":13196,"formations":13197,"jokes":13198,"1794":13199,"pioneers":13200,"sigma":13201,"licensing":13202,"##sus":13203,"wheelchair":13204,"polite":13205,"1807":13206,"liquor":13207,"pratt":13208,"trustee":13209,"##uta":13210,"forewings":13211,"balloon":13212,"##zz":13213,"kilometre":13214,"camping":13215,"explicit":13216,"casually":13217,"shawn":13218,"foolish":13219,"teammates":13220,"nm":13221,"hassan":13222,"carrie":13223,"judged":13224,"satisfy":13225,"vanessa":13226,"knives":13227,"selective":13228,"cnn":13229,"flowed":13230,"##lice":13231,"eclipse":13232,"stressed":13233,"eliza":13234,"mathematician":13235,"cease":13236,"cultivated":13237,"##roy":13238,"commissions":13239,"browns":13240,"##ania":13241,"destroyers":13242,"sheridan":13243,"meadow":13244,"##rius":13245,"minerals":13246,"##cial":13247,"downstream":13248,"clash":13249,"gram":13250,"memoirs":13251,"ventures":13252,"baha":13253,"seymour":13254,"archie":13255,"midlands":13256,"edith":13257,"fare":13258,"flynn":13259,"invite":13260,"canceled":13261,"tiles":13262,"stabbed":13263,"boulder":13264,"incorporate":13265,"amended":13266,"camden":13267,"facial":13268,"mollusk":13269,"unreleased":13270,"descriptions":13271,"yoga":13272,"grabs":13273,"550":13274,"raises":13275,"ramp":13276,"shiver":13277,"##rose":13278,"coined":13279,"pioneering":13280,"tunes":13281,"qing":13282,"warwick":13283,"tops":13284,"119":13285,"melanie":13286,"giles":13287,"##rous":13288,"wandered":13289,"##inal":13290,"annexed":13291,"nov":13292,"30th":13293,"unnamed":13294,"##ished":13295,"organizational":13296,"airplane":13297,"normandy":13298,"stoke":13299,"whistle":13300,"blessing":13301,"violations":13302,"chased":13303,"holders":13304,"shotgun":13305,"##ctic":13306,"outlet":13307,"reactor":13308,"##vik":13309,"tires":13310,"tearing":13311,"shores":13312,"fortified":13313,"mascot":13314,"constituencies":13315,"nc":13316,"columnist":13317,"productive":13318,"tibet":13319,"##rta":13320,"lineage":13321,"hooked":13322,"oct":13323,"tapes":13324,"judging":13325,"cody":13326,"##gger":13327,"hansen":13328,"kashmir":13329,"triggered":13330,"##eva":13331,"solved":13332,"cliffs":13333,"##tree":13334,"resisted":13335,"anatomy":13336,"protesters":13337,"transparent":13338,"implied":13339,"##iga":13340,"injection":13341,"mattress":13342,"excluding":13343,"##mbo":13344,"defenses":13345,"helpless":13346,"devotion":13347,"##elli":13348,"growl":13349,"liberals":13350,"weber":13351,"phenomena":13352,"atoms":13353,"plug":13354,"##iff":13355,"mortality":13356,"apprentice":13357,"howe":13358,"convincing":13359,"aaa":13360,"swimmer":13361,"barber":13362,"leone":13363,"promptly":13364,"sodium":13365,"def":13366,"nowadays":13367,"arise":13368,"##oning":13369,"gloucester":13370,"corrected":13371,"dignity":13372,"norm":13373,"erie":13374,"##ders":13375,"elders":13376,"evacuated":13377,"sylvia":13378,"compression":13379,"##yar":13380,"hartford":13381,"pose":13382,"backpack":13383,"reasoning":13384,"accepts":13385,"24th":13386,"wipe":13387,"millimetres":13388,"marcel":13389,"##oda":13390,"dodgers":13391,"albion":13392,"1790":13393,"overwhelmed":13394,"aerospace":13395,"oaks":13396,"1795":13397,"showcase":13398,"acknowledge":13399,"recovering":13400,"nolan":13401,"ashe":13402,"hurts":13403,"geology":13404,"fashioned":13405,"disappearance":13406,"farewell":13407,"swollen":13408,"shrug":13409,"marquis":13410,"wimbledon":13411,"124":13412,"rue":13413,"1792":13414,"commemorate":13415,"reduces":13416,"experiencing":13417,"inevitable":13418,"calcutta":13419,"intel":13420,"##court":13421,"murderer":13422,"sticking":13423,"fisheries":13424,"imagery":13425,"bloom":13426,"280":13427,"brake":13428,"##inus":13429,"gustav":13430,"hesitation":13431,"memorable":13432,"po":13433,"viral":13434,"beans":13435,"accidents":13436,"tunisia":13437,"antenna":13438,"spilled":13439,"consort":13440,"treatments":13441,"aye":13442,"perimeter":13443,"##gard":13444,"donation":13445,"hostage":13446,"migrated":13447,"banker":13448,"addiction":13449,"apex":13450,"lil":13451,"trout":13452,"##ously":13453,"conscience":13454,"##nova":13455,"rams":13456,"sands":13457,"genome":13458,"passionate":13459,"troubles":13460,"##lets":13461,"##set":13462,"amid":13463,"##ibility":13464,"##ret":13465,"higgins":13466,"exceed":13467,"vikings":13468,"##vie":13469,"payne":13470,"##zan":13471,"muscular":13472,"##ste":13473,"defendant":13474,"sucking":13475,"##wal":13476,"ibrahim":13477,"fuselage":13478,"claudia":13479,"vfl":13480,"europeans":13481,"snails":13482,"interval":13483,"##garh":13484,"preparatory":13485,"statewide":13486,"tasked":13487,"lacrosse":13488,"viktor":13489,"##lation":13490,"angola":13491,"##hra":13492,"flint":13493,"implications":13494,"employs":13495,"teens":13496,"patrons":13497,"stall":13498,"weekends":13499,"barriers":13500,"scrambled":13501,"nucleus":13502,"tehran":13503,"jenna":13504,"parsons":13505,"lifelong":13506,"robots":13507,"displacement":13508,"5000":13509,"##bles":13510,"precipitation":13511,"##gt":13512,"knuckles":13513,"clutched":13514,"1802":13515,"marrying":13516,"ecology":13517,"marx":13518,"accusations":13519,"declare":13520,"scars":13521,"kolkata":13522,"mat":13523,"meadows":13524,"bermuda":13525,"skeleton":13526,"finalists":13527,"vintage":13528,"crawl":13529,"coordinate":13530,"affects":13531,"subjected":13532,"orchestral":13533,"mistaken":13534,"##tc":13535,"mirrors":13536,"dipped":13537,"relied":13538,"260":13539,"arches":13540,"candle":13541,"##nick":13542,"incorporating":13543,"wildly":13544,"fond":13545,"basilica":13546,"owl":13547,"fringe":13548,"rituals":13549,"whispering":13550,"stirred":13551,"feud":13552,"tertiary":13553,"slick":13554,"goat":13555,"honorable":13556,"whereby":13557,"skip":13558,"ricardo":13559,"stripes":13560,"parachute":13561,"adjoining":13562,"submerged":13563,"synthesizer":13564,"##gren":13565,"intend":13566,"positively":13567,"ninety":13568,"phi":13569,"beaver":13570,"partition":13571,"fellows":13572,"alexis":13573,"prohibition":13574,"carlisle":13575,"bizarre":13576,"fraternity":13577,"##bre":13578,"doubts":13579,"icy":13580,"cbc":13581,"aquatic":13582,"sneak":13583,"sonny":13584,"combines":13585,"airports":13586,"crude":13587,"supervised":13588,"spatial":13589,"merge":13590,"alfonso":13591,"##bic":13592,"corrupt":13593,"scan":13594,"undergo":13595,"##ams":13596,"disabilities":13597,"colombian":13598,"comparing":13599,"dolphins":13600,"perkins":13601,"##lish":13602,"reprinted":13603,"unanimous":13604,"bounced":13605,"hairs":13606,"underworld":13607,"midwest":13608,"semester":13609,"bucket":13610,"paperback":13611,"miniseries":13612,"coventry":13613,"demise":13614,"##leigh":13615,"demonstrations":13616,"sensor":13617,"rotating":13618,"yan":13619,"##hler":13620,"arrange":13621,"soils":13622,"##idge":13623,"hyderabad":13624,"labs":13625,"##dr":13626,"brakes":13627,"grandchildren":13628,"##nde":13629,"negotiated":13630,"rover":13631,"ferrari":13632,"continuation":13633,"directorate":13634,"augusta":13635,"stevenson":13636,"counterpart":13637,"gore":13638,"##rda":13639,"nursery":13640,"rican":13641,"ave":13642,"collectively":13643,"broadly":13644,"pastoral":13645,"repertoire":13646,"asserted":13647,"discovering":13648,"nordic":13649,"styled":13650,"fiba":13651,"cunningham":13652,"harley":13653,"middlesex":13654,"survives":13655,"tumor":13656,"tempo":13657,"zack":13658,"aiming":13659,"lok":13660,"urgent":13661,"##rade":13662,"##nto":13663,"devils":13664,"##ement":13665,"contractor":13666,"turin":13667,"##wl":13668,"##ool":13669,"bliss":13670,"repaired":13671,"simmons":13672,"moan":13673,"astronomical":13674,"cr":13675,"negotiate":13676,"lyric":13677,"1890s":13678,"lara":13679,"bred":13680,"clad":13681,"angus":13682,"pbs":13683,"##ience":13684,"engineered":13685,"posed":13686,"##lk":13687,"hernandez":13688,"possessions":13689,"elbows":13690,"psychiatric":13691,"strokes":13692,"confluence":13693,"electorate":13694,"lifts":13695,"campuses":13696,"lava":13697,"alps":13698,"##ep":13699,"##ution":13700,"##date":13701,"physicist":13702,"woody":13703,"##page":13704,"##ographic":13705,"##itis":13706,"juliet":13707,"reformation":13708,"sparhawk":13709,"320":13710,"complement":13711,"suppressed":13712,"jewel":13713,"##½":13714,"floated":13715,"##kas":13716,"continuity":13717,"sadly":13718,"##ische":13719,"inability":13720,"melting":13721,"scanning":13722,"paula":13723,"flour":13724,"judaism":13725,"safer":13726,"vague":13727,"##lm":13728,"solving":13729,"curb":13730,"##stown":13731,"financially":13732,"gable":13733,"bees":13734,"expired":13735,"miserable":13736,"cassidy":13737,"dominion":13738,"1789":13739,"cupped":13740,"145":13741,"robbery":13742,"facto":13743,"amos":13744,"warden":13745,"resume":13746,"tallest":13747,"marvin":13748,"ing":13749,"pounded":13750,"usd":13751,"declaring":13752,"gasoline":13753,"##aux":13754,"darkened":13755,"270":13756,"650":13757,"sophomore":13758,"##mere":13759,"erection":13760,"gossip":13761,"televised":13762,"risen":13763,"dial":13764,"##eu":13765,"pillars":13766,"##link":13767,"passages":13768,"profound":13769,"##tina":13770,"arabian":13771,"ashton":13772,"silicon":13773,"nail":13774,"##ead":13775,"##lated":13776,"##wer":13777,"##hardt":13778,"fleming":13779,"firearms":13780,"ducked":13781,"circuits":13782,"blows":13783,"waterloo":13784,"titans":13785,"##lina":13786,"atom":13787,"fireplace":13788,"cheshire":13789,"financed":13790,"activation":13791,"algorithms":13792,"##zzi":13793,"constituent":13794,"catcher":13795,"cherokee":13796,"partnerships":13797,"sexuality":13798,"platoon":13799,"tragic":13800,"vivian":13801,"guarded":13802,"whiskey":13803,"meditation":13804,"poetic":13805,"##late":13806,"##nga":13807,"##ake":13808,"porto":13809,"listeners":13810,"dominance":13811,"kendra":13812,"mona":13813,"chandler":13814,"factions":13815,"22nd":13816,"salisbury":13817,"attitudes":13818,"derivative":13819,"##ido":13820,"##haus":13821,"intake":13822,"paced":13823,"javier":13824,"illustrator":13825,"barrels":13826,"bias":13827,"cockpit":13828,"burnett":13829,"dreamed":13830,"ensuing":13831,"##anda":13832,"receptors":13833,"someday":13834,"hawkins":13835,"mattered":13836,"##lal":13837,"slavic":13838,"1799":13839,"jesuit":13840,"cameroon":13841,"wasted":13842,"tai":13843,"wax":13844,"lowering":13845,"victorious":13846,"freaking":13847,"outright":13848,"hancock":13849,"librarian":13850,"sensing":13851,"bald":13852,"calcium":13853,"myers":13854,"tablet":13855,"announcing":13856,"barack":13857,"shipyard":13858,"pharmaceutical":13859,"##uan":13860,"greenwich":13861,"flush":13862,"medley":13863,"patches":13864,"wolfgang":13865,"pt":13866,"speeches":13867,"acquiring":13868,"exams":13869,"nikolai":13870,"##gg":13871,"hayden":13872,"kannada":13873,"##type":13874,"reilly":13875,"##pt":13876,"waitress":13877,"abdomen":13878,"devastated":13879,"capped":13880,"pseudonym":13881,"pharmacy":13882,"fulfill":13883,"paraguay":13884,"1796":13885,"clicked":13886,"##trom":13887,"archipelago":13888,"syndicated":13889,"##hman":13890,"lumber":13891,"orgasm":13892,"rejection":13893,"clifford":13894,"lorraine":13895,"advent":13896,"mafia":13897,"rodney":13898,"brock":13899,"##ght":13900,"##used":13901,"##elia":13902,"cassette":13903,"chamberlain":13904,"despair":13905,"mongolia":13906,"sensors":13907,"developmental":13908,"upstream":13909,"##eg":13910,"##alis":13911,"spanning":13912,"165":13913,"trombone":13914,"basque":13915,"seeded":13916,"interred":13917,"renewable":13918,"rhys":13919,"leapt":13920,"revision":13921,"molecule":13922,"##ages":13923,"chord":13924,"vicious":13925,"nord":13926,"shivered":13927,"23rd":13928,"arlington":13929,"debts":13930,"corpus":13931,"sunrise":13932,"bays":13933,"blackburn":13934,"centimetres":13935,"##uded":13936,"shuddered":13937,"gm":13938,"strangely":13939,"gripping":13940,"cartoons":13941,"isabelle":13942,"orbital":13943,"##ppa":13944,"seals":13945,"proving":13946,"##lton":13947,"refusal":13948,"strengthened":13949,"bust":13950,"assisting":13951,"baghdad":13952,"batsman":13953,"portrayal":13954,"mara":13955,"pushes":13956,"spears":13957,"og":13958,"##cock":13959,"reside":13960,"nathaniel":13961,"brennan":13962,"1776":13963,"confirmation":13964,"caucus":13965,"##worthy":13966,"markings":13967,"yemen":13968,"nobles":13969,"ku":13970,"lazy":13971,"viewer":13972,"catalan":13973,"encompasses":13974,"sawyer":13975,"##fall":13976,"sparked":13977,"substances":13978,"patents":13979,"braves":13980,"arranger":13981,"evacuation":13982,"sergio":13983,"persuade":13984,"dover":13985,"tolerance":13986,"penguin":13987,"cum":13988,"jockey":13989,"insufficient":13990,"townships":13991,"occupying":13992,"declining":13993,"plural":13994,"processed":13995,"projection":13996,"puppet":13997,"flanders":13998,"introduces":13999,"liability":14000,"##yon":14001,"gymnastics":14002,"antwerp":14003,"taipei":14004,"hobart":14005,"candles":14006,"jeep":14007,"wes":14008,"observers":14009,"126":14010,"chaplain":14011,"bundle":14012,"glorious":14013,"##hine":14014,"hazel":14015,"flung":14016,"sol":14017,"excavations":14018,"dumped":14019,"stares":14020,"sh":14021,"bangalore":14022,"triangular":14023,"icelandic":14024,"intervals":14025,"expressing":14026,"turbine":14027,"##vers":14028,"songwriting":14029,"crafts":14030,"##igo":14031,"jasmine":14032,"ditch":14033,"rite":14034,"##ways":14035,"entertaining":14036,"comply":14037,"sorrow":14038,"wrestlers":14039,"basel":14040,"emirates":14041,"marian":14042,"rivera":14043,"helpful":14044,"##some":14045,"caution":14046,"downward":14047,"networking":14048,"##atory":14049,"##tered":14050,"darted":14051,"genocide":14052,"emergence":14053,"replies":14054,"specializing":14055,"spokesman":14056,"convenient":14057,"unlocked":14058,"fading":14059,"augustine":14060,"concentrations":14061,"resemblance":14062,"elijah":14063,"investigator":14064,"andhra":14065,"##uda":14066,"promotes":14067,"bean":14068,"##rrell":14069,"fleeing":14070,"wan":14071,"simone":14072,"announcer":14073,"##ame":14074,"##bby":14075,"lydia":14076,"weaver":14077,"132":14078,"residency":14079,"modification":14080,"##fest":14081,"stretches":14082,"##ast":14083,"alternatively":14084,"nat":14085,"lowe":14086,"lacks":14087,"##ented":14088,"pam":14089,"tile":14090,"concealed":14091,"inferior":14092,"abdullah":14093,"residences":14094,"tissues":14095,"vengeance":14096,"##ided":14097,"moisture":14098,"peculiar":14099,"groove":14100,"zip":14101,"bologna":14102,"jennings":14103,"ninja":14104,"oversaw":14105,"zombies":14106,"pumping":14107,"batch":14108,"livingston":14109,"emerald":14110,"installations":14111,"1797":14112,"peel":14113,"nitrogen":14114,"rama":14115,"##fying":14116,"##star":14117,"schooling":14118,"strands":14119,"responding":14120,"werner":14121,"##ost":14122,"lime":14123,"casa":14124,"accurately":14125,"targeting":14126,"##rod":14127,"underway":14128,"##uru":14129,"hemisphere":14130,"lester":14131,"##yard":14132,"occupies":14133,"2d":14134,"griffith":14135,"angrily":14136,"reorganized":14137,"##owing":14138,"courtney":14139,"deposited":14140,"##dd":14141,"##30":14142,"estadio":14143,"##ifies":14144,"dunn":14145,"exiled":14146,"##ying":14147,"checks":14148,"##combe":14149,"##о":14150,"##fly":14151,"successes":14152,"unexpectedly":14153,"blu":14154,"assessed":14155,"##flower":14156,"##ه":14157,"observing":14158,"sacked":14159,"spiders":14160,"kn":14161,"##tail":14162,"mu":14163,"nodes":14164,"prosperity":14165,"audrey":14166,"divisional":14167,"155":14168,"broncos":14169,"tangled":14170,"adjust":14171,"feeds":14172,"erosion":14173,"paolo":14174,"surf":14175,"directory":14176,"snatched":14177,"humid":14178,"admiralty":14179,"screwed":14180,"gt":14181,"reddish":14182,"##nese":14183,"modules":14184,"trench":14185,"lamps":14186,"bind":14187,"leah":14188,"bucks":14189,"competes":14190,"##nz":14191,"##form":14192,"transcription":14193,"##uc":14194,"isles":14195,"violently":14196,"clutching":14197,"pga":14198,"cyclist":14199,"inflation":14200,"flats":14201,"ragged":14202,"unnecessary":14203,"##hian":14204,"stubborn":14205,"coordinated":14206,"harriet":14207,"baba":14208,"disqualified":14209,"330":14210,"insect":14211,"wolfe":14212,"##fies":14213,"reinforcements":14214,"rocked":14215,"duel":14216,"winked":14217,"embraced":14218,"bricks":14219,"##raj":14220,"hiatus":14221,"defeats":14222,"pending":14223,"brightly":14224,"jealousy":14225,"##xton":14226,"##hm":14227,"##uki":14228,"lena":14229,"gdp":14230,"colorful":14231,"##dley":14232,"stein":14233,"kidney":14234,"##shu":14235,"underwear":14236,"wanderers":14237,"##haw":14238,"##icus":14239,"guardians":14240,"m³":14241,"roared":14242,"habits":14243,"##wise":14244,"permits":14245,"gp":14246,"uranium":14247,"punished":14248,"disguise":14249,"bundesliga":14250,"elise":14251,"dundee":14252,"erotic":14253,"partisan":14254,"pi":14255,"collectors":14256,"float":14257,"individually":14258,"rendering":14259,"behavioral":14260,"bucharest":14261,"ser":14262,"hare":14263,"valerie":14264,"corporal":14265,"nutrition":14266,"proportional":14267,"##isa":14268,"immense":14269,"##kis":14270,"pavement":14271,"##zie":14272,"##eld":14273,"sutherland":14274,"crouched":14275,"1775":14276,"##lp":14277,"suzuki":14278,"trades":14279,"endurance":14280,"operas":14281,"crosby":14282,"prayed":14283,"priory":14284,"rory":14285,"socially":14286,"##urn":14287,"gujarat":14288,"##pu":14289,"walton":14290,"cube":14291,"pasha":14292,"privilege":14293,"lennon":14294,"floods":14295,"thorne":14296,"waterfall":14297,"nipple":14298,"scouting":14299,"approve":14300,"##lov":14301,"minorities":14302,"voter":14303,"dwight":14304,"extensions":14305,"assure":14306,"ballroom":14307,"slap":14308,"dripping":14309,"privileges":14310,"rejoined":14311,"confessed":14312,"demonstrating":14313,"patriotic":14314,"yell":14315,"investor":14316,"##uth":14317,"pagan":14318,"slumped":14319,"squares":14320,"##cle":14321,"##kins":14322,"confront":14323,"bert":14324,"embarrassment":14325,"##aid":14326,"aston":14327,"urging":14328,"sweater":14329,"starr":14330,"yuri":14331,"brains":14332,"williamson":14333,"commuter":14334,"mortar":14335,"structured":14336,"selfish":14337,"exports":14338,"##jon":14339,"cds":14340,"##him":14341,"unfinished":14342,"##rre":14343,"mortgage":14344,"destinations":14345,"##nagar":14346,"canoe":14347,"solitary":14348,"buchanan":14349,"delays":14350,"magistrate":14351,"fk":14352,"##pling":14353,"motivation":14354,"##lier":14355,"##vier":14356,"recruiting":14357,"assess":14358,"##mouth":14359,"malik":14360,"antique":14361,"1791":14362,"pius":14363,"rahman":14364,"reich":14365,"tub":14366,"zhou":14367,"smashed":14368,"airs":14369,"galway":14370,"xii":14371,"conditioning":14372,"honduras":14373,"discharged":14374,"dexter":14375,"##pf":14376,"lionel":14377,"129":14378,"debates":14379,"lemon":14380,"tiffany":14381,"volunteered":14382,"dom":14383,"dioxide":14384,"procession":14385,"devi":14386,"sic":14387,"tremendous":14388,"advertisements":14389,"colts":14390,"transferring":14391,"verdict":14392,"hanover":14393,"decommissioned":14394,"utter":14395,"relate":14396,"pac":14397,"racism":14398,"##top":14399,"beacon":14400,"limp":14401,"similarity":14402,"terra":14403,"occurrence":14404,"ant":14405,"##how":14406,"becky":14407,"capt":14408,"updates":14409,"armament":14410,"richie":14411,"pal":14412,"##graph":14413,"halloween":14414,"mayo":14415,"##ssen":14416,"##bone":14417,"cara":14418,"serena":14419,"fcc":14420,"dolls":14421,"obligations":14422,"##dling":14423,"violated":14424,"lafayette":14425,"jakarta":14426,"exploitation":14427,"##ime":14428,"infamous":14429,"iconic":14430,"##lah":14431,"##park":14432,"kitty":14433,"moody":14434,"reginald":14435,"dread":14436,"spill":14437,"crystals":14438,"olivier":14439,"modeled":14440,"bluff":14441,"equilibrium":14442,"separating":14443,"notices":14444,"ordnance":14445,"extinction":14446,"onset":14447,"cosmic":14448,"attachment":14449,"sammy":14450,"expose":14451,"privy":14452,"anchored":14453,"##bil":14454,"abbott":14455,"admits":14456,"bending":14457,"baritone":14458,"emmanuel":14459,"policeman":14460,"vaughan":14461,"winged":14462,"climax":14463,"dresses":14464,"denny":14465,"polytechnic":14466,"mohamed":14467,"burmese":14468,"authentic":14469,"nikki":14470,"genetics":14471,"grandparents":14472,"homestead":14473,"gaza":14474,"postponed":14475,"metacritic":14476,"una":14477,"##sby":14478,"##bat":14479,"unstable":14480,"dissertation":14481,"##rial":14482,"##cian":14483,"curls":14484,"obscure":14485,"uncovered":14486,"bronx":14487,"praying":14488,"disappearing":14489,"##hoe":14490,"prehistoric":14491,"coke":14492,"turret":14493,"mutations":14494,"nonprofit":14495,"pits":14496,"monaco":14497,"##ي":14498,"##usion":14499,"prominently":14500,"dispatched":14501,"podium":14502,"##mir":14503,"uci":14504,"##uation":14505,"133":14506,"fortifications":14507,"birthplace":14508,"kendall":14509,"##lby":14510,"##oll":14511,"preacher":14512,"rack":14513,"goodman":14514,"##rman":14515,"persistent":14516,"##ott":14517,"countless":14518,"jaime":14519,"recorder":14520,"lexington":14521,"persecution":14522,"jumps":14523,"renewal":14524,"wagons":14525,"##11":14526,"crushing":14527,"##holder":14528,"decorations":14529,"##lake":14530,"abundance":14531,"wrath":14532,"laundry":14533,"£1":14534,"garde":14535,"##rp":14536,"jeanne":14537,"beetles":14538,"peasant":14539,"##sl":14540,"splitting":14541,"caste":14542,"sergei":14543,"##rer":14544,"##ema":14545,"scripts":14546,"##ively":14547,"rub":14548,"satellites":14549,"##vor":14550,"inscribed":14551,"verlag":14552,"scrapped":14553,"gale":14554,"packages":14555,"chick":14556,"potato":14557,"slogan":14558,"kathleen":14559,"arabs":14560,"##culture":14561,"counterparts":14562,"reminiscent":14563,"choral":14564,"##tead":14565,"rand":14566,"retains":14567,"bushes":14568,"dane":14569,"accomplish":14570,"courtesy":14571,"closes":14572,"##oth":14573,"slaughter":14574,"hague":14575,"krakow":14576,"lawson":14577,"tailed":14578,"elias":14579,"ginger":14580,"##ttes":14581,"canopy":14582,"betrayal":14583,"rebuilding":14584,"turf":14585,"##hof":14586,"frowning":14587,"allegiance":14588,"brigades":14589,"kicks":14590,"rebuild":14591,"polls":14592,"alias":14593,"nationalism":14594,"td":14595,"rowan":14596,"audition":14597,"bowie":14598,"fortunately":14599,"recognizes":14600,"harp":14601,"dillon":14602,"horrified":14603,"##oro":14604,"renault":14605,"##tics":14606,"ropes":14607,"##α":14608,"presumed":14609,"rewarded":14610,"infrared":14611,"wiping":14612,"accelerated":14613,"illustration":14614,"##rid":14615,"presses":14616,"practitioners":14617,"badminton":14618,"##iard":14619,"detained":14620,"##tera":14621,"recognizing":14622,"relates":14623,"misery":14624,"##sies":14625,"##tly":14626,"reproduction":14627,"piercing":14628,"potatoes":14629,"thornton":14630,"esther":14631,"manners":14632,"hbo":14633,"##aan":14634,"ours":14635,"bullshit":14636,"ernie":14637,"perennial":14638,"sensitivity":14639,"illuminated":14640,"rupert":14641,"##jin":14642,"##iss":14643,"##ear":14644,"rfc":14645,"nassau":14646,"##dock":14647,"staggered":14648,"socialism":14649,"##haven":14650,"appointments":14651,"nonsense":14652,"prestige":14653,"sharma":14654,"haul":14655,"##tical":14656,"solidarity":14657,"gps":14658,"##ook":14659,"##rata":14660,"igor":14661,"pedestrian":14662,"##uit":14663,"baxter":14664,"tenants":14665,"wires":14666,"medication":14667,"unlimited":14668,"guiding":14669,"impacts":14670,"diabetes":14671,"##rama":14672,"sasha":14673,"pas":14674,"clive":14675,"extraction":14676,"131":14677,"continually":14678,"constraints":14679,"##bilities":14680,"sonata":14681,"hunted":14682,"sixteenth":14683,"chu":14684,"planting":14685,"quote":14686,"mayer":14687,"pretended":14688,"abs":14689,"spat":14690,"##hua":14691,"ceramic":14692,"##cci":14693,"curtains":14694,"pigs":14695,"pitching":14696,"##dad":14697,"latvian":14698,"sore":14699,"dayton":14700,"##sted":14701,"##qi":14702,"patrols":14703,"slice":14704,"playground":14705,"##nted":14706,"shone":14707,"stool":14708,"apparatus":14709,"inadequate":14710,"mates":14711,"treason":14712,"##ija":14713,"desires":14714,"##liga":14715,"##croft":14716,"somalia":14717,"laurent":14718,"mir":14719,"leonardo":14720,"oracle":14721,"grape":14722,"obliged":14723,"chevrolet":14724,"thirteenth":14725,"stunning":14726,"enthusiastic":14727,"##ede":14728,"accounted":14729,"concludes":14730,"currents":14731,"basil":14732,"##kovic":14733,"drought":14734,"##rica":14735,"mai":14736,"##aire":14737,"shove":14738,"posting":14739,"##shed":14740,"pilgrimage":14741,"humorous":14742,"packing":14743,"fry":14744,"pencil":14745,"wines":14746,"smells":14747,"144":14748,"marilyn":14749,"aching":14750,"newest":14751,"clung":14752,"bon":14753,"neighbours":14754,"sanctioned":14755,"##pie":14756,"mug":14757,"##stock":14758,"drowning":14759,"##mma":14760,"hydraulic":14761,"##vil":14762,"hiring":14763,"reminder":14764,"lilly":14765,"investigators":14766,"##ncies":14767,"sour":14768,"##eous":14769,"compulsory":14770,"packet":14771,"##rion":14772,"##graphic":14773,"##elle":14774,"cannes":14775,"##inate":14776,"depressed":14777,"##rit":14778,"heroic":14779,"importantly":14780,"theresa":14781,"##tled":14782,"conway":14783,"saturn":14784,"marginal":14785,"rae":14786,"##xia":14787,"corresponds":14788,"royce":14789,"pact":14790,"jasper":14791,"explosives":14792,"packaging":14793,"aluminium":14794,"##ttered":14795,"denotes":14796,"rhythmic":14797,"spans":14798,"assignments":14799,"hereditary":14800,"outlined":14801,"originating":14802,"sundays":14803,"lad":14804,"reissued":14805,"greeting":14806,"beatrice":14807,"##dic":14808,"pillar":14809,"marcos":14810,"plots":14811,"handbook":14812,"alcoholic":14813,"judiciary":14814,"avant":14815,"slides":14816,"extract":14817,"masculine":14818,"blur":14819,"##eum":14820,"##force":14821,"homage":14822,"trembled":14823,"owens":14824,"hymn":14825,"trey":14826,"omega":14827,"signaling":14828,"socks":14829,"accumulated":14830,"reacted":14831,"attic":14832,"theo":14833,"lining":14834,"angie":14835,"distraction":14836,"primera":14837,"talbot":14838,"##key":14839,"1200":14840,"ti":14841,"creativity":14842,"billed":14843,"##hey":14844,"deacon":14845,"eduardo":14846,"identifies":14847,"proposition":14848,"dizzy":14849,"gunner":14850,"hogan":14851,"##yam":14852,"##pping":14853,"##hol":14854,"ja":14855,"##chan":14856,"jensen":14857,"reconstructed":14858,"##berger":14859,"clearance":14860,"darius":14861,"##nier":14862,"abe":14863,"harlem":14864,"plea":14865,"dei":14866,"circled":14867,"emotionally":14868,"notation":14869,"fascist":14870,"neville":14871,"exceeded":14872,"upwards":14873,"viable":14874,"ducks":14875,"##fo":14876,"workforce":14877,"racer":14878,"limiting":14879,"shri":14880,"##lson":14881,"possesses":14882,"1600":14883,"kerr":14884,"moths":14885,"devastating":14886,"laden":14887,"disturbing":14888,"locking":14889,"##cture":14890,"gal":14891,"fearing":14892,"accreditation":14893,"flavor":14894,"aide":14895,"1870s":14896,"mountainous":14897,"##baum":14898,"melt":14899,"##ures":14900,"motel":14901,"texture":14902,"servers":14903,"soda":14904,"##mb":14905,"herd":14906,"##nium":14907,"erect":14908,"puzzled":14909,"hum":14910,"peggy":14911,"examinations":14912,"gould":14913,"testified":14914,"geoff":14915,"ren":14916,"devised":14917,"sacks":14918,"##law":14919,"denial":14920,"posters":14921,"grunted":14922,"cesar":14923,"tutor":14924,"ec":14925,"gerry":14926,"offerings":14927,"byrne":14928,"falcons":14929,"combinations":14930,"ct":14931,"incoming":14932,"pardon":14933,"rocking":14934,"26th":14935,"avengers":14936,"flared":14937,"mankind":14938,"seller":14939,"uttar":14940,"loch":14941,"nadia":14942,"stroking":14943,"exposing":14944,"##hd":14945,"fertile":14946,"ancestral":14947,"instituted":14948,"##has":14949,"noises":14950,"prophecy":14951,"taxation":14952,"eminent":14953,"vivid":14954,"pol":14955,"##bol":14956,"dart":14957,"indirect":14958,"multimedia":14959,"notebook":14960,"upside":14961,"displaying":14962,"adrenaline":14963,"referenced":14964,"geometric":14965,"##iving":14966,"progression":14967,"##ddy":14968,"blunt":14969,"announce":14970,"##far":14971,"implementing":14972,"##lav":14973,"aggression":14974,"liaison":14975,"cooler":14976,"cares":14977,"headache":14978,"plantations":14979,"gorge":14980,"dots":14981,"impulse":14982,"thickness":14983,"ashamed":14984,"averaging":14985,"kathy":14986,"obligation":14987,"precursor":14988,"137":14989,"fowler":14990,"symmetry":14991,"thee":14992,"225":14993,"hears":14994,"##rai":14995,"undergoing":14996,"ads":14997,"butcher":14998,"bowler":14999,"##lip":15000,"cigarettes":15001,"subscription":15002,"goodness":15003,"##ically":15004,"browne":15005,"##hos":15006,"##tech":15007,"kyoto":15008,"donor":15009,"##erty":15010,"damaging":15011,"friction":15012,"drifting":15013,"expeditions":15014,"hardened":15015,"prostitution":15016,"152":15017,"fauna":15018,"blankets":15019,"claw":15020,"tossing":15021,"snarled":15022,"butterflies":15023,"recruits":15024,"investigative":15025,"coated":15026,"healed":15027,"138":15028,"communal":15029,"hai":15030,"xiii":15031,"academics":15032,"boone":15033,"psychologist":15034,"restless":15035,"lahore":15036,"stephens":15037,"mba":15038,"brendan":15039,"foreigners":15040,"printer":15041,"##pc":15042,"ached":15043,"explode":15044,"27th":15045,"deed":15046,"scratched":15047,"dared":15048,"##pole":15049,"cardiac":15050,"1780":15051,"okinawa":15052,"proto":15053,"commando":15054,"compelled":15055,"oddly":15056,"electrons":15057,"##base":15058,"replica":15059,"thanksgiving":15060,"##rist":15061,"sheila":15062,"deliberate":15063,"stafford":15064,"tidal":15065,"representations":15066,"hercules":15067,"ou":15068,"##path":15069,"##iated":15070,"kidnapping":15071,"lenses":15072,"##tling":15073,"deficit":15074,"samoa":15075,"mouths":15076,"consuming":15077,"computational":15078,"maze":15079,"granting":15080,"smirk":15081,"razor":15082,"fixture":15083,"ideals":15084,"inviting":15085,"aiden":15086,"nominal":15087,"##vs":15088,"issuing":15089,"julio":15090,"pitt":15091,"ramsey":15092,"docks":15093,"##oss":15094,"exhaust":15095,"##owed":15096,"bavarian":15097,"draped":15098,"anterior":15099,"mating":15100,"ethiopian":15101,"explores":15102,"noticing":15103,"##nton":15104,"discarded":15105,"convenience":15106,"hoffman":15107,"endowment":15108,"beasts":15109,"cartridge":15110,"mormon":15111,"paternal":15112,"probe":15113,"sleeves":15114,"interfere":15115,"lump":15116,"deadline":15117,"##rail":15118,"jenks":15119,"bulldogs":15120,"scrap":15121,"alternating":15122,"justified":15123,"reproductive":15124,"nam":15125,"seize":15126,"descending":15127,"secretariat":15128,"kirby":15129,"coupe":15130,"grouped":15131,"smash":15132,"panther":15133,"sedan":15134,"tapping":15135,"##18":15136,"lola":15137,"cheer":15138,"germanic":15139,"unfortunate":15140,"##eter":15141,"unrelated":15142,"##fan":15143,"subordinate":15144,"##sdale":15145,"suzanne":15146,"advertisement":15147,"##ility":15148,"horsepower":15149,"##lda":15150,"cautiously":15151,"discourse":15152,"luigi":15153,"##mans":15154,"##fields":15155,"noun":15156,"prevalent":15157,"mao":15158,"schneider":15159,"everett":15160,"surround":15161,"governorate":15162,"kira":15163,"##avia":15164,"westward":15165,"##take":15166,"misty":15167,"rails":15168,"sustainability":15169,"134":15170,"unused":15171,"##rating":15172,"packs":15173,"toast":15174,"unwilling":15175,"regulate":15176,"thy":15177,"suffrage":15178,"nile":15179,"awe":15180,"assam":15181,"definitions":15182,"travelers":15183,"affordable":15184,"##rb":15185,"conferred":15186,"sells":15187,"undefeated":15188,"beneficial":15189,"torso":15190,"basal":15191,"repeating":15192,"remixes":15193,"##pass":15194,"bahrain":15195,"cables":15196,"fang":15197,"##itated":15198,"excavated":15199,"numbering":15200,"statutory":15201,"##rey":15202,"deluxe":15203,"##lian":15204,"forested":15205,"ramirez":15206,"derbyshire":15207,"zeus":15208,"slamming":15209,"transfers":15210,"astronomer":15211,"banana":15212,"lottery":15213,"berg":15214,"histories":15215,"bamboo":15216,"##uchi":15217,"resurrection":15218,"posterior":15219,"bowls":15220,"vaguely":15221,"##thi":15222,"thou":15223,"preserving":15224,"tensed":15225,"offence":15226,"##inas":15227,"meyrick":15228,"callum":15229,"ridden":15230,"watt":15231,"langdon":15232,"tying":15233,"lowland":15234,"snorted":15235,"daring":15236,"truman":15237,"##hale":15238,"##girl":15239,"aura":15240,"overly":15241,"filing":15242,"weighing":15243,"goa":15244,"infections":15245,"philanthropist":15246,"saunders":15247,"eponymous":15248,"##owski":15249,"latitude":15250,"perspectives":15251,"reviewing":15252,"mets":15253,"commandant":15254,"radial":15255,"##kha":15256,"flashlight":15257,"reliability":15258,"koch":15259,"vowels":15260,"amazed":15261,"ada":15262,"elaine":15263,"supper":15264,"##rth":15265,"##encies":15266,"predator":15267,"debated":15268,"soviets":15269,"cola":15270,"##boards":15271,"##nah":15272,"compartment":15273,"crooked":15274,"arbitrary":15275,"fourteenth":15276,"##ctive":15277,"havana":15278,"majors":15279,"steelers":15280,"clips":15281,"profitable":15282,"ambush":15283,"exited":15284,"packers":15285,"##tile":15286,"nude":15287,"cracks":15288,"fungi":15289,"##е":15290,"limb":15291,"trousers":15292,"josie":15293,"shelby":15294,"tens":15295,"frederic":15296,"##ος":15297,"definite":15298,"smoothly":15299,"constellation":15300,"insult":15301,"baton":15302,"discs":15303,"lingering":15304,"##nco":15305,"conclusions":15306,"lent":15307,"staging":15308,"becker":15309,"grandpa":15310,"shaky":15311,"##tron":15312,"einstein":15313,"obstacles":15314,"sk":15315,"adverse":15316,"elle":15317,"economically":15318,"##moto":15319,"mccartney":15320,"thor":15321,"dismissal":15322,"motions":15323,"readings":15324,"nostrils":15325,"treatise":15326,"##pace":15327,"squeezing":15328,"evidently":15329,"prolonged":15330,"1783":15331,"venezuelan":15332,"je":15333,"marguerite":15334,"beirut":15335,"takeover":15336,"shareholders":15337,"##vent":15338,"denise":15339,"digit":15340,"airplay":15341,"norse":15342,"##bbling":15343,"imaginary":15344,"pills":15345,"hubert":15346,"blaze":15347,"vacated":15348,"eliminating":15349,"##ello":15350,"vine":15351,"mansfield":15352,"##tty":15353,"retrospective":15354,"barrow":15355,"borne":15356,"clutch":15357,"bail":15358,"forensic":15359,"weaving":15360,"##nett":15361,"##witz":15362,"desktop":15363,"citadel":15364,"promotions":15365,"worrying":15366,"dorset":15367,"ieee":15368,"subdivided":15369,"##iating":15370,"manned":15371,"expeditionary":15372,"pickup":15373,"synod":15374,"chuckle":15375,"185":15376,"barney":15377,"##rz":15378,"##ffin":15379,"functionality":15380,"karachi":15381,"litigation":15382,"meanings":15383,"uc":15384,"lick":15385,"turbo":15386,"anders":15387,"##ffed":15388,"execute":15389,"curl":15390,"oppose":15391,"ankles":15392,"typhoon":15393,"##د":15394,"##ache":15395,"##asia":15396,"linguistics":15397,"compassion":15398,"pressures":15399,"grazing":15400,"perfection":15401,"##iting":15402,"immunity":15403,"monopoly":15404,"muddy":15405,"backgrounds":15406,"136":15407,"namibia":15408,"francesca":15409,"monitors":15410,"attracting":15411,"stunt":15412,"tuition":15413,"##ии":15414,"vegetable":15415,"##mates":15416,"##quent":15417,"mgm":15418,"jen":15419,"complexes":15420,"forts":15421,"##ond":15422,"cellar":15423,"bites":15424,"seventeenth":15425,"royals":15426,"flemish":15427,"failures":15428,"mast":15429,"charities":15430,"##cular":15431,"peruvian":15432,"capitals":15433,"macmillan":15434,"ipswich":15435,"outward":15436,"frigate":15437,"postgraduate":15438,"folds":15439,"employing":15440,"##ouse":15441,"concurrently":15442,"fiery":15443,"##tai":15444,"contingent":15445,"nightmares":15446,"monumental":15447,"nicaragua":15448,"##kowski":15449,"lizard":15450,"mal":15451,"fielding":15452,"gig":15453,"reject":15454,"##pad":15455,"harding":15456,"##ipe":15457,"coastline":15458,"##cin":15459,"##nos":15460,"beethoven":15461,"humphrey":15462,"innovations":15463,"##tam":15464,"##nge":15465,"norris":15466,"doris":15467,"solicitor":15468,"huang":15469,"obey":15470,"141":15471,"##lc":15472,"niagara":15473,"##tton":15474,"shelves":15475,"aug":15476,"bourbon":15477,"curry":15478,"nightclub":15479,"specifications":15480,"hilton":15481,"##ndo":15482,"centennial":15483,"dispersed":15484,"worm":15485,"neglected":15486,"briggs":15487,"sm":15488,"font":15489,"kuala":15490,"uneasy":15491,"plc":15492,"##nstein":15493,"##bound":15494,"##aking":15495,"##burgh":15496,"awaiting":15497,"pronunciation":15498,"##bbed":15499,"##quest":15500,"eh":15501,"optimal":15502,"zhu":15503,"raped":15504,"greens":15505,"presided":15506,"brenda":15507,"worries":15508,"##life":15509,"venetian":15510,"marxist":15511,"turnout":15512,"##lius":15513,"refined":15514,"braced":15515,"sins":15516,"grasped":15517,"sunderland":15518,"nickel":15519,"speculated":15520,"lowell":15521,"cyrillic":15522,"communism":15523,"fundraising":15524,"resembling":15525,"colonists":15526,"mutant":15527,"freddie":15528,"usc":15529,"##mos":15530,"gratitude":15531,"##run":15532,"mural":15533,"##lous":15534,"chemist":15535,"wi":15536,"reminds":15537,"28th":15538,"steals":15539,"tess":15540,"pietro":15541,"##ingen":15542,"promoter":15543,"ri":15544,"microphone":15545,"honoured":15546,"rai":15547,"sant":15548,"##qui":15549,"feather":15550,"##nson":15551,"burlington":15552,"kurdish":15553,"terrorists":15554,"deborah":15555,"sickness":15556,"##wed":15557,"##eet":15558,"hazard":15559,"irritated":15560,"desperation":15561,"veil":15562,"clarity":15563,"##rik":15564,"jewels":15565,"xv":15566,"##gged":15567,"##ows":15568,"##cup":15569,"berkshire":15570,"unfair":15571,"mysteries":15572,"orchid":15573,"winced":15574,"exhaustion":15575,"renovations":15576,"stranded":15577,"obe":15578,"infinity":15579,"##nies":15580,"adapt":15581,"redevelopment":15582,"thanked":15583,"registry":15584,"olga":15585,"domingo":15586,"noir":15587,"tudor":15588,"ole":15589,"##atus":15590,"commenting":15591,"behaviors":15592,"##ais":15593,"crisp":15594,"pauline":15595,"probable":15596,"stirling":15597,"wigan":15598,"##bian":15599,"paralympics":15600,"panting":15601,"surpassed":15602,"##rew":15603,"luca":15604,"barred":15605,"pony":15606,"famed":15607,"##sters":15608,"cassandra":15609,"waiter":15610,"carolyn":15611,"exported":15612,"##orted":15613,"andres":15614,"destructive":15615,"deeds":15616,"jonah":15617,"castles":15618,"vacancy":15619,"suv":15620,"##glass":15621,"1788":15622,"orchard":15623,"yep":15624,"famine":15625,"belarusian":15626,"sprang":15627,"##forth":15628,"skinny":15629,"##mis":15630,"administrators":15631,"rotterdam":15632,"zambia":15633,"zhao":15634,"boiler":15635,"discoveries":15636,"##ride":15637,"##physics":15638,"lucius":15639,"disappointing":15640,"outreach":15641,"spoon":15642,"##frame":15643,"qualifications":15644,"unanimously":15645,"enjoys":15646,"regency":15647,"##iidae":15648,"stade":15649,"realism":15650,"veterinary":15651,"rodgers":15652,"dump":15653,"alain":15654,"chestnut":15655,"castile":15656,"censorship":15657,"rumble":15658,"gibbs":15659,"##itor":15660,"communion":15661,"reggae":15662,"inactivated":15663,"logs":15664,"loads":15665,"##houses":15666,"homosexual":15667,"##iano":15668,"ale":15669,"informs":15670,"##cas":15671,"phrases":15672,"plaster":15673,"linebacker":15674,"ambrose":15675,"kaiser":15676,"fascinated":15677,"850":15678,"limerick":15679,"recruitment":15680,"forge":15681,"mastered":15682,"##nding":15683,"leinster":15684,"rooted":15685,"threaten":15686,"##strom":15687,"borneo":15688,"##hes":15689,"suggestions":15690,"scholarships":15691,"propeller":15692,"documentaries":15693,"patronage":15694,"coats":15695,"constructing":15696,"invest":15697,"neurons":15698,"comet":15699,"entirety":15700,"shouts":15701,"identities":15702,"annoying":15703,"unchanged":15704,"wary":15705,"##antly":15706,"##ogy":15707,"neat":15708,"oversight":15709,"##kos":15710,"phillies":15711,"replay":15712,"constance":15713,"##kka":15714,"incarnation":15715,"humble":15716,"skies":15717,"minus":15718,"##acy":15719,"smithsonian":15720,"##chel":15721,"guerrilla":15722,"jar":15723,"cadets":15724,"##plate":15725,"surplus":15726,"audit":15727,"##aru":15728,"cracking":15729,"joanna":15730,"louisa":15731,"pacing":15732,"##lights":15733,"intentionally":15734,"##iri":15735,"diner":15736,"nwa":15737,"imprint":15738,"australians":15739,"tong":15740,"unprecedented":15741,"bunker":15742,"naive":15743,"specialists":15744,"ark":15745,"nichols":15746,"railing":15747,"leaked":15748,"pedal":15749,"##uka":15750,"shrub":15751,"longing":15752,"roofs":15753,"v8":15754,"captains":15755,"neural":15756,"tuned":15757,"##ntal":15758,"##jet":15759,"emission":15760,"medina":15761,"frantic":15762,"codex":15763,"definitive":15764,"sid":15765,"abolition":15766,"intensified":15767,"stocks":15768,"enrique":15769,"sustain":15770,"genoa":15771,"oxide":15772,"##written":15773,"clues":15774,"cha":15775,"##gers":15776,"tributaries":15777,"fragment":15778,"venom":15779,"##rity":15780,"##ente":15781,"##sca":15782,"muffled":15783,"vain":15784,"sire":15785,"laos":15786,"##ingly":15787,"##hana":15788,"hastily":15789,"snapping":15790,"surfaced":15791,"sentiment":15792,"motive":15793,"##oft":15794,"contests":15795,"approximate":15796,"mesa":15797,"luckily":15798,"dinosaur":15799,"exchanges":15800,"propelled":15801,"accord":15802,"bourne":15803,"relieve":15804,"tow":15805,"masks":15806,"offended":15807,"##ues":15808,"cynthia":15809,"##mmer":15810,"rains":15811,"bartender":15812,"zinc":15813,"reviewers":15814,"lois":15815,"##sai":15816,"legged":15817,"arrogant":15818,"rafe":15819,"rosie":15820,"comprise":15821,"handicap":15822,"blockade":15823,"inlet":15824,"lagoon":15825,"copied":15826,"drilling":15827,"shelley":15828,"petals":15829,"##inian":15830,"mandarin":15831,"obsolete":15832,"##inated":15833,"onward":15834,"arguably":15835,"productivity":15836,"cindy":15837,"praising":15838,"seldom":15839,"busch":15840,"discusses":15841,"raleigh":15842,"shortage":15843,"ranged":15844,"stanton":15845,"encouragement":15846,"firstly":15847,"conceded":15848,"overs":15849,"temporal":15850,"##uke":15851,"cbe":15852,"##bos":15853,"woo":15854,"certainty":15855,"pumps":15856,"##pton":15857,"stalked":15858,"##uli":15859,"lizzie":15860,"periodic":15861,"thieves":15862,"weaker":15863,"##night":15864,"gases":15865,"shoving":15866,"chooses":15867,"wc":15868,"##chemical":15869,"prompting":15870,"weights":15871,"##kill":15872,"robust":15873,"flanked":15874,"sticky":15875,"hu":15876,"tuberculosis":15877,"##eb":15878,"##eal":15879,"christchurch":15880,"resembled":15881,"wallet":15882,"reese":15883,"inappropriate":15884,"pictured":15885,"distract":15886,"fixing":15887,"fiddle":15888,"giggled":15889,"burger":15890,"heirs":15891,"hairy":15892,"mechanic":15893,"torque":15894,"apache":15895,"obsessed":15896,"chiefly":15897,"cheng":15898,"logging":15899,"##tag":15900,"extracted":15901,"meaningful":15902,"numb":15903,"##vsky":15904,"gloucestershire":15905,"reminding":15906,"##bay":15907,"unite":15908,"##lit":15909,"breeds":15910,"diminished":15911,"clown":15912,"glove":15913,"1860s":15914,"##ن":15915,"##ug":15916,"archibald":15917,"focal":15918,"freelance":15919,"sliced":15920,"depiction":15921,"##yk":15922,"organism":15923,"switches":15924,"sights":15925,"stray":15926,"crawling":15927,"##ril":15928,"lever":15929,"leningrad":15930,"interpretations":15931,"loops":15932,"anytime":15933,"reel":15934,"alicia":15935,"delighted":15936,"##ech":15937,"inhaled":15938,"xiv":15939,"suitcase":15940,"bernie":15941,"vega":15942,"licenses":15943,"northampton":15944,"exclusion":15945,"induction":15946,"monasteries":15947,"racecourse":15948,"homosexuality":15949,"##right":15950,"##sfield":15951,"##rky":15952,"dimitri":15953,"michele":15954,"alternatives":15955,"ions":15956,"commentators":15957,"genuinely":15958,"objected":15959,"pork":15960,"hospitality":15961,"fencing":15962,"stephan":15963,"warships":15964,"peripheral":15965,"wit":15966,"drunken":15967,"wrinkled":15968,"quentin":15969,"spends":15970,"departing":15971,"chung":15972,"numerical":15973,"spokesperson":15974,"##zone":15975,"johannesburg":15976,"caliber":15977,"killers":15978,"##udge":15979,"assumes":15980,"neatly":15981,"demographic":15982,"abigail":15983,"bloc":15984,"##vel":15985,"mounting":15986,"##lain":15987,"bentley":15988,"slightest":15989,"xu":15990,"recipients":15991,"##jk":15992,"merlin":15993,"##writer":15994,"seniors":15995,"prisons":15996,"blinking":15997,"hindwings":15998,"flickered":15999,"kappa":16000,"##hel":16001,"80s":16002,"strengthening":16003,"appealing":16004,"brewing":16005,"gypsy":16006,"mali":16007,"lashes":16008,"hulk":16009,"unpleasant":16010,"harassment":16011,"bio":16012,"treaties":16013,"predict":16014,"instrumentation":16015,"pulp":16016,"troupe":16017,"boiling":16018,"mantle":16019,"##ffe":16020,"ins":16021,"##vn":16022,"dividing":16023,"handles":16024,"verbs":16025,"##onal":16026,"coconut":16027,"senegal":16028,"340":16029,"thorough":16030,"gum":16031,"momentarily":16032,"##sto":16033,"cocaine":16034,"panicked":16035,"destined":16036,"##turing":16037,"teatro":16038,"denying":16039,"weary":16040,"captained":16041,"mans":16042,"##hawks":16043,"##code":16044,"wakefield":16045,"bollywood":16046,"thankfully":16047,"##16":16048,"cyril":16049,"##wu":16050,"amendments":16051,"##bahn":16052,"consultation":16053,"stud":16054,"reflections":16055,"kindness":16056,"1787":16057,"internally":16058,"##ovo":16059,"tex":16060,"mosaic":16061,"distribute":16062,"paddy":16063,"seeming":16064,"143":16065,"##hic":16066,"piers":16067,"##15":16068,"##mura":16069,"##verse":16070,"popularly":16071,"winger":16072,"kang":16073,"sentinel":16074,"mccoy":16075,"##anza":16076,"covenant":16077,"##bag":16078,"verge":16079,"fireworks":16080,"suppress":16081,"thrilled":16082,"dominate":16083,"##jar":16084,"swansea":16085,"##60":16086,"142":16087,"reconciliation":16088,"##ndi":16089,"stiffened":16090,"cue":16091,"dorian":16092,"##uf":16093,"damascus":16094,"amor":16095,"ida":16096,"foremost":16097,"##aga":16098,"porsche":16099,"unseen":16100,"dir":16101,"##had":16102,"##azi":16103,"stony":16104,"lexi":16105,"melodies":16106,"##nko":16107,"angular":16108,"integer":16109,"podcast":16110,"ants":16111,"inherent":16112,"jaws":16113,"justify":16114,"persona":16115,"##olved":16116,"josephine":16117,"##nr":16118,"##ressed":16119,"customary":16120,"flashes":16121,"gala":16122,"cyrus":16123,"glaring":16124,"backyard":16125,"ariel":16126,"physiology":16127,"greenland":16128,"html":16129,"stir":16130,"avon":16131,"atletico":16132,"finch":16133,"methodology":16134,"ked":16135,"##lent":16136,"mas":16137,"catholicism":16138,"townsend":16139,"branding":16140,"quincy":16141,"fits":16142,"containers":16143,"1777":16144,"ashore":16145,"aragon":16146,"##19":16147,"forearm":16148,"poisoning":16149,"##sd":16150,"adopting":16151,"conquer":16152,"grinding":16153,"amnesty":16154,"keller":16155,"finances":16156,"evaluate":16157,"forged":16158,"lankan":16159,"instincts":16160,"##uto":16161,"guam":16162,"bosnian":16163,"photographed":16164,"workplace":16165,"desirable":16166,"protector":16167,"##dog":16168,"allocation":16169,"intently":16170,"encourages":16171,"willy":16172,"##sten":16173,"bodyguard":16174,"electro":16175,"brighter":16176,"##ν":16177,"bihar":16178,"##chev":16179,"lasts":16180,"opener":16181,"amphibious":16182,"sal":16183,"verde":16184,"arte":16185,"##cope":16186,"captivity":16187,"vocabulary":16188,"yields":16189,"##tted":16190,"agreeing":16191,"desmond":16192,"pioneered":16193,"##chus":16194,"strap":16195,"campaigned":16196,"railroads":16197,"##ович":16198,"emblem":16199,"##dre":16200,"stormed":16201,"501":16202,"##ulous":16203,"marijuana":16204,"northumberland":16205,"##gn":16206,"##nath":16207,"bowen":16208,"landmarks":16209,"beaumont":16210,"##qua":16211,"danube":16212,"##bler":16213,"attorneys":16214,"th":16215,"ge":16216,"flyers":16217,"critique":16218,"villains":16219,"cass":16220,"mutation":16221,"acc":16222,"##0s":16223,"colombo":16224,"mckay":16225,"motif":16226,"sampling":16227,"concluding":16228,"syndicate":16229,"##rell":16230,"neon":16231,"stables":16232,"ds":16233,"warnings":16234,"clint":16235,"mourning":16236,"wilkinson":16237,"##tated":16238,"merrill":16239,"leopard":16240,"evenings":16241,"exhaled":16242,"emil":16243,"sonia":16244,"ezra":16245,"discrete":16246,"stove":16247,"farrell":16248,"fifteenth":16249,"prescribed":16250,"superhero":16251,"##rier":16252,"worms":16253,"helm":16254,"wren":16255,"##duction":16256,"##hc":16257,"expo":16258,"##rator":16259,"hq":16260,"unfamiliar":16261,"antony":16262,"prevents":16263,"acceleration":16264,"fiercely":16265,"mari":16266,"painfully":16267,"calculations":16268,"cheaper":16269,"ign":16270,"clifton":16271,"irvine":16272,"davenport":16273,"mozambique":16274,"##np":16275,"pierced":16276,"##evich":16277,"wonders":16278,"##wig":16279,"##cate":16280,"##iling":16281,"crusade":16282,"ware":16283,"##uel":16284,"enzymes":16285,"reasonably":16286,"mls":16287,"##coe":16288,"mater":16289,"ambition":16290,"bunny":16291,"eliot":16292,"kernel":16293,"##fin":16294,"asphalt":16295,"headmaster":16296,"torah":16297,"aden":16298,"lush":16299,"pins":16300,"waived":16301,"##care":16302,"##yas":16303,"joao":16304,"substrate":16305,"enforce":16306,"##grad":16307,"##ules":16308,"alvarez":16309,"selections":16310,"epidemic":16311,"tempted":16312,"##bit":16313,"bremen":16314,"translates":16315,"ensured":16316,"waterfront":16317,"29th":16318,"forrest":16319,"manny":16320,"malone":16321,"kramer":16322,"reigning":16323,"cookies":16324,"simpler":16325,"absorption":16326,"205":16327,"engraved":16328,"##ffy":16329,"evaluated":16330,"1778":16331,"haze":16332,"146":16333,"comforting":16334,"crossover":16335,"##abe":16336,"thorn":16337,"##rift":16338,"##imo":16339,"##pop":16340,"suppression":16341,"fatigue":16342,"cutter":16343,"##tr":16344,"201":16345,"wurttemberg":16346,"##orf":16347,"enforced":16348,"hovering":16349,"proprietary":16350,"gb":16351,"samurai":16352,"syllable":16353,"ascent":16354,"lacey":16355,"tick":16356,"lars":16357,"tractor":16358,"merchandise":16359,"rep":16360,"bouncing":16361,"defendants":16362,"##yre":16363,"huntington":16364,"##ground":16365,"##oko":16366,"standardized":16367,"##hor":16368,"##hima":16369,"assassinated":16370,"nu":16371,"predecessors":16372,"rainy":16373,"liar":16374,"assurance":16375,"lyrical":16376,"##uga":16377,"secondly":16378,"flattened":16379,"ios":16380,"parameter":16381,"undercover":16382,"##mity":16383,"bordeaux":16384,"punish":16385,"ridges":16386,"markers":16387,"exodus":16388,"inactive":16389,"hesitate":16390,"debbie":16391,"nyc":16392,"pledge":16393,"savoy":16394,"nagar":16395,"offset":16396,"organist":16397,"##tium":16398,"hesse":16399,"marin":16400,"converting":16401,"##iver":16402,"diagram":16403,"propulsion":16404,"pu":16405,"validity":16406,"reverted":16407,"supportive":16408,"##dc":16409,"ministries":16410,"clans":16411,"responds":16412,"proclamation":16413,"##inae":16414,"##ø":16415,"##rea":16416,"ein":16417,"pleading":16418,"patriot":16419,"sf":16420,"birch":16421,"islanders":16422,"strauss":16423,"hates":16424,"##dh":16425,"brandenburg":16426,"concession":16427,"rd":16428,"##ob":16429,"1900s":16430,"killings":16431,"textbook":16432,"antiquity":16433,"cinematography":16434,"wharf":16435,"embarrassing":16436,"setup":16437,"creed":16438,"farmland":16439,"inequality":16440,"centred":16441,"signatures":16442,"fallon":16443,"370":16444,"##ingham":16445,"##uts":16446,"ceylon":16447,"gazing":16448,"directive":16449,"laurie":16450,"##tern":16451,"globally":16452,"##uated":16453,"##dent":16454,"allah":16455,"excavation":16456,"threads":16457,"##cross":16458,"148":16459,"frantically":16460,"icc":16461,"utilize":16462,"determines":16463,"respiratory":16464,"thoughtful":16465,"receptions":16466,"##dicate":16467,"merging":16468,"chandra":16469,"seine":16470,"147":16471,"builders":16472,"builds":16473,"diagnostic":16474,"dev":16475,"visibility":16476,"goddamn":16477,"analyses":16478,"dhaka":16479,"cho":16480,"proves":16481,"chancel":16482,"concurrent":16483,"curiously":16484,"canadians":16485,"pumped":16486,"restoring":16487,"1850s":16488,"turtles":16489,"jaguar":16490,"sinister":16491,"spinal":16492,"traction":16493,"declan":16494,"vows":16495,"1784":16496,"glowed":16497,"capitalism":16498,"swirling":16499,"install":16500,"universidad":16501,"##lder":16502,"##oat":16503,"soloist":16504,"##genic":16505,"##oor":16506,"coincidence":16507,"beginnings":16508,"nissan":16509,"dip":16510,"resorts":16511,"caucasus":16512,"combustion":16513,"infectious":16514,"##eno":16515,"pigeon":16516,"serpent":16517,"##itating":16518,"conclude":16519,"masked":16520,"salad":16521,"jew":16522,"##gr":16523,"surreal":16524,"toni":16525,"##wc":16526,"harmonica":16527,"151":16528,"##gins":16529,"##etic":16530,"##coat":16531,"fishermen":16532,"intending":16533,"bravery":16534,"##wave":16535,"klaus":16536,"titan":16537,"wembley":16538,"taiwanese":16539,"ransom":16540,"40th":16541,"incorrect":16542,"hussein":16543,"eyelids":16544,"jp":16545,"cooke":16546,"dramas":16547,"utilities":16548,"##etta":16549,"##print":16550,"eisenhower":16551,"principally":16552,"granada":16553,"lana":16554,"##rak":16555,"openings":16556,"concord":16557,"##bl":16558,"bethany":16559,"connie":16560,"morality":16561,"sega":16562,"##mons":16563,"##nard":16564,"earnings":16565,"##kara":16566,"##cine":16567,"wii":16568,"communes":16569,"##rel":16570,"coma":16571,"composing":16572,"softened":16573,"severed":16574,"grapes":16575,"##17":16576,"nguyen":16577,"analyzed":16578,"warlord":16579,"hubbard":16580,"heavenly":16581,"behave":16582,"slovenian":16583,"##hit":16584,"##ony":16585,"hailed":16586,"filmmakers":16587,"trance":16588,"caldwell":16589,"skye":16590,"unrest":16591,"coward":16592,"likelihood":16593,"##aging":16594,"bern":16595,"sci":16596,"taliban":16597,"honolulu":16598,"propose":16599,"##wang":16600,"1700":16601,"browser":16602,"imagining":16603,"cobra":16604,"contributes":16605,"dukes":16606,"instinctively":16607,"conan":16608,"violinist":16609,"##ores":16610,"accessories":16611,"gradual":16612,"##amp":16613,"quotes":16614,"sioux":16615,"##dating":16616,"undertake":16617,"intercepted":16618,"sparkling":16619,"compressed":16620,"139":16621,"fungus":16622,"tombs":16623,"haley":16624,"imposing":16625,"rests":16626,"degradation":16627,"lincolnshire":16628,"retailers":16629,"wetlands":16630,"tulsa":16631,"distributor":16632,"dungeon":16633,"nun":16634,"greenhouse":16635,"convey":16636,"atlantis":16637,"aft":16638,"exits":16639,"oman":16640,"dresser":16641,"lyons":16642,"##sti":16643,"joking":16644,"eddy":16645,"judgement":16646,"omitted":16647,"digits":16648,"##cts":16649,"##game":16650,"juniors":16651,"##rae":16652,"cents":16653,"stricken":16654,"une":16655,"##ngo":16656,"wizards":16657,"weir":16658,"breton":16659,"nan":16660,"technician":16661,"fibers":16662,"liking":16663,"royalty":16664,"##cca":16665,"154":16666,"persia":16667,"terribly":16668,"magician":16669,"##rable":16670,"##unt":16671,"vance":16672,"cafeteria":16673,"booker":16674,"camille":16675,"warmer":16676,"##static":16677,"consume":16678,"cavern":16679,"gaps":16680,"compass":16681,"contemporaries":16682,"foyer":16683,"soothing":16684,"graveyard":16685,"maj":16686,"plunged":16687,"blush":16688,"##wear":16689,"cascade":16690,"demonstrates":16691,"ordinance":16692,"##nov":16693,"boyle":16694,"##lana":16695,"rockefeller":16696,"shaken":16697,"banjo":16698,"izzy":16699,"##ense":16700,"breathless":16701,"vines":16702,"##32":16703,"##eman":16704,"alterations":16705,"chromosome":16706,"dwellings":16707,"feudal":16708,"mole":16709,"153":16710,"catalonia":16711,"relics":16712,"tenant":16713,"mandated":16714,"##fm":16715,"fridge":16716,"hats":16717,"honesty":16718,"patented":16719,"raul":16720,"heap":16721,"cruisers":16722,"accusing":16723,"enlightenment":16724,"infants":16725,"wherein":16726,"chatham":16727,"contractors":16728,"zen":16729,"affinity":16730,"hc":16731,"osborne":16732,"piston":16733,"156":16734,"traps":16735,"maturity":16736,"##rana":16737,"lagos":16738,"##zal":16739,"peering":16740,"##nay":16741,"attendant":16742,"dealers":16743,"protocols":16744,"subset":16745,"prospects":16746,"biographical":16747,"##cre":16748,"artery":16749,"##zers":16750,"insignia":16751,"nuns":16752,"endured":16753,"##eration":16754,"recommend":16755,"schwartz":16756,"serbs":16757,"berger":16758,"cromwell":16759,"crossroads":16760,"##ctor":16761,"enduring":16762,"clasped":16763,"grounded":16764,"##bine":16765,"marseille":16766,"twitched":16767,"abel":16768,"choke":16769,"https":16770,"catalyst":16771,"moldova":16772,"italians":16773,"##tist":16774,"disastrous":16775,"wee":16776,"##oured":16777,"##nti":16778,"wwf":16779,"nope":16780,"##piration":16781,"##asa":16782,"expresses":16783,"thumbs":16784,"167":16785,"##nza":16786,"coca":16787,"1781":16788,"cheating":16789,"##ption":16790,"skipped":16791,"sensory":16792,"heidelberg":16793,"spies":16794,"satan":16795,"dangers":16796,"semifinal":16797,"202":16798,"bohemia":16799,"whitish":16800,"confusing":16801,"shipbuilding":16802,"relies":16803,"surgeons":16804,"landings":16805,"ravi":16806,"baku":16807,"moor":16808,"suffix":16809,"alejandro":16810,"##yana":16811,"litre":16812,"upheld":16813,"##unk":16814,"rajasthan":16815,"##rek":16816,"coaster":16817,"insists":16818,"posture":16819,"scenarios":16820,"etienne":16821,"favoured":16822,"appoint":16823,"transgender":16824,"elephants":16825,"poked":16826,"greenwood":16827,"defences":16828,"fulfilled":16829,"militant":16830,"somali":16831,"1758":16832,"chalk":16833,"potent":16834,"##ucci":16835,"migrants":16836,"wink":16837,"assistants":16838,"nos":16839,"restriction":16840,"activism":16841,"niger":16842,"##ario":16843,"colon":16844,"shaun":16845,"##sat":16846,"daphne":16847,"##erated":16848,"swam":16849,"congregations":16850,"reprise":16851,"considerations":16852,"magnet":16853,"playable":16854,"xvi":16855,"##р":16856,"overthrow":16857,"tobias":16858,"knob":16859,"chavez":16860,"coding":16861,"##mers":16862,"propped":16863,"katrina":16864,"orient":16865,"newcomer":16866,"##suke":16867,"temperate":16868,"##pool":16869,"farmhouse":16870,"interrogation":16871,"##vd":16872,"committing":16873,"##vert":16874,"forthcoming":16875,"strawberry":16876,"joaquin":16877,"macau":16878,"ponds":16879,"shocking":16880,"siberia":16881,"##cellular":16882,"chant":16883,"contributors":16884,"##nant":16885,"##ologists":16886,"sped":16887,"absorb":16888,"hail":16889,"1782":16890,"spared":16891,"##hore":16892,"barbados":16893,"karate":16894,"opus":16895,"originates":16896,"saul":16897,"##xie":16898,"evergreen":16899,"leaped":16900,"##rock":16901,"correlation":16902,"exaggerated":16903,"weekday":16904,"unification":16905,"bump":16906,"tracing":16907,"brig":16908,"afb":16909,"pathways":16910,"utilizing":16911,"##ners":16912,"mod":16913,"mb":16914,"disturbance":16915,"kneeling":16916,"##stad":16917,"##guchi":16918,"100th":16919,"pune":16920,"##thy":16921,"decreasing":16922,"168":16923,"manipulation":16924,"miriam":16925,"academia":16926,"ecosystem":16927,"occupational":16928,"rbi":16929,"##lem":16930,"rift":16931,"##14":16932,"rotary":16933,"stacked":16934,"incorporation":16935,"awakening":16936,"generators":16937,"guerrero":16938,"racist":16939,"##omy":16940,"cyber":16941,"derivatives":16942,"culminated":16943,"allie":16944,"annals":16945,"panzer":16946,"sainte":16947,"wikipedia":16948,"pops":16949,"zu":16950,"austro":16951,"##vate":16952,"algerian":16953,"politely":16954,"nicholson":16955,"mornings":16956,"educate":16957,"tastes":16958,"thrill":16959,"dartmouth":16960,"##gating":16961,"db":16962,"##jee":16963,"regan":16964,"differing":16965,"concentrating":16966,"choreography":16967,"divinity":16968,"##media":16969,"pledged":16970,"alexandre":16971,"routing":16972,"gregor":16973,"madeline":16974,"##idal":16975,"apocalypse":16976,"##hora":16977,"gunfire":16978,"culminating":16979,"elves":16980,"fined":16981,"liang":16982,"lam":16983,"programmed":16984,"tar":16985,"guessing":16986,"transparency":16987,"gabrielle":16988,"##gna":16989,"cancellation":16990,"flexibility":16991,"##lining":16992,"accession":16993,"shea":16994,"stronghold":16995,"nets":16996,"specializes":16997,"##rgan":16998,"abused":16999,"hasan":17000,"sgt":17001,"ling":17002,"exceeding":17003,"##₄":17004,"admiration":17005,"supermarket":17006,"##ark":17007,"photographers":17008,"specialised":17009,"tilt":17010,"resonance":17011,"hmm":17012,"perfume":17013,"380":17014,"sami":17015,"threatens":17016,"garland":17017,"botany":17018,"guarding":17019,"boiled":17020,"greet":17021,"puppy":17022,"russo":17023,"supplier":17024,"wilmington":17025,"vibrant":17026,"vijay":17027,"##bius":17028,"paralympic":17029,"grumbled":17030,"paige":17031,"faa":17032,"licking":17033,"margins":17034,"hurricanes":17035,"##gong":17036,"fest":17037,"grenade":17038,"ripping":17039,"##uz":17040,"counseling":17041,"weigh":17042,"##sian":17043,"needles":17044,"wiltshire":17045,"edison":17046,"costly":17047,"##not":17048,"fulton":17049,"tramway":17050,"redesigned":17051,"staffordshire":17052,"cache":17053,"gasping":17054,"watkins":17055,"sleepy":17056,"candidacy":17057,"##group":17058,"monkeys":17059,"timeline":17060,"throbbing":17061,"##bid":17062,"##sos":17063,"berth":17064,"uzbekistan":17065,"vanderbilt":17066,"bothering":17067,"overturned":17068,"ballots":17069,"gem":17070,"##iger":17071,"sunglasses":17072,"subscribers":17073,"hooker":17074,"compelling":17075,"ang":17076,"exceptionally":17077,"saloon":17078,"stab":17079,"##rdi":17080,"carla":17081,"terrifying":17082,"rom":17083,"##vision":17084,"coil":17085,"##oids":17086,"satisfying":17087,"vendors":17088,"31st":17089,"mackay":17090,"deities":17091,"overlooked":17092,"ambient":17093,"bahamas":17094,"felipe":17095,"olympia":17096,"whirled":17097,"botanist":17098,"advertised":17099,"tugging":17100,"##dden":17101,"disciples":17102,"morales":17103,"unionist":17104,"rites":17105,"foley":17106,"morse":17107,"motives":17108,"creepy":17109,"##₀":17110,"soo":17111,"##sz":17112,"bargain":17113,"highness":17114,"frightening":17115,"turnpike":17116,"tory":17117,"reorganization":17118,"##cer":17119,"depict":17120,"biographer":17121,"##walk":17122,"unopposed":17123,"manifesto":17124,"##gles":17125,"institut":17126,"emile":17127,"accidental":17128,"kapoor":17129,"##dam":17130,"kilkenny":17131,"cortex":17132,"lively":17133,"##13":17134,"romanesque":17135,"jain":17136,"shan":17137,"cannons":17138,"##ood":17139,"##ske":17140,"petrol":17141,"echoing":17142,"amalgamated":17143,"disappears":17144,"cautious":17145,"proposes":17146,"sanctions":17147,"trenton":17148,"##ر":17149,"flotilla":17150,"aus":17151,"contempt":17152,"tor":17153,"canary":17154,"cote":17155,"theirs":17156,"##hun":17157,"conceptual":17158,"deleted":17159,"fascinating":17160,"paso":17161,"blazing":17162,"elf":17163,"honourable":17164,"hutchinson":17165,"##eiro":17166,"##outh":17167,"##zin":17168,"surveyor":17169,"tee":17170,"amidst":17171,"wooded":17172,"reissue":17173,"intro":17174,"##ono":17175,"cobb":17176,"shelters":17177,"newsletter":17178,"hanson":17179,"brace":17180,"encoding":17181,"confiscated":17182,"dem":17183,"caravan":17184,"marino":17185,"scroll":17186,"melodic":17187,"cows":17188,"imam":17189,"##adi":17190,"##aneous":17191,"northward":17192,"searches":17193,"biodiversity":17194,"cora":17195,"310":17196,"roaring":17197,"##bers":17198,"connell":17199,"theologian":17200,"halo":17201,"compose":17202,"pathetic":17203,"unmarried":17204,"dynamo":17205,"##oot":17206,"az":17207,"calculation":17208,"toulouse":17209,"deserves":17210,"humour":17211,"nr":17212,"forgiveness":17213,"tam":17214,"undergone":17215,"martyr":17216,"pamela":17217,"myths":17218,"whore":17219,"counselor":17220,"hicks":17221,"290":17222,"heavens":17223,"battleship":17224,"electromagnetic":17225,"##bbs":17226,"stellar":17227,"establishments":17228,"presley":17229,"hopped":17230,"##chin":17231,"temptation":17232,"90s":17233,"wills":17234,"nas":17235,"##yuan":17236,"nhs":17237,"##nya":17238,"seminars":17239,"##yev":17240,"adaptations":17241,"gong":17242,"asher":17243,"lex":17244,"indicator":17245,"sikh":17246,"tobago":17247,"cites":17248,"goin":17249,"##yte":17250,"satirical":17251,"##gies":17252,"characterised":17253,"correspond":17254,"bubbles":17255,"lure":17256,"participates":17257,"##vid":17258,"eruption":17259,"skate":17260,"therapeutic":17261,"1785":17262,"canals":17263,"wholesale":17264,"defaulted":17265,"sac":17266,"460":17267,"petit":17268,"##zzled":17269,"virgil":17270,"leak":17271,"ravens":17272,"256":17273,"portraying":17274,"##yx":17275,"ghetto":17276,"creators":17277,"dams":17278,"portray":17279,"vicente":17280,"##rington":17281,"fae":17282,"namesake":17283,"bounty":17284,"##arium":17285,"joachim":17286,"##ota":17287,"##iser":17288,"aforementioned":17289,"axle":17290,"snout":17291,"depended":17292,"dismantled":17293,"reuben":17294,"480":17295,"##ibly":17296,"gallagher":17297,"##lau":17298,"##pd":17299,"earnest":17300,"##ieu":17301,"##iary":17302,"inflicted":17303,"objections":17304,"##llar":17305,"asa":17306,"gritted":17307,"##athy":17308,"jericho":17309,"##sea":17310,"##was":17311,"flick":17312,"underside":17313,"ceramics":17314,"undead":17315,"substituted":17316,"195":17317,"eastward":17318,"undoubtedly":17319,"wheeled":17320,"chimney":17321,"##iche":17322,"guinness":17323,"cb":17324,"##ager":17325,"siding":17326,"##bell":17327,"traitor":17328,"baptiste":17329,"disguised":17330,"inauguration":17331,"149":17332,"tipperary":17333,"choreographer":17334,"perched":17335,"warmed":17336,"stationary":17337,"eco":17338,"##ike":17339,"##ntes":17340,"bacterial":17341,"##aurus":17342,"flores":17343,"phosphate":17344,"##core":17345,"attacker":17346,"invaders":17347,"alvin":17348,"intersects":17349,"a1":17350,"indirectly":17351,"immigrated":17352,"businessmen":17353,"cornelius":17354,"valves":17355,"narrated":17356,"pill":17357,"sober":17358,"ul":17359,"nationale":17360,"monastic":17361,"applicants":17362,"scenery":17363,"##jack":17364,"161":17365,"motifs":17366,"constitutes":17367,"cpu":17368,"##osh":17369,"jurisdictions":17370,"sd":17371,"tuning":17372,"irritation":17373,"woven":17374,"##uddin":17375,"fertility":17376,"gao":17377,"##erie":17378,"antagonist":17379,"impatient":17380,"glacial":17381,"hides":17382,"boarded":17383,"denominations":17384,"interception":17385,"##jas":17386,"cookie":17387,"nicola":17388,"##tee":17389,"algebraic":17390,"marquess":17391,"bahn":17392,"parole":17393,"buyers":17394,"bait":17395,"turbines":17396,"paperwork":17397,"bestowed":17398,"natasha":17399,"renee":17400,"oceans":17401,"purchases":17402,"157":17403,"vaccine":17404,"215":17405,"##tock":17406,"fixtures":17407,"playhouse":17408,"integrate":17409,"jai":17410,"oswald":17411,"intellectuals":17412,"##cky":17413,"booked":17414,"nests":17415,"mortimer":17416,"##isi":17417,"obsession":17418,"sept":17419,"##gler":17420,"##sum":17421,"440":17422,"scrutiny":17423,"simultaneous":17424,"squinted":17425,"##shin":17426,"collects":17427,"oven":17428,"shankar":17429,"penned":17430,"remarkably":17431,"##я":17432,"slips":17433,"luggage":17434,"spectral":17435,"1786":17436,"collaborations":17437,"louie":17438,"consolidation":17439,"##ailed":17440,"##ivating":17441,"420":17442,"hoover":17443,"blackpool":17444,"harness":17445,"ignition":17446,"vest":17447,"tails":17448,"belmont":17449,"mongol":17450,"skinner":17451,"##nae":17452,"visually":17453,"mage":17454,"derry":17455,"##tism":17456,"##unce":17457,"stevie":17458,"transitional":17459,"##rdy":17460,"redskins":17461,"drying":17462,"prep":17463,"prospective":17464,"##21":17465,"annoyance":17466,"oversee":17467,"##loaded":17468,"fills":17469,"##books":17470,"##iki":17471,"announces":17472,"fda":17473,"scowled":17474,"respects":17475,"prasad":17476,"mystic":17477,"tucson":17478,"##vale":17479,"revue":17480,"springer":17481,"bankrupt":17482,"1772":17483,"aristotle":17484,"salvatore":17485,"habsburg":17486,"##geny":17487,"dal":17488,"natal":17489,"nut":17490,"pod":17491,"chewing":17492,"darts":17493,"moroccan":17494,"walkover":17495,"rosario":17496,"lenin":17497,"punjabi":17498,"##ße":17499,"grossed":17500,"scattering":17501,"wired":17502,"invasive":17503,"hui":17504,"polynomial":17505,"corridors":17506,"wakes":17507,"gina":17508,"portrays":17509,"##cratic":17510,"arid":17511,"retreating":17512,"erich":17513,"irwin":17514,"sniper":17515,"##dha":17516,"linen":17517,"lindsey":17518,"maneuver":17519,"butch":17520,"shutting":17521,"socio":17522,"bounce":17523,"commemorative":17524,"postseason":17525,"jeremiah":17526,"pines":17527,"275":17528,"mystical":17529,"beads":17530,"bp":17531,"abbas":17532,"furnace":17533,"bidding":17534,"consulted":17535,"assaulted":17536,"empirical":17537,"rubble":17538,"enclosure":17539,"sob":17540,"weakly":17541,"cancel":17542,"polly":17543,"yielded":17544,"##emann":17545,"curly":17546,"prediction":17547,"battered":17548,"70s":17549,"vhs":17550,"jacqueline":17551,"render":17552,"sails":17553,"barked":17554,"detailing":17555,"grayson":17556,"riga":17557,"sloane":17558,"raging":17559,"##yah":17560,"herbs":17561,"bravo":17562,"##athlon":17563,"alloy":17564,"giggle":17565,"imminent":17566,"suffers":17567,"assumptions":17568,"waltz":17569,"##itate":17570,"accomplishments":17571,"##ited":17572,"bathing":17573,"remixed":17574,"deception":17575,"prefix":17576,"##emia":17577,"deepest":17578,"##tier":17579,"##eis":17580,"balkan":17581,"frogs":17582,"##rong":17583,"slab":17584,"##pate":17585,"philosophers":17586,"peterborough":17587,"grains":17588,"imports":17589,"dickinson":17590,"rwanda":17591,"##atics":17592,"1774":17593,"dirk":17594,"lan":17595,"tablets":17596,"##rove":17597,"clone":17598,"##rice":17599,"caretaker":17600,"hostilities":17601,"mclean":17602,"##gre":17603,"regimental":17604,"treasures":17605,"norms":17606,"impose":17607,"tsar":17608,"tango":17609,"diplomacy":17610,"variously":17611,"complain":17612,"192":17613,"recognise":17614,"arrests":17615,"1779":17616,"celestial":17617,"pulitzer":17618,"##dus":17619,"bing":17620,"libretto":17621,"##moor":17622,"adele":17623,"splash":17624,"##rite":17625,"expectation":17626,"lds":17627,"confronts":17628,"##izer":17629,"spontaneous":17630,"harmful":17631,"wedge":17632,"entrepreneurs":17633,"buyer":17634,"##ope":17635,"bilingual":17636,"translate":17637,"rugged":17638,"conner":17639,"circulated":17640,"uae":17641,"eaton":17642,"##gra":17643,"##zzle":17644,"lingered":17645,"lockheed":17646,"vishnu":17647,"reelection":17648,"alonso":17649,"##oom":17650,"joints":17651,"yankee":17652,"headline":17653,"cooperate":17654,"heinz":17655,"laureate":17656,"invading":17657,"##sford":17658,"echoes":17659,"scandinavian":17660,"##dham":17661,"hugging":17662,"vitamin":17663,"salute":17664,"micah":17665,"hind":17666,"trader":17667,"##sper":17668,"radioactive":17669,"##ndra":17670,"militants":17671,"poisoned":17672,"ratified":17673,"remark":17674,"campeonato":17675,"deprived":17676,"wander":17677,"prop":17678,"##dong":17679,"outlook":17680,"##tani":17681,"##rix":17682,"##eye":17683,"chiang":17684,"darcy":17685,"##oping":17686,"mandolin":17687,"spice":17688,"statesman":17689,"babylon":17690,"182":17691,"walled":17692,"forgetting":17693,"afro":17694,"##cap":17695,"158":17696,"giorgio":17697,"buffer":17698,"##polis":17699,"planetary":17700,"##gis":17701,"overlap":17702,"terminals":17703,"kinda":17704,"centenary":17705,"##bir":17706,"arising":17707,"manipulate":17708,"elm":17709,"ke":17710,"1770":17711,"ak":17712,"##tad":17713,"chrysler":17714,"mapped":17715,"moose":17716,"pomeranian":17717,"quad":17718,"macarthur":17719,"assemblies":17720,"shoreline":17721,"recalls":17722,"stratford":17723,"##rted":17724,"noticeable":17725,"##evic":17726,"imp":17727,"##rita":17728,"##sque":17729,"accustomed":17730,"supplying":17731,"tents":17732,"disgusted":17733,"vogue":17734,"sipped":17735,"filters":17736,"khz":17737,"reno":17738,"selecting":17739,"luftwaffe":17740,"mcmahon":17741,"tyne":17742,"masterpiece":17743,"carriages":17744,"collided":17745,"dunes":17746,"exercised":17747,"flare":17748,"remembers":17749,"muzzle":17750,"##mobile":17751,"heck":17752,"##rson":17753,"burgess":17754,"lunged":17755,"middleton":17756,"boycott":17757,"bilateral":17758,"##sity":17759,"hazardous":17760,"lumpur":17761,"multiplayer":17762,"spotlight":17763,"jackets":17764,"goldman":17765,"liege":17766,"porcelain":17767,"rag":17768,"waterford":17769,"benz":17770,"attracts":17771,"hopeful":17772,"battling":17773,"ottomans":17774,"kensington":17775,"baked":17776,"hymns":17777,"cheyenne":17778,"lattice":17779,"levine":17780,"borrow":17781,"polymer":17782,"clashes":17783,"michaels":17784,"monitored":17785,"commitments":17786,"denounced":17787,"##25":17788,"##von":17789,"cavity":17790,"##oney":17791,"hobby":17792,"akin":17793,"##holders":17794,"futures":17795,"intricate":17796,"cornish":17797,"patty":17798,"##oned":17799,"illegally":17800,"dolphin":17801,"##lag":17802,"barlow":17803,"yellowish":17804,"maddie":17805,"apologized":17806,"luton":17807,"plagued":17808,"##puram":17809,"nana":17810,"##rds":17811,"sway":17812,"fanny":17813,"łodz":17814,"##rino":17815,"psi":17816,"suspicions":17817,"hanged":17818,"##eding":17819,"initiate":17820,"charlton":17821,"##por":17822,"nak":17823,"competent":17824,"235":17825,"analytical":17826,"annex":17827,"wardrobe":17828,"reservations":17829,"##rma":17830,"sect":17831,"162":17832,"fairfax":17833,"hedge":17834,"piled":17835,"buckingham":17836,"uneven":17837,"bauer":17838,"simplicity":17839,"snyder":17840,"interpret":17841,"accountability":17842,"donors":17843,"moderately":17844,"byrd":17845,"continents":17846,"##cite":17847,"##max":17848,"disciple":17849,"hr":17850,"jamaican":17851,"ping":17852,"nominees":17853,"##uss":17854,"mongolian":17855,"diver":17856,"attackers":17857,"eagerly":17858,"ideological":17859,"pillows":17860,"miracles":17861,"apartheid":17862,"revolver":17863,"sulfur":17864,"clinics":17865,"moran":17866,"163":17867,"##enko":17868,"ile":17869,"katy":17870,"rhetoric":17871,"##icated":17872,"chronology":17873,"recycling":17874,"##hrer":17875,"elongated":17876,"mughal":17877,"pascal":17878,"profiles":17879,"vibration":17880,"databases":17881,"domination":17882,"##fare":17883,"##rant":17884,"matthias":17885,"digest":17886,"rehearsal":17887,"polling":17888,"weiss":17889,"initiation":17890,"reeves":17891,"clinging":17892,"flourished":17893,"impress":17894,"ngo":17895,"##hoff":17896,"##ume":17897,"buckley":17898,"symposium":17899,"rhythms":17900,"weed":17901,"emphasize":17902,"transforming":17903,"##taking":17904,"##gence":17905,"##yman":17906,"accountant":17907,"analyze":17908,"flicker":17909,"foil":17910,"priesthood":17911,"voluntarily":17912,"decreases":17913,"##80":17914,"##hya":17915,"slater":17916,"sv":17917,"charting":17918,"mcgill":17919,"##lde":17920,"moreno":17921,"##iu":17922,"besieged":17923,"zur":17924,"robes":17925,"##phic":17926,"admitting":17927,"api":17928,"deported":17929,"turmoil":17930,"peyton":17931,"earthquakes":17932,"##ares":17933,"nationalists":17934,"beau":17935,"clair":17936,"brethren":17937,"interrupt":17938,"welch":17939,"curated":17940,"galerie":17941,"requesting":17942,"164":17943,"##ested":17944,"impending":17945,"steward":17946,"viper":17947,"##vina":17948,"complaining":17949,"beautifully":17950,"brandy":17951,"foam":17952,"nl":17953,"1660":17954,"##cake":17955,"alessandro":17956,"punches":17957,"laced":17958,"explanations":17959,"##lim":17960,"attribute":17961,"clit":17962,"reggie":17963,"discomfort":17964,"##cards":17965,"smoothed":17966,"whales":17967,"##cene":17968,"adler":17969,"countered":17970,"duffy":17971,"disciplinary":17972,"widening":17973,"recipe":17974,"reliance":17975,"conducts":17976,"goats":17977,"gradient":17978,"preaching":17979,"##shaw":17980,"matilda":17981,"quasi":17982,"striped":17983,"meridian":17984,"cannabis":17985,"cordoba":17986,"certificates":17987,"##agh":17988,"##tering":17989,"graffiti":17990,"hangs":17991,"pilgrims":17992,"repeats":17993,"##ych":17994,"revive":17995,"urine":17996,"etat":17997,"##hawk":17998,"fueled":17999,"belts":18000,"fuzzy":18001,"susceptible":18002,"##hang":18003,"mauritius":18004,"salle":18005,"sincere":18006,"beers":18007,"hooks":18008,"##cki":18009,"arbitration":18010,"entrusted":18011,"advise":18012,"sniffed":18013,"seminar":18014,"junk":18015,"donnell":18016,"processors":18017,"principality":18018,"strapped":18019,"celia":18020,"mendoza":18021,"everton":18022,"fortunes":18023,"prejudice":18024,"starving":18025,"reassigned":18026,"steamer":18027,"##lund":18028,"tuck":18029,"evenly":18030,"foreman":18031,"##ffen":18032,"dans":18033,"375":18034,"envisioned":18035,"slit":18036,"##xy":18037,"baseman":18038,"liberia":18039,"rosemary":18040,"##weed":18041,"electrified":18042,"periodically":18043,"potassium":18044,"stride":18045,"contexts":18046,"sperm":18047,"slade":18048,"mariners":18049,"influx":18050,"bianca":18051,"subcommittee":18052,"##rane":18053,"spilling":18054,"icao":18055,"estuary":18056,"##nock":18057,"delivers":18058,"iphone":18059,"##ulata":18060,"isa":18061,"mira":18062,"bohemian":18063,"dessert":18064,"##sbury":18065,"welcoming":18066,"proudly":18067,"slowing":18068,"##chs":18069,"musee":18070,"ascension":18071,"russ":18072,"##vian":18073,"waits":18074,"##psy":18075,"africans":18076,"exploit":18077,"##morphic":18078,"gov":18079,"eccentric":18080,"crab":18081,"peck":18082,"##ull":18083,"entrances":18084,"formidable":18085,"marketplace":18086,"groom":18087,"bolted":18088,"metabolism":18089,"patton":18090,"robbins":18091,"courier":18092,"payload":18093,"endure":18094,"##ifier":18095,"andes":18096,"refrigerator":18097,"##pr":18098,"ornate":18099,"##uca":18100,"ruthless":18101,"illegitimate":18102,"masonry":18103,"strasbourg":18104,"bikes":18105,"adobe":18106,"##³":18107,"apples":18108,"quintet":18109,"willingly":18110,"niche":18111,"bakery":18112,"corpses":18113,"energetic":18114,"##cliffe":18115,"##sser":18116,"##ards":18117,"177":18118,"centimeters":18119,"centro":18120,"fuscous":18121,"cretaceous":18122,"rancho":18123,"##yde":18124,"andrei":18125,"telecom":18126,"tottenham":18127,"oasis":18128,"ordination":18129,"vulnerability":18130,"presiding":18131,"corey":18132,"cp":18133,"penguins":18134,"sims":18135,"##pis":18136,"malawi":18137,"piss":18138,"##48":18139,"correction":18140,"##cked":18141,"##ffle":18142,"##ryn":18143,"countdown":18144,"detectives":18145,"psychiatrist":18146,"psychedelic":18147,"dinosaurs":18148,"blouse":18149,"##get":18150,"choi":18151,"vowed":18152,"##oz":18153,"randomly":18154,"##pol":18155,"49ers":18156,"scrub":18157,"blanche":18158,"bruins":18159,"dusseldorf":18160,"##using":18161,"unwanted":18162,"##ums":18163,"212":18164,"dominique":18165,"elevations":18166,"headlights":18167,"om":18168,"laguna":18169,"##oga":18170,"1750":18171,"famously":18172,"ignorance":18173,"shrewsbury":18174,"##aine":18175,"ajax":18176,"breuning":18177,"che":18178,"confederacy":18179,"greco":18180,"overhaul":18181,"##screen":18182,"paz":18183,"skirts":18184,"disagreement":18185,"cruelty":18186,"jagged":18187,"phoebe":18188,"shifter":18189,"hovered":18190,"viruses":18191,"##wes":18192,"mandy":18193,"##lined":18194,"##gc":18195,"landlord":18196,"squirrel":18197,"dashed":18198,"##ι":18199,"ornamental":18200,"gag":18201,"wally":18202,"grange":18203,"literal":18204,"spurs":18205,"undisclosed":18206,"proceeding":18207,"yin":18208,"##text":18209,"billie":18210,"orphan":18211,"spanned":18212,"humidity":18213,"indy":18214,"weighted":18215,"presentations":18216,"explosions":18217,"lucian":18218,"##tary":18219,"vaughn":18220,"hindus":18221,"##anga":18222,"##hell":18223,"psycho":18224,"171":18225,"daytona":18226,"protects":18227,"efficiently":18228,"rematch":18229,"sly":18230,"tandem":18231,"##oya":18232,"rebranded":18233,"impaired":18234,"hee":18235,"metropolis":18236,"peach":18237,"godfrey":18238,"diaspora":18239,"ethnicity":18240,"prosperous":18241,"gleaming":18242,"dar":18243,"grossing":18244,"playback":18245,"##rden":18246,"stripe":18247,"pistols":18248,"##tain":18249,"births":18250,"labelled":18251,"##cating":18252,"172":18253,"rudy":18254,"alba":18255,"##onne":18256,"aquarium":18257,"hostility":18258,"##gb":18259,"##tase":18260,"shudder":18261,"sumatra":18262,"hardest":18263,"lakers":18264,"consonant":18265,"creeping":18266,"demos":18267,"homicide":18268,"capsule":18269,"zeke":18270,"liberties":18271,"expulsion":18272,"pueblo":18273,"##comb":18274,"trait":18275,"transporting":18276,"##ddin":18277,"##neck":18278,"##yna":18279,"depart":18280,"gregg":18281,"mold":18282,"ledge":18283,"hangar":18284,"oldham":18285,"playboy":18286,"termination":18287,"analysts":18288,"gmbh":18289,"romero":18290,"##itic":18291,"insist":18292,"cradle":18293,"filthy":18294,"brightness":18295,"slash":18296,"shootout":18297,"deposed":18298,"bordering":18299,"##truct":18300,"isis":18301,"microwave":18302,"tumbled":18303,"sheltered":18304,"cathy":18305,"werewolves":18306,"messy":18307,"andersen":18308,"convex":18309,"clapped":18310,"clinched":18311,"satire":18312,"wasting":18313,"edo":18314,"vc":18315,"rufus":18316,"##jak":18317,"mont":18318,"##etti":18319,"poznan":18320,"##keeping":18321,"restructuring":18322,"transverse":18323,"##rland":18324,"azerbaijani":18325,"slovene":18326,"gestures":18327,"roommate":18328,"choking":18329,"shear":18330,"##quist":18331,"vanguard":18332,"oblivious":18333,"##hiro":18334,"disagreed":18335,"baptism":18336,"##lich":18337,"coliseum":18338,"##aceae":18339,"salvage":18340,"societe":18341,"cory":18342,"locke":18343,"relocation":18344,"relying":18345,"versailles":18346,"ahl":18347,"swelling":18348,"##elo":18349,"cheerful":18350,"##word":18351,"##edes":18352,"gin":18353,"sarajevo":18354,"obstacle":18355,"diverted":18356,"##nac":18357,"messed":18358,"thoroughbred":18359,"fluttered":18360,"utrecht":18361,"chewed":18362,"acquaintance":18363,"assassins":18364,"dispatch":18365,"mirza":18366,"##wart":18367,"nike":18368,"salzburg":18369,"swell":18370,"yen":18371,"##gee":18372,"idle":18373,"ligue":18374,"samson":18375,"##nds":18376,"##igh":18377,"playful":18378,"spawned":18379,"##cise":18380,"tease":18381,"##case":18382,"burgundy":18383,"##bot":18384,"stirring":18385,"skeptical":18386,"interceptions":18387,"marathi":18388,"##dies":18389,"bedrooms":18390,"aroused":18391,"pinch":18392,"##lik":18393,"preferences":18394,"tattoos":18395,"buster":18396,"digitally":18397,"projecting":18398,"rust":18399,"##ital":18400,"kitten":18401,"priorities":18402,"addison":18403,"pseudo":18404,"##guard":18405,"dusk":18406,"icons":18407,"sermon":18408,"##psis":18409,"##iba":18410,"bt":18411,"##lift":18412,"##xt":18413,"ju":18414,"truce":18415,"rink":18416,"##dah":18417,"##wy":18418,"defects":18419,"psychiatry":18420,"offences":18421,"calculate":18422,"glucose":18423,"##iful":18424,"##rized":18425,"##unda":18426,"francaise":18427,"##hari":18428,"richest":18429,"warwickshire":18430,"carly":18431,"1763":18432,"purity":18433,"redemption":18434,"lending":18435,"##cious":18436,"muse":18437,"bruises":18438,"cerebral":18439,"aero":18440,"carving":18441,"##name":18442,"preface":18443,"terminology":18444,"invade":18445,"monty":18446,"##int":18447,"anarchist":18448,"blurred":18449,"##iled":18450,"rossi":18451,"treats":18452,"guts":18453,"shu":18454,"foothills":18455,"ballads":18456,"undertaking":18457,"premise":18458,"cecilia":18459,"affiliates":18460,"blasted":18461,"conditional":18462,"wilder":18463,"minors":18464,"drone":18465,"rudolph":18466,"buffy":18467,"swallowing":18468,"horton":18469,"attested":18470,"##hop":18471,"rutherford":18472,"howell":18473,"primetime":18474,"livery":18475,"penal":18476,"##bis":18477,"minimize":18478,"hydro":18479,"wrecked":18480,"wrought":18481,"palazzo":18482,"##gling":18483,"cans":18484,"vernacular":18485,"friedman":18486,"nobleman":18487,"shale":18488,"walnut":18489,"danielle":18490,"##ection":18491,"##tley":18492,"sears":18493,"##kumar":18494,"chords":18495,"lend":18496,"flipping":18497,"streamed":18498,"por":18499,"dracula":18500,"gallons":18501,"sacrifices":18502,"gamble":18503,"orphanage":18504,"##iman":18505,"mckenzie":18506,"##gible":18507,"boxers":18508,"daly":18509,"##balls":18510,"##ان":18511,"208":18512,"##ific":18513,"##rative":18514,"##iq":18515,"exploited":18516,"slated":18517,"##uity":18518,"circling":18519,"hillary":18520,"pinched":18521,"goldberg":18522,"provost":18523,"campaigning":18524,"lim":18525,"piles":18526,"ironically":18527,"jong":18528,"mohan":18529,"successors":18530,"usaf":18531,"##tem":18532,"##ught":18533,"autobiographical":18534,"haute":18535,"preserves":18536,"##ending":18537,"acquitted":18538,"comparisons":18539,"203":18540,"hydroelectric":18541,"gangs":18542,"cypriot":18543,"torpedoes":18544,"rushes":18545,"chrome":18546,"derive":18547,"bumps":18548,"instability":18549,"fiat":18550,"pets":18551,"##mbe":18552,"silas":18553,"dye":18554,"reckless":18555,"settler":18556,"##itation":18557,"info":18558,"heats":18559,"##writing":18560,"176":18561,"canonical":18562,"maltese":18563,"fins":18564,"mushroom":18565,"stacy":18566,"aspen":18567,"avid":18568,"##kur":18569,"##loading":18570,"vickers":18571,"gaston":18572,"hillside":18573,"statutes":18574,"wilde":18575,"gail":18576,"kung":18577,"sabine":18578,"comfortably":18579,"motorcycles":18580,"##rgo":18581,"169":18582,"pneumonia":18583,"fetch":18584,"##sonic":18585,"axel":18586,"faintly":18587,"parallels":18588,"##oop":18589,"mclaren":18590,"spouse":18591,"compton":18592,"interdisciplinary":18593,"miner":18594,"##eni":18595,"181":18596,"clamped":18597,"##chal":18598,"##llah":18599,"separates":18600,"versa":18601,"##mler":18602,"scarborough":18603,"labrador":18604,"##lity":18605,"##osing":18606,"rutgers":18607,"hurdles":18608,"como":18609,"166":18610,"burt":18611,"divers":18612,"##100":18613,"wichita":18614,"cade":18615,"coincided":18616,"##erson":18617,"bruised":18618,"mla":18619,"##pper":18620,"vineyard":18621,"##ili":18622,"##brush":18623,"notch":18624,"mentioning":18625,"jase":18626,"hearted":18627,"kits":18628,"doe":18629,"##acle":18630,"pomerania":18631,"##ady":18632,"ronan":18633,"seizure":18634,"pavel":18635,"problematic":18636,"##zaki":18637,"domenico":18638,"##ulin":18639,"catering":18640,"penelope":18641,"dependence":18642,"parental":18643,"emilio":18644,"ministerial":18645,"atkinson":18646,"##bolic":18647,"clarkson":18648,"chargers":18649,"colby":18650,"grill":18651,"peeked":18652,"arises":18653,"summon":18654,"##aged":18655,"fools":18656,"##grapher":18657,"faculties":18658,"qaeda":18659,"##vial":18660,"garner":18661,"refurbished":18662,"##hwa":18663,"geelong":18664,"disasters":18665,"nudged":18666,"bs":18667,"shareholder":18668,"lori":18669,"algae":18670,"reinstated":18671,"rot":18672,"##ades":18673,"##nous":18674,"invites":18675,"stainless":18676,"183":18677,"inclusive":18678,"##itude":18679,"diocesan":18680,"til":18681,"##icz":18682,"denomination":18683,"##xa":18684,"benton":18685,"floral":18686,"registers":18687,"##ider":18688,"##erman":18689,"##kell":18690,"absurd":18691,"brunei":18692,"guangzhou":18693,"hitter":18694,"retaliation":18695,"##uled":18696,"##eve":18697,"blanc":18698,"nh":18699,"consistency":18700,"contamination":18701,"##eres":18702,"##rner":18703,"dire":18704,"palermo":18705,"broadcasters":18706,"diaries":18707,"inspire":18708,"vols":18709,"brewer":18710,"tightening":18711,"ky":18712,"mixtape":18713,"hormone":18714,"##tok":18715,"stokes":18716,"##color":18717,"##dly":18718,"##ssi":18719,"pg":18720,"##ometer":18721,"##lington":18722,"sanitation":18723,"##tility":18724,"intercontinental":18725,"apps":18726,"##adt":18727,"¹⁄₂":18728,"cylinders":18729,"economies":18730,"favourable":18731,"unison":18732,"croix":18733,"gertrude":18734,"odyssey":18735,"vanity":18736,"dangling":18737,"##logists":18738,"upgrades":18739,"dice":18740,"middleweight":18741,"practitioner":18742,"##ight":18743,"206":18744,"henrik":18745,"parlor":18746,"orion":18747,"angered":18748,"lac":18749,"python":18750,"blurted":18751,"##rri":18752,"sensual":18753,"intends":18754,"swings":18755,"angled":18756,"##phs":18757,"husky":18758,"attain":18759,"peerage":18760,"precinct":18761,"textiles":18762,"cheltenham":18763,"shuffled":18764,"dai":18765,"confess":18766,"tasting":18767,"bhutan":18768,"##riation":18769,"tyrone":18770,"segregation":18771,"abrupt":18772,"ruiz":18773,"##rish":18774,"smirked":18775,"blackwell":18776,"confidential":18777,"browning":18778,"amounted":18779,"##put":18780,"vase":18781,"scarce":18782,"fabulous":18783,"raided":18784,"staple":18785,"guyana":18786,"unemployed":18787,"glider":18788,"shay":18789,"##tow":18790,"carmine":18791,"troll":18792,"intervene":18793,"squash":18794,"superstar":18795,"##uce":18796,"cylindrical":18797,"len":18798,"roadway":18799,"researched":18800,"handy":18801,"##rium":18802,"##jana":18803,"meta":18804,"lao":18805,"declares":18806,"##rring":18807,"##tadt":18808,"##elin":18809,"##kova":18810,"willem":18811,"shrubs":18812,"napoleonic":18813,"realms":18814,"skater":18815,"qi":18816,"volkswagen":18817,"##ł":18818,"tad":18819,"hara":18820,"archaeologist":18821,"awkwardly":18822,"eerie":18823,"##kind":18824,"wiley":18825,"##heimer":18826,"##24":18827,"titus":18828,"organizers":18829,"cfl":18830,"crusaders":18831,"lama":18832,"usb":18833,"vent":18834,"enraged":18835,"thankful":18836,"occupants":18837,"maximilian":18838,"##gaard":18839,"possessing":18840,"textbooks":18841,"##oran":18842,"collaborator":18843,"quaker":18844,"##ulo":18845,"avalanche":18846,"mono":18847,"silky":18848,"straits":18849,"isaiah":18850,"mustang":18851,"surged":18852,"resolutions":18853,"potomac":18854,"descend":18855,"cl":18856,"kilograms":18857,"plato":18858,"strains":18859,"saturdays":18860,"##olin":18861,"bernstein":18862,"##ype":18863,"holstein":18864,"ponytail":18865,"##watch":18866,"belize":18867,"conversely":18868,"heroine":18869,"perpetual":18870,"##ylus":18871,"charcoal":18872,"piedmont":18873,"glee":18874,"negotiating":18875,"backdrop":18876,"prologue":18877,"##jah":18878,"##mmy":18879,"pasadena":18880,"climbs":18881,"ramos":18882,"sunni":18883,"##holm":18884,"##tner":18885,"##tri":18886,"anand":18887,"deficiency":18888,"hertfordshire":18889,"stout":18890,"##avi":18891,"aperture":18892,"orioles":18893,"##irs":18894,"doncaster":18895,"intrigued":18896,"bombed":18897,"coating":18898,"otis":18899,"##mat":18900,"cocktail":18901,"##jit":18902,"##eto":18903,"amir":18904,"arousal":18905,"sar":18906,"##proof":18907,"##act":18908,"##ories":18909,"dixie":18910,"pots":18911,"##bow":18912,"whereabouts":18913,"159":18914,"##fted":18915,"drains":18916,"bullying":18917,"cottages":18918,"scripture":18919,"coherent":18920,"fore":18921,"poe":18922,"appetite":18923,"##uration":18924,"sampled":18925,"##ators":18926,"##dp":18927,"derrick":18928,"rotor":18929,"jays":18930,"peacock":18931,"installment":18932,"##rro":18933,"advisors":18934,"##coming":18935,"rodeo":18936,"scotch":18937,"##mot":18938,"##db":18939,"##fen":18940,"##vant":18941,"ensued":18942,"rodrigo":18943,"dictatorship":18944,"martyrs":18945,"twenties":18946,"##н":18947,"towed":18948,"incidence":18949,"marta":18950,"rainforest":18951,"sai":18952,"scaled":18953,"##cles":18954,"oceanic":18955,"qualifiers":18956,"symphonic":18957,"mcbride":18958,"dislike":18959,"generalized":18960,"aubrey":18961,"colonization":18962,"##iation":18963,"##lion":18964,"##ssing":18965,"disliked":18966,"lublin":18967,"salesman":18968,"##ulates":18969,"spherical":18970,"whatsoever":18971,"sweating":18972,"avalon":18973,"contention":18974,"punt":18975,"severity":18976,"alderman":18977,"atari":18978,"##dina":18979,"##grant":18980,"##rop":18981,"scarf":18982,"seville":18983,"vertices":18984,"annexation":18985,"fairfield":18986,"fascination":18987,"inspiring":18988,"launches":18989,"palatinate":18990,"regretted":18991,"##rca":18992,"feral":18993,"##iom":18994,"elk":18995,"nap":18996,"olsen":18997,"reddy":18998,"yong":18999,"##leader":19000,"##iae":19001,"garment":19002,"transports":19003,"feng":19004,"gracie":19005,"outrage":19006,"viceroy":19007,"insides":19008,"##esis":19009,"breakup":19010,"grady":19011,"organizer":19012,"softer":19013,"grimaced":19014,"222":19015,"murals":19016,"galicia":19017,"arranging":19018,"vectors":19019,"##rsten":19020,"bas":19021,"##sb":19022,"##cens":19023,"sloan":19024,"##eka":19025,"bitten":19026,"ara":19027,"fender":19028,"nausea":19029,"bumped":19030,"kris":19031,"banquet":19032,"comrades":19033,"detector":19034,"persisted":19035,"##llan":19036,"adjustment":19037,"endowed":19038,"cinemas":19039,"##shot":19040,"sellers":19041,"##uman":19042,"peek":19043,"epa":19044,"kindly":19045,"neglect":19046,"simpsons":19047,"talon":19048,"mausoleum":19049,"runaway":19050,"hangul":19051,"lookout":19052,"##cic":19053,"rewards":19054,"coughed":19055,"acquainted":19056,"chloride":19057,"##ald":19058,"quicker":19059,"accordion":19060,"neolithic":19061,"##qa":19062,"artemis":19063,"coefficient":19064,"lenny":19065,"pandora":19066,"tx":19067,"##xed":19068,"ecstasy":19069,"litter":19070,"segunda":19071,"chairperson":19072,"gemma":19073,"hiss":19074,"rumor":19075,"vow":19076,"nasal":19077,"antioch":19078,"compensate":19079,"patiently":19080,"transformers":19081,"##eded":19082,"judo":19083,"morrow":19084,"penis":19085,"posthumous":19086,"philips":19087,"bandits":19088,"husbands":19089,"denote":19090,"flaming":19091,"##any":19092,"##phones":19093,"langley":19094,"yorker":19095,"1760":19096,"walters":19097,"##uo":19098,"##kle":19099,"gubernatorial":19100,"fatty":19101,"samsung":19102,"leroy":19103,"outlaw":19104,"##nine":19105,"unpublished":19106,"poole":19107,"jakob":19108,"##ᵢ":19109,"##ₙ":19110,"crete":19111,"distorted":19112,"superiority":19113,"##dhi":19114,"intercept":19115,"crust":19116,"mig":19117,"claus":19118,"crashes":19119,"positioning":19120,"188":19121,"stallion":19122,"301":19123,"frontal":19124,"armistice":19125,"##estinal":19126,"elton":19127,"aj":19128,"encompassing":19129,"camel":19130,"commemorated":19131,"malaria":19132,"woodward":19133,"calf":19134,"cigar":19135,"penetrate":19136,"##oso":19137,"willard":19138,"##rno":19139,"##uche":19140,"illustrate":19141,"amusing":19142,"convergence":19143,"noteworthy":19144,"##lma":19145,"##rva":19146,"journeys":19147,"realise":19148,"manfred":19149,"##sable":19150,"410":19151,"##vocation":19152,"hearings":19153,"fiance":19154,"##posed":19155,"educators":19156,"provoked":19157,"adjusting":19158,"##cturing":19159,"modular":19160,"stockton":19161,"paterson":19162,"vlad":19163,"rejects":19164,"electors":19165,"selena":19166,"maureen":19167,"##tres":19168,"uber":19169,"##rce":19170,"swirled":19171,"##num":19172,"proportions":19173,"nanny":19174,"pawn":19175,"naturalist":19176,"parma":19177,"apostles":19178,"awoke":19179,"ethel":19180,"wen":19181,"##bey":19182,"monsoon":19183,"overview":19184,"##inating":19185,"mccain":19186,"rendition":19187,"risky":19188,"adorned":19189,"##ih":19190,"equestrian":19191,"germain":19192,"nj":19193,"conspicuous":19194,"confirming":19195,"##yoshi":19196,"shivering":19197,"##imeter":19198,"milestone":19199,"rumours":19200,"flinched":19201,"bounds":19202,"smacked":19203,"token":19204,"##bei":19205,"lectured":19206,"automobiles":19207,"##shore":19208,"impacted":19209,"##iable":19210,"nouns":19211,"nero":19212,"##leaf":19213,"ismail":19214,"prostitute":19215,"trams":19216,"##lace":19217,"bridget":19218,"sud":19219,"stimulus":19220,"impressions":19221,"reins":19222,"revolves":19223,"##oud":19224,"##gned":19225,"giro":19226,"honeymoon":19227,"##swell":19228,"criterion":19229,"##sms":19230,"##uil":19231,"libyan":19232,"prefers":19233,"##osition":19234,"211":19235,"preview":19236,"sucks":19237,"accusation":19238,"bursts":19239,"metaphor":19240,"diffusion":19241,"tolerate":19242,"faye":19243,"betting":19244,"cinematographer":19245,"liturgical":19246,"specials":19247,"bitterly":19248,"humboldt":19249,"##ckle":19250,"flux":19251,"rattled":19252,"##itzer":19253,"archaeologists":19254,"odor":19255,"authorised":19256,"marshes":19257,"discretion":19258,"##ов":19259,"alarmed":19260,"archaic":19261,"inverse":19262,"##leton":19263,"explorers":19264,"##pine":19265,"drummond":19266,"tsunami":19267,"woodlands":19268,"##minate":19269,"##tland":19270,"booklet":19271,"insanity":19272,"owning":19273,"insert":19274,"crafted":19275,"calculus":19276,"##tore":19277,"receivers":19278,"##bt":19279,"stung":19280,"##eca":19281,"##nched":19282,"prevailing":19283,"travellers":19284,"eyeing":19285,"lila":19286,"graphs":19287,"##borne":19288,"178":19289,"julien":19290,"##won":19291,"morale":19292,"adaptive":19293,"therapist":19294,"erica":19295,"cw":19296,"libertarian":19297,"bowman":19298,"pitches":19299,"vita":19300,"##ional":19301,"crook":19302,"##ads":19303,"##entation":19304,"caledonia":19305,"mutiny":19306,"##sible":19307,"1840s":19308,"automation":19309,"##ß":19310,"flock":19311,"##pia":19312,"ironic":19313,"pathology":19314,"##imus":19315,"remarried":19316,"##22":19317,"joker":19318,"withstand":19319,"energies":19320,"##att":19321,"shropshire":19322,"hostages":19323,"madeleine":19324,"tentatively":19325,"conflicting":19326,"mateo":19327,"recipes":19328,"euros":19329,"ol":19330,"mercenaries":19331,"nico":19332,"##ndon":19333,"albuquerque":19334,"augmented":19335,"mythical":19336,"bel":19337,"freud":19338,"##child":19339,"cough":19340,"##lica":19341,"365":19342,"freddy":19343,"lillian":19344,"genetically":19345,"nuremberg":19346,"calder":19347,"209":19348,"bonn":19349,"outdoors":19350,"paste":19351,"suns":19352,"urgency":19353,"vin":19354,"restraint":19355,"tyson":19356,"##cera":19357,"##selle":19358,"barrage":19359,"bethlehem":19360,"kahn":19361,"##par":19362,"mounts":19363,"nippon":19364,"barony":19365,"happier":19366,"ryu":19367,"makeshift":19368,"sheldon":19369,"blushed":19370,"castillo":19371,"barking":19372,"listener":19373,"taped":19374,"bethel":19375,"fluent":19376,"headlines":19377,"pornography":19378,"rum":19379,"disclosure":19380,"sighing":19381,"mace":19382,"doubling":19383,"gunther":19384,"manly":19385,"##plex":19386,"rt":19387,"interventions":19388,"physiological":19389,"forwards":19390,"emerges":19391,"##tooth":19392,"##gny":19393,"compliment":19394,"rib":19395,"recession":19396,"visibly":19397,"barge":19398,"faults":19399,"connector":19400,"exquisite":19401,"prefect":19402,"##rlin":19403,"patio":19404,"##cured":19405,"elevators":19406,"brandt":19407,"italics":19408,"pena":19409,"173":19410,"wasp":19411,"satin":19412,"ea":19413,"botswana":19414,"graceful":19415,"respectable":19416,"##jima":19417,"##rter":19418,"##oic":19419,"franciscan":19420,"generates":19421,"##dl":19422,"alfredo":19423,"disgusting":19424,"##olate":19425,"##iously":19426,"sherwood":19427,"warns":19428,"cod":19429,"promo":19430,"cheryl":19431,"sino":19432,"##ة":19433,"##escu":19434,"twitch":19435,"##zhi":19436,"brownish":19437,"thom":19438,"ortiz":19439,"##dron":19440,"densely":19441,"##beat":19442,"carmel":19443,"reinforce":19444,"##bana":19445,"187":19446,"anastasia":19447,"downhill":19448,"vertex":19449,"contaminated":19450,"remembrance":19451,"harmonic":19452,"homework":19453,"##sol":19454,"fiancee":19455,"gears":19456,"olds":19457,"angelica":19458,"loft":19459,"ramsay":19460,"quiz":19461,"colliery":19462,"sevens":19463,"##cape":19464,"autism":19465,"##hil":19466,"walkway":19467,"##boats":19468,"ruben":19469,"abnormal":19470,"ounce":19471,"khmer":19472,"##bbe":19473,"zachary":19474,"bedside":19475,"morphology":19476,"punching":19477,"##olar":19478,"sparrow":19479,"convinces":19480,"##35":19481,"hewitt":19482,"queer":19483,"remastered":19484,"rods":19485,"mabel":19486,"solemn":19487,"notified":19488,"lyricist":19489,"symmetric":19490,"##xide":19491,"174":19492,"encore":19493,"passports":19494,"wildcats":19495,"##uni":19496,"baja":19497,"##pac":19498,"mildly":19499,"##ease":19500,"bleed":19501,"commodity":19502,"mounds":19503,"glossy":19504,"orchestras":19505,"##omo":19506,"damian":19507,"prelude":19508,"ambitions":19509,"##vet":19510,"awhile":19511,"remotely":19512,"##aud":19513,"asserts":19514,"imply":19515,"##iques":19516,"distinctly":19517,"modelling":19518,"remedy":19519,"##dded":19520,"windshield":19521,"dani":19522,"xiao":19523,"##endra":19524,"audible":19525,"powerplant":19526,"1300":19527,"invalid":19528,"elemental":19529,"acquisitions":19530,"##hala":19531,"immaculate":19532,"libby":19533,"plata":19534,"smuggling":19535,"ventilation":19536,"denoted":19537,"minh":19538,"##morphism":19539,"430":19540,"differed":19541,"dion":19542,"kelley":19543,"lore":19544,"mocking":19545,"sabbath":19546,"spikes":19547,"hygiene":19548,"drown":19549,"runoff":19550,"stylized":19551,"tally":19552,"liberated":19553,"aux":19554,"interpreter":19555,"righteous":19556,"aba":19557,"siren":19558,"reaper":19559,"pearce":19560,"millie":19561,"##cier":19562,"##yra":19563,"gaius":19564,"##iso":19565,"captures":19566,"##ttering":19567,"dorm":19568,"claudio":19569,"##sic":19570,"benches":19571,"knighted":19572,"blackness":19573,"##ored":19574,"discount":19575,"fumble":19576,"oxidation":19577,"routed":19578,"##ς":19579,"novak":19580,"perpendicular":19581,"spoiled":19582,"fracture":19583,"splits":19584,"##urt":19585,"pads":19586,"topology":19587,"##cats":19588,"axes":19589,"fortunate":19590,"offenders":19591,"protestants":19592,"esteem":19593,"221":19594,"broadband":19595,"convened":19596,"frankly":19597,"hound":19598,"prototypes":19599,"isil":19600,"facilitated":19601,"keel":19602,"##sher":19603,"sahara":19604,"awaited":19605,"bubba":19606,"orb":19607,"prosecutors":19608,"186":19609,"hem":19610,"520":19611,"##xing":19612,"relaxing":19613,"remnant":19614,"romney":19615,"sorted":19616,"slalom":19617,"stefano":19618,"ulrich":19619,"##active":19620,"exemption":19621,"folder":19622,"pauses":19623,"foliage":19624,"hitchcock":19625,"epithet":19626,"204":19627,"criticisms":19628,"##aca":19629,"ballistic":19630,"brody":19631,"hinduism":19632,"chaotic":19633,"youths":19634,"equals":19635,"##pala":19636,"pts":19637,"thicker":19638,"analogous":19639,"capitalist":19640,"improvised":19641,"overseeing":19642,"sinatra":19643,"ascended":19644,"beverage":19645,"##tl":19646,"straightforward":19647,"##kon":19648,"curran":19649,"##west":19650,"bois":19651,"325":19652,"induce":19653,"surveying":19654,"emperors":19655,"sax":19656,"unpopular":19657,"##kk":19658,"cartoonist":19659,"fused":19660,"##mble":19661,"unto":19662,"##yuki":19663,"localities":19664,"##cko":19665,"##ln":19666,"darlington":19667,"slain":19668,"academie":19669,"lobbying":19670,"sediment":19671,"puzzles":19672,"##grass":19673,"defiance":19674,"dickens":19675,"manifest":19676,"tongues":19677,"alumnus":19678,"arbor":19679,"coincide":19680,"184":19681,"appalachian":19682,"mustafa":19683,"examiner":19684,"cabaret":19685,"traumatic":19686,"yves":19687,"bracelet":19688,"draining":19689,"heroin":19690,"magnum":19691,"baths":19692,"odessa":19693,"consonants":19694,"mitsubishi":19695,"##gua":19696,"kellan":19697,"vaudeville":19698,"##fr":19699,"joked":19700,"null":19701,"straps":19702,"probation":19703,"##ław":19704,"ceded":19705,"interfaces":19706,"##pas":19707,"##zawa":19708,"blinding":19709,"viet":19710,"224":19711,"rothschild":19712,"museo":19713,"640":19714,"huddersfield":19715,"##vr":19716,"tactic":19717,"##storm":19718,"brackets":19719,"dazed":19720,"incorrectly":19721,"##vu":19722,"reg":19723,"glazed":19724,"fearful":19725,"manifold":19726,"benefited":19727,"irony":19728,"##sun":19729,"stumbling":19730,"##rte":19731,"willingness":19732,"balkans":19733,"mei":19734,"wraps":19735,"##aba":19736,"injected":19737,"##lea":19738,"gu":19739,"syed":19740,"harmless":19741,"##hammer":19742,"bray":19743,"takeoff":19744,"poppy":19745,"timor":19746,"cardboard":19747,"astronaut":19748,"purdue":19749,"weeping":19750,"southbound":19751,"cursing":19752,"stalls":19753,"diagonal":19754,"##neer":19755,"lamar":19756,"bryce":19757,"comte":19758,"weekdays":19759,"harrington":19760,"##uba":19761,"negatively":19762,"##see":19763,"lays":19764,"grouping":19765,"##cken":19766,"##henko":19767,"affirmed":19768,"halle":19769,"modernist":19770,"##lai":19771,"hodges":19772,"smelling":19773,"aristocratic":19774,"baptized":19775,"dismiss":19776,"justification":19777,"oilers":19778,"##now":19779,"coupling":19780,"qin":19781,"snack":19782,"healer":19783,"##qing":19784,"gardener":19785,"layla":19786,"battled":19787,"formulated":19788,"stephenson":19789,"gravitational":19790,"##gill":19791,"##jun":19792,"1768":19793,"granny":19794,"coordinating":19795,"suites":19796,"##cd":19797,"##ioned":19798,"monarchs":19799,"##cote":19800,"##hips":19801,"sep":19802,"blended":19803,"apr":19804,"barrister":19805,"deposition":19806,"fia":19807,"mina":19808,"policemen":19809,"paranoid":19810,"##pressed":19811,"churchyard":19812,"covert":19813,"crumpled":19814,"creep":19815,"abandoning":19816,"tr":19817,"transmit":19818,"conceal":19819,"barr":19820,"understands":19821,"readiness":19822,"spire":19823,"##cology":19824,"##enia":19825,"##erry":19826,"610":19827,"startling":19828,"unlock":19829,"vida":19830,"bowled":19831,"slots":19832,"##nat":19833,"##islav":19834,"spaced":19835,"trusting":19836,"admire":19837,"rig":19838,"##ink":19839,"slack":19840,"##70":19841,"mv":19842,"207":19843,"casualty":19844,"##wei":19845,"classmates":19846,"##odes":19847,"##rar":19848,"##rked":19849,"amherst":19850,"furnished":19851,"evolve":19852,"foundry":19853,"menace":19854,"mead":19855,"##lein":19856,"flu":19857,"wesleyan":19858,"##kled":19859,"monterey":19860,"webber":19861,"##vos":19862,"wil":19863,"##mith":19864,"##на":19865,"bartholomew":19866,"justices":19867,"restrained":19868,"##cke":19869,"amenities":19870,"191":19871,"mediated":19872,"sewage":19873,"trenches":19874,"ml":19875,"mainz":19876,"##thus":19877,"1800s":19878,"##cula":19879,"##inski":19880,"caine":19881,"bonding":19882,"213":19883,"converts":19884,"spheres":19885,"superseded":19886,"marianne":19887,"crypt":19888,"sweaty":19889,"ensign":19890,"historia":19891,"##br":19892,"spruce":19893,"##post":19894,"##ask":19895,"forks":19896,"thoughtfully":19897,"yukon":19898,"pamphlet":19899,"ames":19900,"##uter":19901,"karma":19902,"##yya":19903,"bryn":19904,"negotiation":19905,"sighs":19906,"incapable":19907,"##mbre":19908,"##ntial":19909,"actresses":19910,"taft":19911,"##mill":19912,"luce":19913,"prevailed":19914,"##amine":19915,"1773":19916,"motionless":19917,"envoy":19918,"testify":19919,"investing":19920,"sculpted":19921,"instructors":19922,"provence":19923,"kali":19924,"cullen":19925,"horseback":19926,"##while":19927,"goodwin":19928,"##jos":19929,"gaa":19930,"norte":19931,"##ldon":19932,"modify":19933,"wavelength":19934,"abd":19935,"214":19936,"skinned":19937,"sprinter":19938,"forecast":19939,"scheduling":19940,"marries":19941,"squared":19942,"tentative":19943,"##chman":19944,"boer":19945,"##isch":19946,"bolts":19947,"swap":19948,"fisherman":19949,"assyrian":19950,"impatiently":19951,"guthrie":19952,"martins":19953,"murdoch":19954,"194":19955,"tanya":19956,"nicely":19957,"dolly":19958,"lacy":19959,"med":19960,"##45":19961,"syn":19962,"decks":19963,"fashionable":19964,"millionaire":19965,"##ust":19966,"surfing":19967,"##ml":19968,"##ision":19969,"heaved":19970,"tammy":19971,"consulate":19972,"attendees":19973,"routinely":19974,"197":19975,"fuse":19976,"saxophonist":19977,"backseat":19978,"malaya":19979,"##lord":19980,"scowl":19981,"tau":19982,"##ishly":19983,"193":19984,"sighted":19985,"steaming":19986,"##rks":19987,"303":19988,"911":19989,"##holes":19990,"##hong":19991,"ching":19992,"##wife":19993,"bless":19994,"conserved":19995,"jurassic":19996,"stacey":19997,"unix":19998,"zion":19999,"chunk":20000,"rigorous":20001,"blaine":20002,"198":20003,"peabody":20004,"slayer":20005,"dismay":20006,"brewers":20007,"nz":20008,"##jer":20009,"det":20010,"##glia":20011,"glover":20012,"postwar":20013,"int":20014,"penetration":20015,"sylvester":20016,"imitation":20017,"vertically":20018,"airlift":20019,"heiress":20020,"knoxville":20021,"viva":20022,"##uin":20023,"390":20024,"macon":20025,"##rim":20026,"##fighter":20027,"##gonal":20028,"janice":20029,"##orescence":20030,"##wari":20031,"marius":20032,"belongings":20033,"leicestershire":20034,"196":20035,"blanco":20036,"inverted":20037,"preseason":20038,"sanity":20039,"sobbing":20040,"##due":20041,"##elt":20042,"##dled":20043,"collingwood":20044,"regeneration":20045,"flickering":20046,"shortest":20047,"##mount":20048,"##osi":20049,"feminism":20050,"##lat":20051,"sherlock":20052,"cabinets":20053,"fumbled":20054,"northbound":20055,"precedent":20056,"snaps":20057,"##mme":20058,"researching":20059,"##akes":20060,"guillaume":20061,"insights":20062,"manipulated":20063,"vapor":20064,"neighbour":20065,"sap":20066,"gangster":20067,"frey":20068,"f1":20069,"stalking":20070,"scarcely":20071,"callie":20072,"barnett":20073,"tendencies":20074,"audi":20075,"doomed":20076,"assessing":20077,"slung":20078,"panchayat":20079,"ambiguous":20080,"bartlett":20081,"##etto":20082,"distributing":20083,"violating":20084,"wolverhampton":20085,"##hetic":20086,"swami":20087,"histoire":20088,"##urus":20089,"liable":20090,"pounder":20091,"groin":20092,"hussain":20093,"larsen":20094,"popping":20095,"surprises":20096,"##atter":20097,"vie":20098,"curt":20099,"##station":20100,"mute":20101,"relocate":20102,"musicals":20103,"authorization":20104,"richter":20105,"##sef":20106,"immortality":20107,"tna":20108,"bombings":20109,"##press":20110,"deteriorated":20111,"yiddish":20112,"##acious":20113,"robbed":20114,"colchester":20115,"cs":20116,"pmid":20117,"ao":20118,"verified":20119,"balancing":20120,"apostle":20121,"swayed":20122,"recognizable":20123,"oxfordshire":20124,"retention":20125,"nottinghamshire":20126,"contender":20127,"judd":20128,"invitational":20129,"shrimp":20130,"uhf":20131,"##icient":20132,"cleaner":20133,"longitudinal":20134,"tanker":20135,"##mur":20136,"acronym":20137,"broker":20138,"koppen":20139,"sundance":20140,"suppliers":20141,"##gil":20142,"4000":20143,"clipped":20144,"fuels":20145,"petite":20146,"##anne":20147,"landslide":20148,"helene":20149,"diversion":20150,"populous":20151,"landowners":20152,"auspices":20153,"melville":20154,"quantitative":20155,"##xes":20156,"ferries":20157,"nicky":20158,"##llus":20159,"doo":20160,"haunting":20161,"roche":20162,"carver":20163,"downed":20164,"unavailable":20165,"##pathy":20166,"approximation":20167,"hiroshima":20168,"##hue":20169,"garfield":20170,"valle":20171,"comparatively":20172,"keyboardist":20173,"traveler":20174,"##eit":20175,"congestion":20176,"calculating":20177,"subsidiaries":20178,"##bate":20179,"serb":20180,"modernization":20181,"fairies":20182,"deepened":20183,"ville":20184,"averages":20185,"##lore":20186,"inflammatory":20187,"tonga":20188,"##itch":20189,"co₂":20190,"squads":20191,"##hea":20192,"gigantic":20193,"serum":20194,"enjoyment":20195,"retailer":20196,"verona":20197,"35th":20198,"cis":20199,"##phobic":20200,"magna":20201,"technicians":20202,"##vati":20203,"arithmetic":20204,"##sport":20205,"levin":20206,"##dation":20207,"amtrak":20208,"chow":20209,"sienna":20210,"##eyer":20211,"backstage":20212,"entrepreneurship":20213,"##otic":20214,"learnt":20215,"tao":20216,"##udy":20217,"worcestershire":20218,"formulation":20219,"baggage":20220,"hesitant":20221,"bali":20222,"sabotage":20223,"##kari":20224,"barren":20225,"enhancing":20226,"murmur":20227,"pl":20228,"freshly":20229,"putnam":20230,"syntax":20231,"aces":20232,"medicines":20233,"resentment":20234,"bandwidth":20235,"##sier":20236,"grins":20237,"chili":20238,"guido":20239,"##sei":20240,"framing":20241,"implying":20242,"gareth":20243,"lissa":20244,"genevieve":20245,"pertaining":20246,"admissions":20247,"geo":20248,"thorpe":20249,"proliferation":20250,"sato":20251,"bela":20252,"analyzing":20253,"parting":20254,"##gor":20255,"awakened":20256,"##isman":20257,"huddled":20258,"secrecy":20259,"##kling":20260,"hush":20261,"gentry":20262,"540":20263,"dungeons":20264,"##ego":20265,"coasts":20266,"##utz":20267,"sacrificed":20268,"##chule":20269,"landowner":20270,"mutually":20271,"prevalence":20272,"programmer":20273,"adolescent":20274,"disrupted":20275,"seaside":20276,"gee":20277,"trusts":20278,"vamp":20279,"georgie":20280,"##nesian":20281,"##iol":20282,"schedules":20283,"sindh":20284,"##market":20285,"etched":20286,"hm":20287,"sparse":20288,"bey":20289,"beaux":20290,"scratching":20291,"gliding":20292,"unidentified":20293,"216":20294,"collaborating":20295,"gems":20296,"jesuits":20297,"oro":20298,"accumulation":20299,"shaping":20300,"mbe":20301,"anal":20302,"##xin":20303,"231":20304,"enthusiasts":20305,"newscast":20306,"##egan":20307,"janata":20308,"dewey":20309,"parkinson":20310,"179":20311,"ankara":20312,"biennial":20313,"towering":20314,"dd":20315,"inconsistent":20316,"950":20317,"##chet":20318,"thriving":20319,"terminate":20320,"cabins":20321,"furiously":20322,"eats":20323,"advocating":20324,"donkey":20325,"marley":20326,"muster":20327,"phyllis":20328,"leiden":20329,"##user":20330,"grassland":20331,"glittering":20332,"iucn":20333,"loneliness":20334,"217":20335,"memorandum":20336,"armenians":20337,"##ddle":20338,"popularized":20339,"rhodesia":20340,"60s":20341,"lame":20342,"##illon":20343,"sans":20344,"bikini":20345,"header":20346,"orbits":20347,"##xx":20348,"##finger":20349,"##ulator":20350,"sharif":20351,"spines":20352,"biotechnology":20353,"strolled":20354,"naughty":20355,"yates":20356,"##wire":20357,"fremantle":20358,"milo":20359,"##mour":20360,"abducted":20361,"removes":20362,"##atin":20363,"humming":20364,"wonderland":20365,"##chrome":20366,"##ester":20367,"hume":20368,"pivotal":20369,"##rates":20370,"armand":20371,"grams":20372,"believers":20373,"elector":20374,"rte":20375,"apron":20376,"bis":20377,"scraped":20378,"##yria":20379,"endorsement":20380,"initials":20381,"##llation":20382,"eps":20383,"dotted":20384,"hints":20385,"buzzing":20386,"emigration":20387,"nearer":20388,"##tom":20389,"indicators":20390,"##ulu":20391,"coarse":20392,"neutron":20393,"protectorate":20394,"##uze":20395,"directional":20396,"exploits":20397,"pains":20398,"loire":20399,"1830s":20400,"proponents":20401,"guggenheim":20402,"rabbits":20403,"ritchie":20404,"305":20405,"hectare":20406,"inputs":20407,"hutton":20408,"##raz":20409,"verify":20410,"##ako":20411,"boilers":20412,"longitude":20413,"##lev":20414,"skeletal":20415,"yer":20416,"emilia":20417,"citrus":20418,"compromised":20419,"##gau":20420,"pokemon":20421,"prescription":20422,"paragraph":20423,"eduard":20424,"cadillac":20425,"attire":20426,"categorized":20427,"kenyan":20428,"weddings":20429,"charley":20430,"##bourg":20431,"entertain":20432,"monmouth":20433,"##lles":20434,"nutrients":20435,"davey":20436,"mesh":20437,"incentive":20438,"practised":20439,"ecosystems":20440,"kemp":20441,"subdued":20442,"overheard":20443,"##rya":20444,"bodily":20445,"maxim":20446,"##nius":20447,"apprenticeship":20448,"ursula":20449,"##fight":20450,"lodged":20451,"rug":20452,"silesian":20453,"unconstitutional":20454,"patel":20455,"inspected":20456,"coyote":20457,"unbeaten":20458,"##hak":20459,"34th":20460,"disruption":20461,"convict":20462,"parcel":20463,"##cl":20464,"##nham":20465,"collier":20466,"implicated":20467,"mallory":20468,"##iac":20469,"##lab":20470,"susannah":20471,"winkler":20472,"##rber":20473,"shia":20474,"phelps":20475,"sediments":20476,"graphical":20477,"robotic":20478,"##sner":20479,"adulthood":20480,"mart":20481,"smoked":20482,"##isto":20483,"kathryn":20484,"clarified":20485,"##aran":20486,"divides":20487,"convictions":20488,"oppression":20489,"pausing":20490,"burying":20491,"##mt":20492,"federico":20493,"mathias":20494,"eileen":20495,"##tana":20496,"kite":20497,"hunched":20498,"##acies":20499,"189":20500,"##atz":20501,"disadvantage":20502,"liza":20503,"kinetic":20504,"greedy":20505,"paradox":20506,"yokohama":20507,"dowager":20508,"trunks":20509,"ventured":20510,"##gement":20511,"gupta":20512,"vilnius":20513,"olaf":20514,"##thest":20515,"crimean":20516,"hopper":20517,"##ej":20518,"progressively":20519,"arturo":20520,"mouthed":20521,"arrondissement":20522,"##fusion":20523,"rubin":20524,"simulcast":20525,"oceania":20526,"##orum":20527,"##stra":20528,"##rred":20529,"busiest":20530,"intensely":20531,"navigator":20532,"cary":20533,"##vine":20534,"##hini":20535,"##bies":20536,"fife":20537,"rowe":20538,"rowland":20539,"posing":20540,"insurgents":20541,"shafts":20542,"lawsuits":20543,"activate":20544,"conor":20545,"inward":20546,"culturally":20547,"garlic":20548,"265":20549,"##eering":20550,"eclectic":20551,"##hui":20552,"##kee":20553,"##nl":20554,"furrowed":20555,"vargas":20556,"meteorological":20557,"rendezvous":20558,"##aus":20559,"culinary":20560,"commencement":20561,"##dition":20562,"quota":20563,"##notes":20564,"mommy":20565,"salaries":20566,"overlapping":20567,"mule":20568,"##iology":20569,"##mology":20570,"sums":20571,"wentworth":20572,"##isk":20573,"##zione":20574,"mainline":20575,"subgroup":20576,"##illy":20577,"hack":20578,"plaintiff":20579,"verdi":20580,"bulb":20581,"differentiation":20582,"engagements":20583,"multinational":20584,"supplemented":20585,"bertrand":20586,"caller":20587,"regis":20588,"##naire":20589,"##sler":20590,"##arts":20591,"##imated":20592,"blossom":20593,"propagation":20594,"kilometer":20595,"viaduct":20596,"vineyards":20597,"##uate":20598,"beckett":20599,"optimization":20600,"golfer":20601,"songwriters":20602,"seminal":20603,"semitic":20604,"thud":20605,"volatile":20606,"evolving":20607,"ridley":20608,"##wley":20609,"trivial":20610,"distributions":20611,"scandinavia":20612,"jiang":20613,"##ject":20614,"wrestled":20615,"insistence":20616,"##dio":20617,"emphasizes":20618,"napkin":20619,"##ods":20620,"adjunct":20621,"rhyme":20622,"##ricted":20623,"##eti":20624,"hopeless":20625,"surrounds":20626,"tremble":20627,"32nd":20628,"smoky":20629,"##ntly":20630,"oils":20631,"medicinal":20632,"padded":20633,"steer":20634,"wilkes":20635,"219":20636,"255":20637,"concessions":20638,"hue":20639,"uniquely":20640,"blinded":20641,"landon":20642,"yahoo":20643,"##lane":20644,"hendrix":20645,"commemorating":20646,"dex":20647,"specify":20648,"chicks":20649,"##ggio":20650,"intercity":20651,"1400":20652,"morley":20653,"##torm":20654,"highlighting":20655,"##oting":20656,"pang":20657,"oblique":20658,"stalled":20659,"##liner":20660,"flirting":20661,"newborn":20662,"1769":20663,"bishopric":20664,"shaved":20665,"232":20666,"currie":20667,"##ush":20668,"dharma":20669,"spartan":20670,"##ooped":20671,"favorites":20672,"smug":20673,"novella":20674,"sirens":20675,"abusive":20676,"creations":20677,"espana":20678,"##lage":20679,"paradigm":20680,"semiconductor":20681,"sheen":20682,"##rdo":20683,"##yen":20684,"##zak":20685,"nrl":20686,"renew":20687,"##pose":20688,"##tur":20689,"adjutant":20690,"marches":20691,"norma":20692,"##enity":20693,"ineffective":20694,"weimar":20695,"grunt":20696,"##gat":20697,"lordship":20698,"plotting":20699,"expenditure":20700,"infringement":20701,"lbs":20702,"refrain":20703,"av":20704,"mimi":20705,"mistakenly":20706,"postmaster":20707,"1771":20708,"##bara":20709,"ras":20710,"motorsports":20711,"tito":20712,"199":20713,"subjective":20714,"##zza":20715,"bully":20716,"stew":20717,"##kaya":20718,"prescott":20719,"1a":20720,"##raphic":20721,"##zam":20722,"bids":20723,"styling":20724,"paranormal":20725,"reeve":20726,"sneaking":20727,"exploding":20728,"katz":20729,"akbar":20730,"migrant":20731,"syllables":20732,"indefinitely":20733,"##ogical":20734,"destroys":20735,"replaces":20736,"applause":20737,"##phine":20738,"pest":20739,"##fide":20740,"218":20741,"articulated":20742,"bertie":20743,"##thing":20744,"##cars":20745,"##ptic":20746,"courtroom":20747,"crowley":20748,"aesthetics":20749,"cummings":20750,"tehsil":20751,"hormones":20752,"titanic":20753,"dangerously":20754,"##ibe":20755,"stadion":20756,"jaenelle":20757,"auguste":20758,"ciudad":20759,"##chu":20760,"mysore":20761,"partisans":20762,"##sio":20763,"lucan":20764,"philipp":20765,"##aly":20766,"debating":20767,"henley":20768,"interiors":20769,"##rano":20770,"##tious":20771,"homecoming":20772,"beyonce":20773,"usher":20774,"henrietta":20775,"prepares":20776,"weeds":20777,"##oman":20778,"ely":20779,"plucked":20780,"##pire":20781,"##dable":20782,"luxurious":20783,"##aq":20784,"artifact":20785,"password":20786,"pasture":20787,"juno":20788,"maddy":20789,"minsk":20790,"##dder":20791,"##ologies":20792,"##rone":20793,"assessments":20794,"martian":20795,"royalist":20796,"1765":20797,"examines":20798,"##mani":20799,"##rge":20800,"nino":20801,"223":20802,"parry":20803,"scooped":20804,"relativity":20805,"##eli":20806,"##uting":20807,"##cao":20808,"congregational":20809,"noisy":20810,"traverse":20811,"##agawa":20812,"strikeouts":20813,"nickelodeon":20814,"obituary":20815,"transylvania":20816,"binds":20817,"depictions":20818,"polk":20819,"trolley":20820,"##yed":20821,"##lard":20822,"breeders":20823,"##under":20824,"dryly":20825,"hokkaido":20826,"1762":20827,"strengths":20828,"stacks":20829,"bonaparte":20830,"connectivity":20831,"neared":20832,"prostitutes":20833,"stamped":20834,"anaheim":20835,"gutierrez":20836,"sinai":20837,"##zzling":20838,"bram":20839,"fresno":20840,"madhya":20841,"##86":20842,"proton":20843,"##lena":20844,"##llum":20845,"##phon":20846,"reelected":20847,"wanda":20848,"##anus":20849,"##lb":20850,"ample":20851,"distinguishing":20852,"##yler":20853,"grasping":20854,"sermons":20855,"tomato":20856,"bland":20857,"stimulation":20858,"avenues":20859,"##eux":20860,"spreads":20861,"scarlett":20862,"fern":20863,"pentagon":20864,"assert":20865,"baird":20866,"chesapeake":20867,"ir":20868,"calmed":20869,"distortion":20870,"fatalities":20871,"##olis":20872,"correctional":20873,"pricing":20874,"##astic":20875,"##gina":20876,"prom":20877,"dammit":20878,"ying":20879,"collaborate":20880,"##chia":20881,"welterweight":20882,"33rd":20883,"pointer":20884,"substitution":20885,"bonded":20886,"umpire":20887,"communicating":20888,"multitude":20889,"paddle":20890,"##obe":20891,"federally":20892,"intimacy":20893,"##insky":20894,"betray":20895,"ssr":20896,"##lett":20897,"##lean":20898,"##lves":20899,"##therapy":20900,"airbus":20901,"##tery":20902,"functioned":20903,"ud":20904,"bearer":20905,"biomedical":20906,"netflix":20907,"##hire":20908,"##nca":20909,"condom":20910,"brink":20911,"ik":20912,"##nical":20913,"macy":20914,"##bet":20915,"flap":20916,"gma":20917,"experimented":20918,"jelly":20919,"lavender":20920,"##icles":20921,"##ulia":20922,"munro":20923,"##mian":20924,"##tial":20925,"rye":20926,"##rle":20927,"60th":20928,"gigs":20929,"hottest":20930,"rotated":20931,"predictions":20932,"fuji":20933,"bu":20934,"##erence":20935,"##omi":20936,"barangay":20937,"##fulness":20938,"##sas":20939,"clocks":20940,"##rwood":20941,"##liness":20942,"cereal":20943,"roe":20944,"wight":20945,"decker":20946,"uttered":20947,"babu":20948,"onion":20949,"xml":20950,"forcibly":20951,"##df":20952,"petra":20953,"sarcasm":20954,"hartley":20955,"peeled":20956,"storytelling":20957,"##42":20958,"##xley":20959,"##ysis":20960,"##ffa":20961,"fibre":20962,"kiel":20963,"auditor":20964,"fig":20965,"harald":20966,"greenville":20967,"##berries":20968,"geographically":20969,"nell":20970,"quartz":20971,"##athic":20972,"cemeteries":20973,"##lr":20974,"crossings":20975,"nah":20976,"holloway":20977,"reptiles":20978,"chun":20979,"sichuan":20980,"snowy":20981,"660":20982,"corrections":20983,"##ivo":20984,"zheng":20985,"ambassadors":20986,"blacksmith":20987,"fielded":20988,"fluids":20989,"hardcover":20990,"turnover":20991,"medications":20992,"melvin":20993,"academies":20994,"##erton":20995,"ro":20996,"roach":20997,"absorbing":20998,"spaniards":20999,"colton":21000,"##founded":21001,"outsider":21002,"espionage":21003,"kelsey":21004,"245":21005,"edible":21006,"##ulf":21007,"dora":21008,"establishes":21009,"##sham":21010,"##tries":21011,"contracting":21012,"##tania":21013,"cinematic":21014,"costello":21015,"nesting":21016,"##uron":21017,"connolly":21018,"duff":21019,"##nology":21020,"mma":21021,"##mata":21022,"fergus":21023,"sexes":21024,"gi":21025,"optics":21026,"spectator":21027,"woodstock":21028,"banning":21029,"##hee":21030,"##fle":21031,"differentiate":21032,"outfielder":21033,"refinery":21034,"226":21035,"312":21036,"gerhard":21037,"horde":21038,"lair":21039,"drastically":21040,"##udi":21041,"landfall":21042,"##cheng":21043,"motorsport":21044,"odi":21045,"##achi":21046,"predominant":21047,"quay":21048,"skins":21049,"##ental":21050,"edna":21051,"harshly":21052,"complementary":21053,"murdering":21054,"##aves":21055,"wreckage":21056,"##90":21057,"ono":21058,"outstretched":21059,"lennox":21060,"munitions":21061,"galen":21062,"reconcile":21063,"470":21064,"scalp":21065,"bicycles":21066,"gillespie":21067,"questionable":21068,"rosenberg":21069,"guillermo":21070,"hostel":21071,"jarvis":21072,"kabul":21073,"volvo":21074,"opium":21075,"yd":21076,"##twined":21077,"abuses":21078,"decca":21079,"outpost":21080,"##cino":21081,"sensible":21082,"neutrality":21083,"##64":21084,"ponce":21085,"anchorage":21086,"atkins":21087,"turrets":21088,"inadvertently":21089,"disagree":21090,"libre":21091,"vodka":21092,"reassuring":21093,"weighs":21094,"##yal":21095,"glide":21096,"jumper":21097,"ceilings":21098,"repertory":21099,"outs":21100,"stain":21101,"##bial":21102,"envy":21103,"##ucible":21104,"smashing":21105,"heightened":21106,"policing":21107,"hyun":21108,"mixes":21109,"lai":21110,"prima":21111,"##ples":21112,"celeste":21113,"##bina":21114,"lucrative":21115,"intervened":21116,"kc":21117,"manually":21118,"##rned":21119,"stature":21120,"staffed":21121,"bun":21122,"bastards":21123,"nairobi":21124,"priced":21125,"##auer":21126,"thatcher":21127,"##kia":21128,"tripped":21129,"comune":21130,"##ogan":21131,"##pled":21132,"brasil":21133,"incentives":21134,"emanuel":21135,"hereford":21136,"musica":21137,"##kim":21138,"benedictine":21139,"biennale":21140,"##lani":21141,"eureka":21142,"gardiner":21143,"rb":21144,"knocks":21145,"sha":21146,"##ael":21147,"##elled":21148,"##onate":21149,"efficacy":21150,"ventura":21151,"masonic":21152,"sanford":21153,"maize":21154,"leverage":21155,"##feit":21156,"capacities":21157,"santana":21158,"##aur":21159,"novelty":21160,"vanilla":21161,"##cter":21162,"##tour":21163,"benin":21164,"##oir":21165,"##rain":21166,"neptune":21167,"drafting":21168,"tallinn":21169,"##cable":21170,"humiliation":21171,"##boarding":21172,"schleswig":21173,"fabian":21174,"bernardo":21175,"liturgy":21176,"spectacle":21177,"sweeney":21178,"pont":21179,"routledge":21180,"##tment":21181,"cosmos":21182,"ut":21183,"hilt":21184,"sleek":21185,"universally":21186,"##eville":21187,"##gawa":21188,"typed":21189,"##dry":21190,"favors":21191,"allegheny":21192,"glaciers":21193,"##rly":21194,"recalling":21195,"aziz":21196,"##log":21197,"parasite":21198,"requiem":21199,"auf":21200,"##berto":21201,"##llin":21202,"illumination":21203,"##breaker":21204,"##issa":21205,"festivities":21206,"bows":21207,"govern":21208,"vibe":21209,"vp":21210,"333":21211,"sprawled":21212,"larson":21213,"pilgrim":21214,"bwf":21215,"leaping":21216,"##rts":21217,"##ssel":21218,"alexei":21219,"greyhound":21220,"hoarse":21221,"##dler":21222,"##oration":21223,"seneca":21224,"##cule":21225,"gaping":21226,"##ulously":21227,"##pura":21228,"cinnamon":21229,"##gens":21230,"##rricular":21231,"craven":21232,"fantasies":21233,"houghton":21234,"engined":21235,"reigned":21236,"dictator":21237,"supervising":21238,"##oris":21239,"bogota":21240,"commentaries":21241,"unnatural":21242,"fingernails":21243,"spirituality":21244,"tighten":21245,"##tm":21246,"canadiens":21247,"protesting":21248,"intentional":21249,"cheers":21250,"sparta":21251,"##ytic":21252,"##iere":21253,"##zine":21254,"widen":21255,"belgarath":21256,"controllers":21257,"dodd":21258,"iaaf":21259,"navarre":21260,"##ication":21261,"defect":21262,"squire":21263,"steiner":21264,"whisky":21265,"##mins":21266,"560":21267,"inevitably":21268,"tome":21269,"##gold":21270,"chew":21271,"##uid":21272,"##lid":21273,"elastic":21274,"##aby":21275,"streaked":21276,"alliances":21277,"jailed":21278,"regal":21279,"##ined":21280,"##phy":21281,"czechoslovak":21282,"narration":21283,"absently":21284,"##uld":21285,"bluegrass":21286,"guangdong":21287,"quran":21288,"criticizing":21289,"hose":21290,"hari":21291,"##liest":21292,"##owa":21293,"skier":21294,"streaks":21295,"deploy":21296,"##lom":21297,"raft":21298,"bose":21299,"dialed":21300,"huff":21301,"##eira":21302,"haifa":21303,"simplest":21304,"bursting":21305,"endings":21306,"ib":21307,"sultanate":21308,"##titled":21309,"franks":21310,"whitman":21311,"ensures":21312,"sven":21313,"##ggs":21314,"collaborators":21315,"forster":21316,"organising":21317,"ui":21318,"banished":21319,"napier":21320,"injustice":21321,"teller":21322,"layered":21323,"thump":21324,"##otti":21325,"roc":21326,"battleships":21327,"evidenced":21328,"fugitive":21329,"sadie":21330,"robotics":21331,"##roud":21332,"equatorial":21333,"geologist":21334,"##iza":21335,"yielding":21336,"##bron":21337,"##sr":21338,"internationale":21339,"mecca":21340,"##diment":21341,"sbs":21342,"skyline":21343,"toad":21344,"uploaded":21345,"reflective":21346,"undrafted":21347,"lal":21348,"leafs":21349,"bayern":21350,"##dai":21351,"lakshmi":21352,"shortlisted":21353,"##stick":21354,"##wicz":21355,"camouflage":21356,"donate":21357,"af":21358,"christi":21359,"lau":21360,"##acio":21361,"disclosed":21362,"nemesis":21363,"1761":21364,"assemble":21365,"straining":21366,"northamptonshire":21367,"tal":21368,"##asi":21369,"bernardino":21370,"premature":21371,"heidi":21372,"42nd":21373,"coefficients":21374,"galactic":21375,"reproduce":21376,"buzzed":21377,"sensations":21378,"zionist":21379,"monsieur":21380,"myrtle":21381,"##eme":21382,"archery":21383,"strangled":21384,"musically":21385,"viewpoint":21386,"antiquities":21387,"bei":21388,"trailers":21389,"seahawks":21390,"cured":21391,"pee":21392,"preferring":21393,"tasmanian":21394,"lange":21395,"sul":21396,"##mail":21397,"##working":21398,"colder":21399,"overland":21400,"lucivar":21401,"massey":21402,"gatherings":21403,"haitian":21404,"##smith":21405,"disapproval":21406,"flaws":21407,"##cco":21408,"##enbach":21409,"1766":21410,"npr":21411,"##icular":21412,"boroughs":21413,"creole":21414,"forums":21415,"techno":21416,"1755":21417,"dent":21418,"abdominal":21419,"streetcar":21420,"##eson":21421,"##stream":21422,"procurement":21423,"gemini":21424,"predictable":21425,"##tya":21426,"acheron":21427,"christoph":21428,"feeder":21429,"fronts":21430,"vendor":21431,"bernhard":21432,"jammu":21433,"tumors":21434,"slang":21435,"##uber":21436,"goaltender":21437,"twists":21438,"curving":21439,"manson":21440,"vuelta":21441,"mer":21442,"peanut":21443,"confessions":21444,"pouch":21445,"unpredictable":21446,"allowance":21447,"theodor":21448,"vascular":21449,"##factory":21450,"bala":21451,"authenticity":21452,"metabolic":21453,"coughing":21454,"nanjing":21455,"##cea":21456,"pembroke":21457,"##bard":21458,"splendid":21459,"36th":21460,"ff":21461,"hourly":21462,"##ahu":21463,"elmer":21464,"handel":21465,"##ivate":21466,"awarding":21467,"thrusting":21468,"dl":21469,"experimentation":21470,"##hesion":21471,"##46":21472,"caressed":21473,"entertained":21474,"steak":21475,"##rangle":21476,"biologist":21477,"orphans":21478,"baroness":21479,"oyster":21480,"stepfather":21481,"##dridge":21482,"mirage":21483,"reefs":21484,"speeding":21485,"##31":21486,"barons":21487,"1764":21488,"227":21489,"inhabit":21490,"preached":21491,"repealed":21492,"##tral":21493,"honoring":21494,"boogie":21495,"captives":21496,"administer":21497,"johanna":21498,"##imate":21499,"gel":21500,"suspiciously":21501,"1767":21502,"sobs":21503,"##dington":21504,"backbone":21505,"hayward":21506,"garry":21507,"##folding":21508,"##nesia":21509,"maxi":21510,"##oof":21511,"##ppe":21512,"ellison":21513,"galileo":21514,"##stand":21515,"crimea":21516,"frenzy":21517,"amour":21518,"bumper":21519,"matrices":21520,"natalia":21521,"baking":21522,"garth":21523,"palestinians":21524,"##grove":21525,"smack":21526,"conveyed":21527,"ensembles":21528,"gardening":21529,"##manship":21530,"##rup":21531,"##stituting":21532,"1640":21533,"harvesting":21534,"topography":21535,"jing":21536,"shifters":21537,"dormitory":21538,"##carriage":21539,"##lston":21540,"ist":21541,"skulls":21542,"##stadt":21543,"dolores":21544,"jewellery":21545,"sarawak":21546,"##wai":21547,"##zier":21548,"fences":21549,"christy":21550,"confinement":21551,"tumbling":21552,"credibility":21553,"fir":21554,"stench":21555,"##bria":21556,"##plication":21557,"##nged":21558,"##sam":21559,"virtues":21560,"##belt":21561,"marjorie":21562,"pba":21563,"##eem":21564,"##made":21565,"celebrates":21566,"schooner":21567,"agitated":21568,"barley":21569,"fulfilling":21570,"anthropologist":21571,"##pro":21572,"restrict":21573,"novi":21574,"regulating":21575,"##nent":21576,"padres":21577,"##rani":21578,"##hesive":21579,"loyola":21580,"tabitha":21581,"milky":21582,"olson":21583,"proprietor":21584,"crambidae":21585,"guarantees":21586,"intercollegiate":21587,"ljubljana":21588,"hilda":21589,"##sko":21590,"ignorant":21591,"hooded":21592,"##lts":21593,"sardinia":21594,"##lidae":21595,"##vation":21596,"frontman":21597,"privileged":21598,"witchcraft":21599,"##gp":21600,"jammed":21601,"laude":21602,"poking":21603,"##than":21604,"bracket":21605,"amazement":21606,"yunnan":21607,"##erus":21608,"maharaja":21609,"linnaeus":21610,"264":21611,"commissioning":21612,"milano":21613,"peacefully":21614,"##logies":21615,"akira":21616,"rani":21617,"regulator":21618,"##36":21619,"grasses":21620,"##rance":21621,"luzon":21622,"crows":21623,"compiler":21624,"gretchen":21625,"seaman":21626,"edouard":21627,"tab":21628,"buccaneers":21629,"ellington":21630,"hamlets":21631,"whig":21632,"socialists":21633,"##anto":21634,"directorial":21635,"easton":21636,"mythological":21637,"##kr":21638,"##vary":21639,"rhineland":21640,"semantic":21641,"taut":21642,"dune":21643,"inventions":21644,"succeeds":21645,"##iter":21646,"replication":21647,"branched":21648,"##pired":21649,"jul":21650,"prosecuted":21651,"kangaroo":21652,"penetrated":21653,"##avian":21654,"middlesbrough":21655,"doses":21656,"bleak":21657,"madam":21658,"predatory":21659,"relentless":21660,"##vili":21661,"reluctance":21662,"##vir":21663,"hailey":21664,"crore":21665,"silvery":21666,"1759":21667,"monstrous":21668,"swimmers":21669,"transmissions":21670,"hawthorn":21671,"informing":21672,"##eral":21673,"toilets":21674,"caracas":21675,"crouch":21676,"kb":21677,"##sett":21678,"295":21679,"cartel":21680,"hadley":21681,"##aling":21682,"alexia":21683,"yvonne":21684,"##biology":21685,"cinderella":21686,"eton":21687,"superb":21688,"blizzard":21689,"stabbing":21690,"industrialist":21691,"maximus":21692,"##gm":21693,"##orus":21694,"groves":21695,"maud":21696,"clade":21697,"oversized":21698,"comedic":21699,"##bella":21700,"rosen":21701,"nomadic":21702,"fulham":21703,"montane":21704,"beverages":21705,"galaxies":21706,"redundant":21707,"swarm":21708,"##rot":21709,"##folia":21710,"##llis":21711,"buckinghamshire":21712,"fen":21713,"bearings":21714,"bahadur":21715,"##rom":21716,"gilles":21717,"phased":21718,"dynamite":21719,"faber":21720,"benoit":21721,"vip":21722,"##ount":21723,"##wd":21724,"booking":21725,"fractured":21726,"tailored":21727,"anya":21728,"spices":21729,"westwood":21730,"cairns":21731,"auditions":21732,"inflammation":21733,"steamed":21734,"##rocity":21735,"##acion":21736,"##urne":21737,"skyla":21738,"thereof":21739,"watford":21740,"torment":21741,"archdeacon":21742,"transforms":21743,"lulu":21744,"demeanor":21745,"fucked":21746,"serge":21747,"##sor":21748,"mckenna":21749,"minas":21750,"entertainer":21751,"##icide":21752,"caress":21753,"originate":21754,"residue":21755,"##sty":21756,"1740":21757,"##ilised":21758,"##org":21759,"beech":21760,"##wana":21761,"subsidies":21762,"##ghton":21763,"emptied":21764,"gladstone":21765,"ru":21766,"firefighters":21767,"voodoo":21768,"##rcle":21769,"het":21770,"nightingale":21771,"tamara":21772,"edmond":21773,"ingredient":21774,"weaknesses":21775,"silhouette":21776,"285":21777,"compatibility":21778,"withdrawing":21779,"hampson":21780,"##mona":21781,"anguish":21782,"giggling":21783,"##mber":21784,"bookstore":21785,"##jiang":21786,"southernmost":21787,"tilting":21788,"##vance":21789,"bai":21790,"economical":21791,"rf":21792,"briefcase":21793,"dreadful":21794,"hinted":21795,"projections":21796,"shattering":21797,"totaling":21798,"##rogate":21799,"analogue":21800,"indicted":21801,"periodical":21802,"fullback":21803,"##dman":21804,"haynes":21805,"##tenberg":21806,"##ffs":21807,"##ishment":21808,"1745":21809,"thirst":21810,"stumble":21811,"penang":21812,"vigorous":21813,"##ddling":21814,"##kor":21815,"##lium":21816,"octave":21817,"##ove":21818,"##enstein":21819,"##inen":21820,"##ones":21821,"siberian":21822,"##uti":21823,"cbn":21824,"repeal":21825,"swaying":21826,"##vington":21827,"khalid":21828,"tanaka":21829,"unicorn":21830,"otago":21831,"plastered":21832,"lobe":21833,"riddle":21834,"##rella":21835,"perch":21836,"##ishing":21837,"croydon":21838,"filtered":21839,"graeme":21840,"tripoli":21841,"##ossa":21842,"crocodile":21843,"##chers":21844,"sufi":21845,"mined":21846,"##tung":21847,"inferno":21848,"lsu":21849,"##phi":21850,"swelled":21851,"utilizes":21852,"£2":21853,"cale":21854,"periodicals":21855,"styx":21856,"hike":21857,"informally":21858,"coop":21859,"lund":21860,"##tidae":21861,"ala":21862,"hen":21863,"qui":21864,"transformations":21865,"disposed":21866,"sheath":21867,"chickens":21868,"##cade":21869,"fitzroy":21870,"sas":21871,"silesia":21872,"unacceptable":21873,"odisha":21874,"1650":21875,"sabrina":21876,"pe":21877,"spokane":21878,"ratios":21879,"athena":21880,"massage":21881,"shen":21882,"dilemma":21883,"##drum":21884,"##riz":21885,"##hul":21886,"corona":21887,"doubtful":21888,"niall":21889,"##pha":21890,"##bino":21891,"fines":21892,"cite":21893,"acknowledging":21894,"bangor":21895,"ballard":21896,"bathurst":21897,"##resh":21898,"huron":21899,"mustered":21900,"alzheimer":21901,"garments":21902,"kinase":21903,"tyre":21904,"warship":21905,"##cp":21906,"flashback":21907,"pulmonary":21908,"braun":21909,"cheat":21910,"kamal":21911,"cyclists":21912,"constructions":21913,"grenades":21914,"ndp":21915,"traveller":21916,"excuses":21917,"stomped":21918,"signalling":21919,"trimmed":21920,"futsal":21921,"mosques":21922,"relevance":21923,"##wine":21924,"wta":21925,"##23":21926,"##vah":21927,"##lter":21928,"hoc":21929,"##riding":21930,"optimistic":21931,"##´s":21932,"deco":21933,"sim":21934,"interacting":21935,"rejecting":21936,"moniker":21937,"waterways":21938,"##ieri":21939,"##oku":21940,"mayors":21941,"gdansk":21942,"outnumbered":21943,"pearls":21944,"##ended":21945,"##hampton":21946,"fairs":21947,"totals":21948,"dominating":21949,"262":21950,"notions":21951,"stairway":21952,"compiling":21953,"pursed":21954,"commodities":21955,"grease":21956,"yeast":21957,"##jong":21958,"carthage":21959,"griffiths":21960,"residual":21961,"amc":21962,"contraction":21963,"laird":21964,"sapphire":21965,"##marine":21966,"##ivated":21967,"amalgamation":21968,"dissolve":21969,"inclination":21970,"lyle":21971,"packaged":21972,"altitudes":21973,"suez":21974,"canons":21975,"graded":21976,"lurched":21977,"narrowing":21978,"boasts":21979,"guise":21980,"wed":21981,"enrico":21982,"##ovsky":21983,"rower":21984,"scarred":21985,"bree":21986,"cub":21987,"iberian":21988,"protagonists":21989,"bargaining":21990,"proposing":21991,"trainers":21992,"voyages":21993,"vans":21994,"fishes":21995,"##aea":21996,"##ivist":21997,"##verance":21998,"encryption":21999,"artworks":22000,"kazan":22001,"sabre":22002,"cleopatra":22003,"hepburn":22004,"rotting":22005,"supremacy":22006,"mecklenburg":22007,"##brate":22008,"burrows":22009,"hazards":22010,"outgoing":22011,"flair":22012,"organizes":22013,"##ctions":22014,"scorpion":22015,"##usions":22016,"boo":22017,"234":22018,"chevalier":22019,"dunedin":22020,"slapping":22021,"##34":22022,"ineligible":22023,"pensions":22024,"##38":22025,"##omic":22026,"manufactures":22027,"emails":22028,"bismarck":22029,"238":22030,"weakening":22031,"blackish":22032,"ding":22033,"mcgee":22034,"quo":22035,"##rling":22036,"northernmost":22037,"xx":22038,"manpower":22039,"greed":22040,"sampson":22041,"clicking":22042,"##ange":22043,"##horpe":22044,"##inations":22045,"##roving":22046,"torre":22047,"##eptive":22048,"##moral":22049,"symbolism":22050,"38th":22051,"asshole":22052,"meritorious":22053,"outfits":22054,"splashed":22055,"biographies":22056,"sprung":22057,"astros":22058,"##tale":22059,"302":22060,"737":22061,"filly":22062,"raoul":22063,"nw":22064,"tokugawa":22065,"linden":22066,"clubhouse":22067,"##apa":22068,"tracts":22069,"romano":22070,"##pio":22071,"putin":22072,"tags":22073,"##note":22074,"chained":22075,"dickson":22076,"gunshot":22077,"moe":22078,"gunn":22079,"rashid":22080,"##tails":22081,"zipper":22082,"##bas":22083,"##nea":22084,"contrasted":22085,"##ply":22086,"##udes":22087,"plum":22088,"pharaoh":22089,"##pile":22090,"aw":22091,"comedies":22092,"ingrid":22093,"sandwiches":22094,"subdivisions":22095,"1100":22096,"mariana":22097,"nokia":22098,"kamen":22099,"hz":22100,"delaney":22101,"veto":22102,"herring":22103,"##words":22104,"possessive":22105,"outlines":22106,"##roup":22107,"siemens":22108,"stairwell":22109,"rc":22110,"gallantry":22111,"messiah":22112,"palais":22113,"yells":22114,"233":22115,"zeppelin":22116,"##dm":22117,"bolivar":22118,"##cede":22119,"smackdown":22120,"mckinley":22121,"##mora":22122,"##yt":22123,"muted":22124,"geologic":22125,"finely":22126,"unitary":22127,"avatar":22128,"hamas":22129,"maynard":22130,"rees":22131,"bog":22132,"contrasting":22133,"##rut":22134,"liv":22135,"chico":22136,"disposition":22137,"pixel":22138,"##erate":22139,"becca":22140,"dmitry":22141,"yeshiva":22142,"narratives":22143,"##lva":22144,"##ulton":22145,"mercenary":22146,"sharpe":22147,"tempered":22148,"navigate":22149,"stealth":22150,"amassed":22151,"keynes":22152,"##lini":22153,"untouched":22154,"##rrie":22155,"havoc":22156,"lithium":22157,"##fighting":22158,"abyss":22159,"graf":22160,"southward":22161,"wolverine":22162,"balloons":22163,"implements":22164,"ngos":22165,"transitions":22166,"##icum":22167,"ambushed":22168,"concacaf":22169,"dormant":22170,"economists":22171,"##dim":22172,"costing":22173,"csi":22174,"rana":22175,"universite":22176,"boulders":22177,"verity":22178,"##llon":22179,"collin":22180,"mellon":22181,"misses":22182,"cypress":22183,"fluorescent":22184,"lifeless":22185,"spence":22186,"##ulla":22187,"crewe":22188,"shepard":22189,"pak":22190,"revelations":22191,"##م":22192,"jolly":22193,"gibbons":22194,"paw":22195,"##dro":22196,"##quel":22197,"freeing":22198,"##test":22199,"shack":22200,"fries":22201,"palatine":22202,"##51":22203,"##hiko":22204,"accompaniment":22205,"cruising":22206,"recycled":22207,"##aver":22208,"erwin":22209,"sorting":22210,"synthesizers":22211,"dyke":22212,"realities":22213,"sg":22214,"strides":22215,"enslaved":22216,"wetland":22217,"##ghan":22218,"competence":22219,"gunpowder":22220,"grassy":22221,"maroon":22222,"reactors":22223,"objection":22224,"##oms":22225,"carlson":22226,"gearbox":22227,"macintosh":22228,"radios":22229,"shelton":22230,"##sho":22231,"clergyman":22232,"prakash":22233,"254":22234,"mongols":22235,"trophies":22236,"oricon":22237,"228":22238,"stimuli":22239,"twenty20":22240,"cantonese":22241,"cortes":22242,"mirrored":22243,"##saurus":22244,"bhp":22245,"cristina":22246,"melancholy":22247,"##lating":22248,"enjoyable":22249,"nuevo":22250,"##wny":22251,"downfall":22252,"schumacher":22253,"##ind":22254,"banging":22255,"lausanne":22256,"rumbled":22257,"paramilitary":22258,"reflex":22259,"ax":22260,"amplitude":22261,"migratory":22262,"##gall":22263,"##ups":22264,"midi":22265,"barnard":22266,"lastly":22267,"sherry":22268,"##hp":22269,"##nall":22270,"keystone":22271,"##kra":22272,"carleton":22273,"slippery":22274,"##53":22275,"coloring":22276,"foe":22277,"socket":22278,"otter":22279,"##rgos":22280,"mats":22281,"##tose":22282,"consultants":22283,"bafta":22284,"bison":22285,"topping":22286,"##km":22287,"490":22288,"primal":22289,"abandonment":22290,"transplant":22291,"atoll":22292,"hideous":22293,"mort":22294,"pained":22295,"reproduced":22296,"tae":22297,"howling":22298,"##turn":22299,"unlawful":22300,"billionaire":22301,"hotter":22302,"poised":22303,"lansing":22304,"##chang":22305,"dinamo":22306,"retro":22307,"messing":22308,"nfc":22309,"domesday":22310,"##mina":22311,"blitz":22312,"timed":22313,"##athing":22314,"##kley":22315,"ascending":22316,"gesturing":22317,"##izations":22318,"signaled":22319,"tis":22320,"chinatown":22321,"mermaid":22322,"savanna":22323,"jameson":22324,"##aint":22325,"catalina":22326,"##pet":22327,"##hers":22328,"cochrane":22329,"cy":22330,"chatting":22331,"##kus":22332,"alerted":22333,"computation":22334,"mused":22335,"noelle":22336,"majestic":22337,"mohawk":22338,"campo":22339,"octagonal":22340,"##sant":22341,"##hend":22342,"241":22343,"aspiring":22344,"##mart":22345,"comprehend":22346,"iona":22347,"paralyzed":22348,"shimmering":22349,"swindon":22350,"rhone":22351,"##eley":22352,"reputed":22353,"configurations":22354,"pitchfork":22355,"agitation":22356,"francais":22357,"gillian":22358,"lipstick":22359,"##ilo":22360,"outsiders":22361,"pontifical":22362,"resisting":22363,"bitterness":22364,"sewer":22365,"rockies":22366,"##edd":22367,"##ucher":22368,"misleading":22369,"1756":22370,"exiting":22371,"galloway":22372,"##nging":22373,"risked":22374,"##heart":22375,"246":22376,"commemoration":22377,"schultz":22378,"##rka":22379,"integrating":22380,"##rsa":22381,"poses":22382,"shrieked":22383,"##weiler":22384,"guineas":22385,"gladys":22386,"jerking":22387,"owls":22388,"goldsmith":22389,"nightly":22390,"penetrating":22391,"##unced":22392,"lia":22393,"##33":22394,"ignited":22395,"betsy":22396,"##aring":22397,"##thorpe":22398,"follower":22399,"vigorously":22400,"##rave":22401,"coded":22402,"kiran":22403,"knit":22404,"zoology":22405,"tbilisi":22406,"##28":22407,"##bered":22408,"repository":22409,"govt":22410,"deciduous":22411,"dino":22412,"growling":22413,"##bba":22414,"enhancement":22415,"unleashed":22416,"chanting":22417,"pussy":22418,"biochemistry":22419,"##eric":22420,"kettle":22421,"repression":22422,"toxicity":22423,"nrhp":22424,"##arth":22425,"##kko":22426,"##bush":22427,"ernesto":22428,"commended":22429,"outspoken":22430,"242":22431,"mca":22432,"parchment":22433,"sms":22434,"kristen":22435,"##aton":22436,"bisexual":22437,"raked":22438,"glamour":22439,"navajo":22440,"a2":22441,"conditioned":22442,"showcased":22443,"##hma":22444,"spacious":22445,"youthful":22446,"##esa":22447,"usl":22448,"appliances":22449,"junta":22450,"brest":22451,"layne":22452,"conglomerate":22453,"enchanted":22454,"chao":22455,"loosened":22456,"picasso":22457,"circulating":22458,"inspect":22459,"montevideo":22460,"##centric":22461,"##kti":22462,"piazza":22463,"spurred":22464,"##aith":22465,"bari":22466,"freedoms":22467,"poultry":22468,"stamford":22469,"lieu":22470,"##ect":22471,"indigo":22472,"sarcastic":22473,"bahia":22474,"stump":22475,"attach":22476,"dvds":22477,"frankenstein":22478,"lille":22479,"approx":22480,"scriptures":22481,"pollen":22482,"##script":22483,"nmi":22484,"overseen":22485,"##ivism":22486,"tides":22487,"proponent":22488,"newmarket":22489,"inherit":22490,"milling":22491,"##erland":22492,"centralized":22493,"##rou":22494,"distributors":22495,"credentials":22496,"drawers":22497,"abbreviation":22498,"##lco":22499,"##xon":22500,"downing":22501,"uncomfortably":22502,"ripe":22503,"##oes":22504,"erase":22505,"franchises":22506,"##ever":22507,"populace":22508,"##bery":22509,"##khar":22510,"decomposition":22511,"pleas":22512,"##tet":22513,"daryl":22514,"sabah":22515,"##stle":22516,"##wide":22517,"fearless":22518,"genie":22519,"lesions":22520,"annette":22521,"##ogist":22522,"oboe":22523,"appendix":22524,"nair":22525,"dripped":22526,"petitioned":22527,"maclean":22528,"mosquito":22529,"parrot":22530,"rpg":22531,"hampered":22532,"1648":22533,"operatic":22534,"reservoirs":22535,"##tham":22536,"irrelevant":22537,"jolt":22538,"summarized":22539,"##fp":22540,"medallion":22541,"##taff":22542,"##−":22543,"clawed":22544,"harlow":22545,"narrower":22546,"goddard":22547,"marcia":22548,"bodied":22549,"fremont":22550,"suarez":22551,"altering":22552,"tempest":22553,"mussolini":22554,"porn":22555,"##isms":22556,"sweetly":22557,"oversees":22558,"walkers":22559,"solitude":22560,"grimly":22561,"shrines":22562,"hk":22563,"ich":22564,"supervisors":22565,"hostess":22566,"dietrich":22567,"legitimacy":22568,"brushes":22569,"expressive":22570,"##yp":22571,"dissipated":22572,"##rse":22573,"localized":22574,"systemic":22575,"##nikov":22576,"gettysburg":22577,"##js":22578,"##uaries":22579,"dialogues":22580,"muttering":22581,"251":22582,"housekeeper":22583,"sicilian":22584,"discouraged":22585,"##frey":22586,"beamed":22587,"kaladin":22588,"halftime":22589,"kidnap":22590,"##amo":22591,"##llet":22592,"1754":22593,"synonymous":22594,"depleted":22595,"instituto":22596,"insulin":22597,"reprised":22598,"##opsis":22599,"clashed":22600,"##ctric":22601,"interrupting":22602,"radcliffe":22603,"insisting":22604,"medici":22605,"1715":22606,"ejected":22607,"playfully":22608,"turbulent":22609,"##47":22610,"starvation":22611,"##rini":22612,"shipment":22613,"rebellious":22614,"petersen":22615,"verification":22616,"merits":22617,"##rified":22618,"cakes":22619,"##charged":22620,"1757":22621,"milford":22622,"shortages":22623,"spying":22624,"fidelity":22625,"##aker":22626,"emitted":22627,"storylines":22628,"harvested":22629,"seismic":22630,"##iform":22631,"cheung":22632,"kilda":22633,"theoretically":22634,"barbie":22635,"lynx":22636,"##rgy":22637,"##tius":22638,"goblin":22639,"mata":22640,"poisonous":22641,"##nburg":22642,"reactive":22643,"residues":22644,"obedience":22645,"##евич":22646,"conjecture":22647,"##rac":22648,"401":22649,"hating":22650,"sixties":22651,"kicker":22652,"moaning":22653,"motown":22654,"##bha":22655,"emancipation":22656,"neoclassical":22657,"##hering":22658,"consoles":22659,"ebert":22660,"professorship":22661,"##tures":22662,"sustaining":22663,"assaults":22664,"obeyed":22665,"affluent":22666,"incurred":22667,"tornadoes":22668,"##eber":22669,"##zow":22670,"emphasizing":22671,"highlanders":22672,"cheated":22673,"helmets":22674,"##ctus":22675,"internship":22676,"terence":22677,"bony":22678,"executions":22679,"legislators":22680,"berries":22681,"peninsular":22682,"tinged":22683,"##aco":22684,"1689":22685,"amplifier":22686,"corvette":22687,"ribbons":22688,"lavish":22689,"pennant":22690,"##lander":22691,"worthless":22692,"##chfield":22693,"##forms":22694,"mariano":22695,"pyrenees":22696,"expenditures":22697,"##icides":22698,"chesterfield":22699,"mandir":22700,"tailor":22701,"39th":22702,"sergey":22703,"nestled":22704,"willed":22705,"aristocracy":22706,"devotees":22707,"goodnight":22708,"raaf":22709,"rumored":22710,"weaponry":22711,"remy":22712,"appropriations":22713,"harcourt":22714,"burr":22715,"riaa":22716,"##lence":22717,"limitation":22718,"unnoticed":22719,"guo":22720,"soaking":22721,"swamps":22722,"##tica":22723,"collapsing":22724,"tatiana":22725,"descriptive":22726,"brigham":22727,"psalm":22728,"##chment":22729,"maddox":22730,"##lization":22731,"patti":22732,"caliph":22733,"##aja":22734,"akron":22735,"injuring":22736,"serra":22737,"##ganj":22738,"basins":22739,"##sari":22740,"astonished":22741,"launcher":22742,"##church":22743,"hilary":22744,"wilkins":22745,"sewing":22746,"##sf":22747,"stinging":22748,"##fia":22749,"##ncia":22750,"underwood":22751,"startup":22752,"##ition":22753,"compilations":22754,"vibrations":22755,"embankment":22756,"jurist":22757,"##nity":22758,"bard":22759,"juventus":22760,"groundwater":22761,"kern":22762,"palaces":22763,"helium":22764,"boca":22765,"cramped":22766,"marissa":22767,"soto":22768,"##worm":22769,"jae":22770,"princely":22771,"##ggy":22772,"faso":22773,"bazaar":22774,"warmly":22775,"##voking":22776,"229":22777,"pairing":22778,"##lite":22779,"##grate":22780,"##nets":22781,"wien":22782,"freaked":22783,"ulysses":22784,"rebirth":22785,"##alia":22786,"##rent":22787,"mummy":22788,"guzman":22789,"jimenez":22790,"stilled":22791,"##nitz":22792,"trajectory":22793,"tha":22794,"woken":22795,"archival":22796,"professions":22797,"##pts":22798,"##pta":22799,"hilly":22800,"shadowy":22801,"shrink":22802,"##bolt":22803,"norwood":22804,"glued":22805,"migrate":22806,"stereotypes":22807,"devoid":22808,"##pheus":22809,"625":22810,"evacuate":22811,"horrors":22812,"infancy":22813,"gotham":22814,"knowles":22815,"optic":22816,"downloaded":22817,"sachs":22818,"kingsley":22819,"parramatta":22820,"darryl":22821,"mor":22822,"##onale":22823,"shady":22824,"commence":22825,"confesses":22826,"kan":22827,"##meter":22828,"##placed":22829,"marlborough":22830,"roundabout":22831,"regents":22832,"frigates":22833,"io":22834,"##imating":22835,"gothenburg":22836,"revoked":22837,"carvings":22838,"clockwise":22839,"convertible":22840,"intruder":22841,"##sche":22842,"banged":22843,"##ogo":22844,"vicky":22845,"bourgeois":22846,"##mony":22847,"dupont":22848,"footing":22849,"##gum":22850,"pd":22851,"##real":22852,"buckle":22853,"yun":22854,"penthouse":22855,"sane":22856,"720":22857,"serviced":22858,"stakeholders":22859,"neumann":22860,"bb":22861,"##eers":22862,"comb":22863,"##gam":22864,"catchment":22865,"pinning":22866,"rallies":22867,"typing":22868,"##elles":22869,"forefront":22870,"freiburg":22871,"sweetie":22872,"giacomo":22873,"widowed":22874,"goodwill":22875,"worshipped":22876,"aspirations":22877,"midday":22878,"##vat":22879,"fishery":22880,"##trick":22881,"bournemouth":22882,"turk":22883,"243":22884,"hearth":22885,"ethanol":22886,"guadalajara":22887,"murmurs":22888,"sl":22889,"##uge":22890,"afforded":22891,"scripted":22892,"##hta":22893,"wah":22894,"##jn":22895,"coroner":22896,"translucent":22897,"252":22898,"memorials":22899,"puck":22900,"progresses":22901,"clumsy":22902,"##race":22903,"315":22904,"candace":22905,"recounted":22906,"##27":22907,"##slin":22908,"##uve":22909,"filtering":22910,"##mac":22911,"howl":22912,"strata":22913,"heron":22914,"leveled":22915,"##ays":22916,"dubious":22917,"##oja":22918,"##т":22919,"##wheel":22920,"citations":22921,"exhibiting":22922,"##laya":22923,"##mics":22924,"##pods":22925,"turkic":22926,"##lberg":22927,"injunction":22928,"##ennial":22929,"##mit":22930,"antibodies":22931,"##44":22932,"organise":22933,"##rigues":22934,"cardiovascular":22935,"cushion":22936,"inverness":22937,"##zquez":22938,"dia":22939,"cocoa":22940,"sibling":22941,"##tman":22942,"##roid":22943,"expanse":22944,"feasible":22945,"tunisian":22946,"algiers":22947,"##relli":22948,"rus":22949,"bloomberg":22950,"dso":22951,"westphalia":22952,"bro":22953,"tacoma":22954,"281":22955,"downloads":22956,"##ours":22957,"konrad":22958,"duran":22959,"##hdi":22960,"continuum":22961,"jett":22962,"compares":22963,"legislator":22964,"secession":22965,"##nable":22966,"##gues":22967,"##zuka":22968,"translating":22969,"reacher":22970,"##gley":22971,"##ła":22972,"aleppo":22973,"##agi":22974,"tc":22975,"orchards":22976,"trapping":22977,"linguist":22978,"versatile":22979,"drumming":22980,"postage":22981,"calhoun":22982,"superiors":22983,"##mx":22984,"barefoot":22985,"leary":22986,"##cis":22987,"ignacio":22988,"alfa":22989,"kaplan":22990,"##rogen":22991,"bratislava":22992,"mori":22993,"##vot":22994,"disturb":22995,"haas":22996,"313":22997,"cartridges":22998,"gilmore":22999,"radiated":23000,"salford":23001,"tunic":23002,"hades":23003,"##ulsive":23004,"archeological":23005,"delilah":23006,"magistrates":23007,"auditioned":23008,"brewster":23009,"charters":23010,"empowerment":23011,"blogs":23012,"cappella":23013,"dynasties":23014,"iroquois":23015,"whipping":23016,"##krishna":23017,"raceway":23018,"truths":23019,"myra":23020,"weaken":23021,"judah":23022,"mcgregor":23023,"##horse":23024,"mic":23025,"refueling":23026,"37th":23027,"burnley":23028,"bosses":23029,"markus":23030,"premio":23031,"query":23032,"##gga":23033,"dunbar":23034,"##economic":23035,"darkest":23036,"lyndon":23037,"sealing":23038,"commendation":23039,"reappeared":23040,"##mun":23041,"addicted":23042,"ezio":23043,"slaughtered":23044,"satisfactory":23045,"shuffle":23046,"##eves":23047,"##thic":23048,"##uj":23049,"fortification":23050,"warrington":23051,"##otto":23052,"resurrected":23053,"fargo":23054,"mane":23055,"##utable":23056,"##lei":23057,"##space":23058,"foreword":23059,"ox":23060,"##aris":23061,"##vern":23062,"abrams":23063,"hua":23064,"##mento":23065,"sakura":23066,"##alo":23067,"uv":23068,"sentimental":23069,"##skaya":23070,"midfield":23071,"##eses":23072,"sturdy":23073,"scrolls":23074,"macleod":23075,"##kyu":23076,"entropy":23077,"##lance":23078,"mitochondrial":23079,"cicero":23080,"excelled":23081,"thinner":23082,"convoys":23083,"perceive":23084,"##oslav":23085,"##urable":23086,"systematically":23087,"grind":23088,"burkina":23089,"287":23090,"##tagram":23091,"ops":23092,"##aman":23093,"guantanamo":23094,"##cloth":23095,"##tite":23096,"forcefully":23097,"wavy":23098,"##jou":23099,"pointless":23100,"##linger":23101,"##tze":23102,"layton":23103,"portico":23104,"superficial":23105,"clerical":23106,"outlaws":23107,"##hism":23108,"burials":23109,"muir":23110,"##inn":23111,"creditors":23112,"hauling":23113,"rattle":23114,"##leg":23115,"calais":23116,"monde":23117,"archers":23118,"reclaimed":23119,"dwell":23120,"wexford":23121,"hellenic":23122,"falsely":23123,"remorse":23124,"##tek":23125,"dough":23126,"furnishings":23127,"##uttered":23128,"gabon":23129,"neurological":23130,"novice":23131,"##igraphy":23132,"contemplated":23133,"pulpit":23134,"nightstand":23135,"saratoga":23136,"##istan":23137,"documenting":23138,"pulsing":23139,"taluk":23140,"##firmed":23141,"busted":23142,"marital":23143,"##rien":23144,"disagreements":23145,"wasps":23146,"##yes":23147,"hodge":23148,"mcdonnell":23149,"mimic":23150,"fran":23151,"pendant":23152,"dhabi":23153,"musa":23154,"##nington":23155,"congratulations":23156,"argent":23157,"darrell":23158,"concussion":23159,"losers":23160,"regrets":23161,"thessaloniki":23162,"reversal":23163,"donaldson":23164,"hardwood":23165,"thence":23166,"achilles":23167,"ritter":23168,"##eran":23169,"demonic":23170,"jurgen":23171,"prophets":23172,"goethe":23173,"eki":23174,"classmate":23175,"buff":23176,"##cking":23177,"yank":23178,"irrational":23179,"##inging":23180,"perished":23181,"seductive":23182,"qur":23183,"sourced":23184,"##crat":23185,"##typic":23186,"mustard":23187,"ravine":23188,"barre":23189,"horizontally":23190,"characterization":23191,"phylogenetic":23192,"boise":23193,"##dit":23194,"##runner":23195,"##tower":23196,"brutally":23197,"intercourse":23198,"seduce":23199,"##bbing":23200,"fay":23201,"ferris":23202,"ogden":23203,"amar":23204,"nik":23205,"unarmed":23206,"##inator":23207,"evaluating":23208,"kyrgyzstan":23209,"sweetness":23210,"##lford":23211,"##oki":23212,"mccormick":23213,"meiji":23214,"notoriety":23215,"stimulate":23216,"disrupt":23217,"figuring":23218,"instructional":23219,"mcgrath":23220,"##zoo":23221,"groundbreaking":23222,"##lto":23223,"flinch":23224,"khorasan":23225,"agrarian":23226,"bengals":23227,"mixer":23228,"radiating":23229,"##sov":23230,"ingram":23231,"pitchers":23232,"nad":23233,"tariff":23234,"##cript":23235,"tata":23236,"##codes":23237,"##emi":23238,"##ungen":23239,"appellate":23240,"lehigh":23241,"##bled":23242,"##giri":23243,"brawl":23244,"duct":23245,"texans":23246,"##ciation":23247,"##ropolis":23248,"skipper":23249,"speculative":23250,"vomit":23251,"doctrines":23252,"stresses":23253,"253":23254,"davy":23255,"graders":23256,"whitehead":23257,"jozef":23258,"timely":23259,"cumulative":23260,"haryana":23261,"paints":23262,"appropriately":23263,"boon":23264,"cactus":23265,"##ales":23266,"##pid":23267,"dow":23268,"legions":23269,"##pit":23270,"perceptions":23271,"1730":23272,"picturesque":23273,"##yse":23274,"periphery":23275,"rune":23276,"wr":23277,"##aha":23278,"celtics":23279,"sentencing":23280,"whoa":23281,"##erin":23282,"confirms":23283,"variance":23284,"425":23285,"moines":23286,"mathews":23287,"spade":23288,"rave":23289,"m1":23290,"fronted":23291,"fx":23292,"blending":23293,"alleging":23294,"reared":23295,"##gl":23296,"237":23297,"##paper":23298,"grassroots":23299,"eroded":23300,"##free":23301,"##physical":23302,"directs":23303,"ordeal":23304,"##sław":23305,"accelerate":23306,"hacker":23307,"rooftop":23308,"##inia":23309,"lev":23310,"buys":23311,"cebu":23312,"devote":23313,"##lce":23314,"specialising":23315,"##ulsion":23316,"choreographed":23317,"repetition":23318,"warehouses":23319,"##ryl":23320,"paisley":23321,"tuscany":23322,"analogy":23323,"sorcerer":23324,"hash":23325,"huts":23326,"shards":23327,"descends":23328,"exclude":23329,"nix":23330,"chaplin":23331,"gaga":23332,"ito":23333,"vane":23334,"##drich":23335,"causeway":23336,"misconduct":23337,"limo":23338,"orchestrated":23339,"glands":23340,"jana":23341,"##kot":23342,"u2":23343,"##mple":23344,"##sons":23345,"branching":23346,"contrasts":23347,"scoop":23348,"longed":23349,"##virus":23350,"chattanooga":23351,"##75":23352,"syrup":23353,"cornerstone":23354,"##tized":23355,"##mind":23356,"##iaceae":23357,"careless":23358,"precedence":23359,"frescoes":23360,"##uet":23361,"chilled":23362,"consult":23363,"modelled":23364,"snatch":23365,"peat":23366,"##thermal":23367,"caucasian":23368,"humane":23369,"relaxation":23370,"spins":23371,"temperance":23372,"##lbert":23373,"occupations":23374,"lambda":23375,"hybrids":23376,"moons":23377,"mp3":23378,"##oese":23379,"247":23380,"rolf":23381,"societal":23382,"yerevan":23383,"ness":23384,"##ssler":23385,"befriended":23386,"mechanized":23387,"nominate":23388,"trough":23389,"boasted":23390,"cues":23391,"seater":23392,"##hom":23393,"bends":23394,"##tangle":23395,"conductors":23396,"emptiness":23397,"##lmer":23398,"eurasian":23399,"adriatic":23400,"tian":23401,"##cie":23402,"anxiously":23403,"lark":23404,"propellers":23405,"chichester":23406,"jock":23407,"ev":23408,"2a":23409,"##holding":23410,"credible":23411,"recounts":23412,"tori":23413,"loyalist":23414,"abduction":23415,"##hoot":23416,"##redo":23417,"nepali":23418,"##mite":23419,"ventral":23420,"tempting":23421,"##ango":23422,"##crats":23423,"steered":23424,"##wice":23425,"javelin":23426,"dipping":23427,"laborers":23428,"prentice":23429,"looming":23430,"titanium":23431,"##ː":23432,"badges":23433,"emir":23434,"tensor":23435,"##ntation":23436,"egyptians":23437,"rash":23438,"denies":23439,"hawthorne":23440,"lombard":23441,"showers":23442,"wehrmacht":23443,"dietary":23444,"trojan":23445,"##reus":23446,"welles":23447,"executing":23448,"horseshoe":23449,"lifeboat":23450,"##lak":23451,"elsa":23452,"infirmary":23453,"nearing":23454,"roberta":23455,"boyer":23456,"mutter":23457,"trillion":23458,"joanne":23459,"##fine":23460,"##oked":23461,"sinks":23462,"vortex":23463,"uruguayan":23464,"clasp":23465,"sirius":23466,"##block":23467,"accelerator":23468,"prohibit":23469,"sunken":23470,"byu":23471,"chronological":23472,"diplomats":23473,"ochreous":23474,"510":23475,"symmetrical":23476,"1644":23477,"maia":23478,"##tology":23479,"salts":23480,"reigns":23481,"atrocities":23482,"##ия":23483,"hess":23484,"bared":23485,"issn":23486,"##vyn":23487,"cater":23488,"saturated":23489,"##cycle":23490,"##isse":23491,"sable":23492,"voyager":23493,"dyer":23494,"yusuf":23495,"##inge":23496,"fountains":23497,"wolff":23498,"##39":23499,"##nni":23500,"engraving":23501,"rollins":23502,"atheist":23503,"ominous":23504,"##ault":23505,"herr":23506,"chariot":23507,"martina":23508,"strung":23509,"##fell":23510,"##farlane":23511,"horrific":23512,"sahib":23513,"gazes":23514,"saetan":23515,"erased":23516,"ptolemy":23517,"##olic":23518,"flushing":23519,"lauderdale":23520,"analytic":23521,"##ices":23522,"530":23523,"navarro":23524,"beak":23525,"gorilla":23526,"herrera":23527,"broom":23528,"guadalupe":23529,"raiding":23530,"sykes":23531,"311":23532,"bsc":23533,"deliveries":23534,"1720":23535,"invasions":23536,"carmichael":23537,"tajikistan":23538,"thematic":23539,"ecumenical":23540,"sentiments":23541,"onstage":23542,"##rians":23543,"##brand":23544,"##sume":23545,"catastrophic":23546,"flanks":23547,"molten":23548,"##arns":23549,"waller":23550,"aimee":23551,"terminating":23552,"##icing":23553,"alternately":23554,"##oche":23555,"nehru":23556,"printers":23557,"outraged":23558,"##eving":23559,"empires":23560,"template":23561,"banners":23562,"repetitive":23563,"za":23564,"##oise":23565,"vegetarian":23566,"##tell":23567,"guiana":23568,"opt":23569,"cavendish":23570,"lucknow":23571,"synthesized":23572,"##hani":23573,"##mada":23574,"finalized":23575,"##ctable":23576,"fictitious":23577,"mayoral":23578,"unreliable":23579,"##enham":23580,"embracing":23581,"peppers":23582,"rbis":23583,"##chio":23584,"##neo":23585,"inhibition":23586,"slashed":23587,"togo":23588,"orderly":23589,"embroidered":23590,"safari":23591,"salty":23592,"236":23593,"barron":23594,"benito":23595,"totaled":23596,"##dak":23597,"pubs":23598,"simulated":23599,"caden":23600,"devin":23601,"tolkien":23602,"momma":23603,"welding":23604,"sesame":23605,"##ept":23606,"gottingen":23607,"hardness":23608,"630":23609,"shaman":23610,"temeraire":23611,"620":23612,"adequately":23613,"pediatric":23614,"##kit":23615,"ck":23616,"assertion":23617,"radicals":23618,"composure":23619,"cadence":23620,"seafood":23621,"beaufort":23622,"lazarus":23623,"mani":23624,"warily":23625,"cunning":23626,"kurdistan":23627,"249":23628,"cantata":23629,"##kir":23630,"ares":23631,"##41":23632,"##clusive":23633,"nape":23634,"townland":23635,"geared":23636,"insulted":23637,"flutter":23638,"boating":23639,"violate":23640,"draper":23641,"dumping":23642,"malmo":23643,"##hh":23644,"##romatic":23645,"firearm":23646,"alta":23647,"bono":23648,"obscured":23649,"##clave":23650,"exceeds":23651,"panorama":23652,"unbelievable":23653,"##train":23654,"preschool":23655,"##essed":23656,"disconnected":23657,"installing":23658,"rescuing":23659,"secretaries":23660,"accessibility":23661,"##castle":23662,"##drive":23663,"##ifice":23664,"##film":23665,"bouts":23666,"slug":23667,"waterway":23668,"mindanao":23669,"##buro":23670,"##ratic":23671,"halves":23672,"##ل":23673,"calming":23674,"liter":23675,"maternity":23676,"adorable":23677,"bragg":23678,"electrification":23679,"mcc":23680,"##dote":23681,"roxy":23682,"schizophrenia":23683,"##body":23684,"munoz":23685,"kaye":23686,"whaling":23687,"239":23688,"mil":23689,"tingling":23690,"tolerant":23691,"##ago":23692,"unconventional":23693,"volcanoes":23694,"##finder":23695,"deportivo":23696,"##llie":23697,"robson":23698,"kaufman":23699,"neuroscience":23700,"wai":23701,"deportation":23702,"masovian":23703,"scraping":23704,"converse":23705,"##bh":23706,"hacking":23707,"bulge":23708,"##oun":23709,"administratively":23710,"yao":23711,"580":23712,"amp":23713,"mammoth":23714,"booster":23715,"claremont":23716,"hooper":23717,"nomenclature":23718,"pursuits":23719,"mclaughlin":23720,"melinda":23721,"##sul":23722,"catfish":23723,"barclay":23724,"substrates":23725,"taxa":23726,"zee":23727,"originals":23728,"kimberly":23729,"packets":23730,"padma":23731,"##ality":23732,"borrowing":23733,"ostensibly":23734,"solvent":23735,"##bri":23736,"##genesis":23737,"##mist":23738,"lukas":23739,"shreveport":23740,"veracruz":23741,"##ь":23742,"##lou":23743,"##wives":23744,"cheney":23745,"tt":23746,"anatolia":23747,"hobbs":23748,"##zyn":23749,"cyclic":23750,"radiant":23751,"alistair":23752,"greenish":23753,"siena":23754,"dat":23755,"independents":23756,"##bation":23757,"conform":23758,"pieter":23759,"hyper":23760,"applicant":23761,"bradshaw":23762,"spores":23763,"telangana":23764,"vinci":23765,"inexpensive":23766,"nuclei":23767,"322":23768,"jang":23769,"nme":23770,"soho":23771,"spd":23772,"##ign":23773,"cradled":23774,"receptionist":23775,"pow":23776,"##43":23777,"##rika":23778,"fascism":23779,"##ifer":23780,"experimenting":23781,"##ading":23782,"##iec":23783,"##region":23784,"345":23785,"jocelyn":23786,"maris":23787,"stair":23788,"nocturnal":23789,"toro":23790,"constabulary":23791,"elgin":23792,"##kker":23793,"msc":23794,"##giving":23795,"##schen":23796,"##rase":23797,"doherty":23798,"doping":23799,"sarcastically":23800,"batter":23801,"maneuvers":23802,"##cano":23803,"##apple":23804,"##gai":23805,"##git":23806,"intrinsic":23807,"##nst":23808,"##stor":23809,"1753":23810,"showtime":23811,"cafes":23812,"gasps":23813,"lviv":23814,"ushered":23815,"##thed":23816,"fours":23817,"restart":23818,"astonishment":23819,"transmitting":23820,"flyer":23821,"shrugs":23822,"##sau":23823,"intriguing":23824,"cones":23825,"dictated":23826,"mushrooms":23827,"medial":23828,"##kovsky":23829,"##elman":23830,"escorting":23831,"gaped":23832,"##26":23833,"godfather":23834,"##door":23835,"##sell":23836,"djs":23837,"recaptured":23838,"timetable":23839,"vila":23840,"1710":23841,"3a":23842,"aerodrome":23843,"mortals":23844,"scientology":23845,"##orne":23846,"angelina":23847,"mag":23848,"convection":23849,"unpaid":23850,"insertion":23851,"intermittent":23852,"lego":23853,"##nated":23854,"endeavor":23855,"kota":23856,"pereira":23857,"##lz":23858,"304":23859,"bwv":23860,"glamorgan":23861,"insults":23862,"agatha":23863,"fey":23864,"##cend":23865,"fleetwood":23866,"mahogany":23867,"protruding":23868,"steamship":23869,"zeta":23870,"##arty":23871,"mcguire":23872,"suspense":23873,"##sphere":23874,"advising":23875,"urges":23876,"##wala":23877,"hurriedly":23878,"meteor":23879,"gilded":23880,"inline":23881,"arroyo":23882,"stalker":23883,"##oge":23884,"excitedly":23885,"revered":23886,"##cure":23887,"earle":23888,"introductory":23889,"##break":23890,"##ilde":23891,"mutants":23892,"puff":23893,"pulses":23894,"reinforcement":23895,"##haling":23896,"curses":23897,"lizards":23898,"stalk":23899,"correlated":23900,"##fixed":23901,"fallout":23902,"macquarie":23903,"##unas":23904,"bearded":23905,"denton":23906,"heaving":23907,"802":23908,"##ocation":23909,"winery":23910,"assign":23911,"dortmund":23912,"##lkirk":23913,"everest":23914,"invariant":23915,"charismatic":23916,"susie":23917,"##elling":23918,"bled":23919,"lesley":23920,"telegram":23921,"sumner":23922,"bk":23923,"##ogen":23924,"##к":23925,"wilcox":23926,"needy":23927,"colbert":23928,"duval":23929,"##iferous":23930,"##mbled":23931,"allotted":23932,"attends":23933,"imperative":23934,"##hita":23935,"replacements":23936,"hawker":23937,"##inda":23938,"insurgency":23939,"##zee":23940,"##eke":23941,"casts":23942,"##yla":23943,"680":23944,"ives":23945,"transitioned":23946,"##pack":23947,"##powering":23948,"authoritative":23949,"baylor":23950,"flex":23951,"cringed":23952,"plaintiffs":23953,"woodrow":23954,"##skie":23955,"drastic":23956,"ape":23957,"aroma":23958,"unfolded":23959,"commotion":23960,"nt":23961,"preoccupied":23962,"theta":23963,"routines":23964,"lasers":23965,"privatization":23966,"wand":23967,"domino":23968,"ek":23969,"clenching":23970,"nsa":23971,"strategically":23972,"showered":23973,"bile":23974,"handkerchief":23975,"pere":23976,"storing":23977,"christophe":23978,"insulting":23979,"316":23980,"nakamura":23981,"romani":23982,"asiatic":23983,"magdalena":23984,"palma":23985,"cruises":23986,"stripping":23987,"405":23988,"konstantin":23989,"soaring":23990,"##berman":23991,"colloquially":23992,"forerunner":23993,"havilland":23994,"incarcerated":23995,"parasites":23996,"sincerity":23997,"##utus":23998,"disks":23999,"plank":24000,"saigon":24001,"##ining":24002,"corbin":24003,"homo":24004,"ornaments":24005,"powerhouse":24006,"##tlement":24007,"chong":24008,"fastened":24009,"feasibility":24010,"idf":24011,"morphological":24012,"usable":24013,"##nish":24014,"##zuki":24015,"aqueduct":24016,"jaguars":24017,"keepers":24018,"##flies":24019,"aleksandr":24020,"faust":24021,"assigns":24022,"ewing":24023,"bacterium":24024,"hurled":24025,"tricky":24026,"hungarians":24027,"integers":24028,"wallis":24029,"321":24030,"yamaha":24031,"##isha":24032,"hushed":24033,"oblivion":24034,"aviator":24035,"evangelist":24036,"friars":24037,"##eller":24038,"monograph":24039,"ode":24040,"##nary":24041,"airplanes":24042,"labourers":24043,"charms":24044,"##nee":24045,"1661":24046,"hagen":24047,"tnt":24048,"rudder":24049,"fiesta":24050,"transcript":24051,"dorothea":24052,"ska":24053,"inhibitor":24054,"maccabi":24055,"retorted":24056,"raining":24057,"encompassed":24058,"clauses":24059,"menacing":24060,"1642":24061,"lineman":24062,"##gist":24063,"vamps":24064,"##ape":24065,"##dick":24066,"gloom":24067,"##rera":24068,"dealings":24069,"easing":24070,"seekers":24071,"##nut":24072,"##pment":24073,"helens":24074,"unmanned":24075,"##anu":24076,"##isson":24077,"basics":24078,"##amy":24079,"##ckman":24080,"adjustments":24081,"1688":24082,"brutality":24083,"horne":24084,"##zell":24085,"sui":24086,"##55":24087,"##mable":24088,"aggregator":24089,"##thal":24090,"rhino":24091,"##drick":24092,"##vira":24093,"counters":24094,"zoom":24095,"##01":24096,"##rting":24097,"mn":24098,"montenegrin":24099,"packard":24100,"##unciation":24101,"##♭":24102,"##kki":24103,"reclaim":24104,"scholastic":24105,"thugs":24106,"pulsed":24107,"##icia":24108,"syriac":24109,"quan":24110,"saddam":24111,"banda":24112,"kobe":24113,"blaming":24114,"buddies":24115,"dissent":24116,"##lusion":24117,"##usia":24118,"corbett":24119,"jaya":24120,"delle":24121,"erratic":24122,"lexie":24123,"##hesis":24124,"435":24125,"amiga":24126,"hermes":24127,"##pressing":24128,"##leen":24129,"chapels":24130,"gospels":24131,"jamal":24132,"##uating":24133,"compute":24134,"revolving":24135,"warp":24136,"##sso":24137,"##thes":24138,"armory":24139,"##eras":24140,"##gol":24141,"antrim":24142,"loki":24143,"##kow":24144,"##asian":24145,"##good":24146,"##zano":24147,"braid":24148,"handwriting":24149,"subdistrict":24150,"funky":24151,"pantheon":24152,"##iculate":24153,"concurrency":24154,"estimation":24155,"improper":24156,"juliana":24157,"##his":24158,"newcomers":24159,"johnstone":24160,"staten":24161,"communicated":24162,"##oco":24163,"##alle":24164,"sausage":24165,"stormy":24166,"##stered":24167,"##tters":24168,"superfamily":24169,"##grade":24170,"acidic":24171,"collateral":24172,"tabloid":24173,"##oped":24174,"##rza":24175,"bladder":24176,"austen":24177,"##ellant":24178,"mcgraw":24179,"##hay":24180,"hannibal":24181,"mein":24182,"aquino":24183,"lucifer":24184,"wo":24185,"badger":24186,"boar":24187,"cher":24188,"christensen":24189,"greenberg":24190,"interruption":24191,"##kken":24192,"jem":24193,"244":24194,"mocked":24195,"bottoms":24196,"cambridgeshire":24197,"##lide":24198,"sprawling":24199,"##bbly":24200,"eastwood":24201,"ghent":24202,"synth":24203,"##buck":24204,"advisers":24205,"##bah":24206,"nominally":24207,"hapoel":24208,"qu":24209,"daggers":24210,"estranged":24211,"fabricated":24212,"towels":24213,"vinnie":24214,"wcw":24215,"misunderstanding":24216,"anglia":24217,"nothin":24218,"unmistakable":24219,"##dust":24220,"##lova":24221,"chilly":24222,"marquette":24223,"truss":24224,"##edge":24225,"##erine":24226,"reece":24227,"##lty":24228,"##chemist":24229,"##connected":24230,"272":24231,"308":24232,"41st":24233,"bash":24234,"raion":24235,"waterfalls":24236,"##ump":24237,"##main":24238,"labyrinth":24239,"queue":24240,"theorist":24241,"##istle":24242,"bharatiya":24243,"flexed":24244,"soundtracks":24245,"rooney":24246,"leftist":24247,"patrolling":24248,"wharton":24249,"plainly":24250,"alleviate":24251,"eastman":24252,"schuster":24253,"topographic":24254,"engages":24255,"immensely":24256,"unbearable":24257,"fairchild":24258,"1620":24259,"dona":24260,"lurking":24261,"parisian":24262,"oliveira":24263,"ia":24264,"indictment":24265,"hahn":24266,"bangladeshi":24267,"##aster":24268,"vivo":24269,"##uming":24270,"##ential":24271,"antonia":24272,"expects":24273,"indoors":24274,"kildare":24275,"harlan":24276,"##logue":24277,"##ogenic":24278,"##sities":24279,"forgiven":24280,"##wat":24281,"childish":24282,"tavi":24283,"##mide":24284,"##orra":24285,"plausible":24286,"grimm":24287,"successively":24288,"scooted":24289,"##bola":24290,"##dget":24291,"##rith":24292,"spartans":24293,"emery":24294,"flatly":24295,"azure":24296,"epilogue":24297,"##wark":24298,"flourish":24299,"##iny":24300,"##tracted":24301,"##overs":24302,"##oshi":24303,"bestseller":24304,"distressed":24305,"receipt":24306,"spitting":24307,"hermit":24308,"topological":24309,"##cot":24310,"drilled":24311,"subunit":24312,"francs":24313,"##layer":24314,"eel":24315,"##fk":24316,"##itas":24317,"octopus":24318,"footprint":24319,"petitions":24320,"ufo":24321,"##say":24322,"##foil":24323,"interfering":24324,"leaking":24325,"palo":24326,"##metry":24327,"thistle":24328,"valiant":24329,"##pic":24330,"narayan":24331,"mcpherson":24332,"##fast":24333,"gonzales":24334,"##ym":24335,"##enne":24336,"dustin":24337,"novgorod":24338,"solos":24339,"##zman":24340,"doin":24341,"##raph":24342,"##patient":24343,"##meyer":24344,"soluble":24345,"ashland":24346,"cuffs":24347,"carole":24348,"pendleton":24349,"whistling":24350,"vassal":24351,"##river":24352,"deviation":24353,"revisited":24354,"constituents":24355,"rallied":24356,"rotate":24357,"loomed":24358,"##eil":24359,"##nting":24360,"amateurs":24361,"augsburg":24362,"auschwitz":24363,"crowns":24364,"skeletons":24365,"##cona":24366,"bonnet":24367,"257":24368,"dummy":24369,"globalization":24370,"simeon":24371,"sleeper":24372,"mandal":24373,"differentiated":24374,"##crow":24375,"##mare":24376,"milne":24377,"bundled":24378,"exasperated":24379,"talmud":24380,"owes":24381,"segregated":24382,"##feng":24383,"##uary":24384,"dentist":24385,"piracy":24386,"props":24387,"##rang":24388,"devlin":24389,"##torium":24390,"malicious":24391,"paws":24392,"##laid":24393,"dependency":24394,"##ergy":24395,"##fers":24396,"##enna":24397,"258":24398,"pistons":24399,"rourke":24400,"jed":24401,"grammatical":24402,"tres":24403,"maha":24404,"wig":24405,"512":24406,"ghostly":24407,"jayne":24408,"##achal":24409,"##creen":24410,"##ilis":24411,"##lins":24412,"##rence":24413,"designate":24414,"##with":24415,"arrogance":24416,"cambodian":24417,"clones":24418,"showdown":24419,"throttle":24420,"twain":24421,"##ception":24422,"lobes":24423,"metz":24424,"nagoya":24425,"335":24426,"braking":24427,"##furt":24428,"385":24429,"roaming":24430,"##minster":24431,"amin":24432,"crippled":24433,"##37":24434,"##llary":24435,"indifferent":24436,"hoffmann":24437,"idols":24438,"intimidating":24439,"1751":24440,"261":24441,"influenza":24442,"memo":24443,"onions":24444,"1748":24445,"bandage":24446,"consciously":24447,"##landa":24448,"##rage":24449,"clandestine":24450,"observes":24451,"swiped":24452,"tangle":24453,"##ener":24454,"##jected":24455,"##trum":24456,"##bill":24457,"##lta":24458,"hugs":24459,"congresses":24460,"josiah":24461,"spirited":24462,"##dek":24463,"humanist":24464,"managerial":24465,"filmmaking":24466,"inmate":24467,"rhymes":24468,"debuting":24469,"grimsby":24470,"ur":24471,"##laze":24472,"duplicate":24473,"vigor":24474,"##tf":24475,"republished":24476,"bolshevik":24477,"refurbishment":24478,"antibiotics":24479,"martini":24480,"methane":24481,"newscasts":24482,"royale":24483,"horizons":24484,"levant":24485,"iain":24486,"visas":24487,"##ischen":24488,"paler":24489,"##around":24490,"manifestation":24491,"snuck":24492,"alf":24493,"chop":24494,"futile":24495,"pedestal":24496,"rehab":24497,"##kat":24498,"bmg":24499,"kerman":24500,"res":24501,"fairbanks":24502,"jarrett":24503,"abstraction":24504,"saharan":24505,"##zek":24506,"1746":24507,"procedural":24508,"clearer":24509,"kincaid":24510,"sash":24511,"luciano":24512,"##ffey":24513,"crunch":24514,"helmut":24515,"##vara":24516,"revolutionaries":24517,"##tute":24518,"creamy":24519,"leach":24520,"##mmon":24521,"1747":24522,"permitting":24523,"nes":24524,"plight":24525,"wendell":24526,"##lese":24527,"contra":24528,"ts":24529,"clancy":24530,"ipa":24531,"mach":24532,"staples":24533,"autopsy":24534,"disturbances":24535,"nueva":24536,"karin":24537,"pontiac":24538,"##uding":24539,"proxy":24540,"venerable":24541,"haunt":24542,"leto":24543,"bergman":24544,"expands":24545,"##helm":24546,"wal":24547,"##pipe":24548,"canning":24549,"celine":24550,"cords":24551,"obesity":24552,"##enary":24553,"intrusion":24554,"planner":24555,"##phate":24556,"reasoned":24557,"sequencing":24558,"307":24559,"harrow":24560,"##chon":24561,"##dora":24562,"marred":24563,"mcintyre":24564,"repay":24565,"tarzan":24566,"darting":24567,"248":24568,"harrisburg":24569,"margarita":24570,"repulsed":24571,"##hur":24572,"##lding":24573,"belinda":24574,"hamburger":24575,"novo":24576,"compliant":24577,"runways":24578,"bingham":24579,"registrar":24580,"skyscraper":24581,"ic":24582,"cuthbert":24583,"improvisation":24584,"livelihood":24585,"##corp":24586,"##elial":24587,"admiring":24588,"##dened":24589,"sporadic":24590,"believer":24591,"casablanca":24592,"popcorn":24593,"##29":24594,"asha":24595,"shovel":24596,"##bek":24597,"##dice":24598,"coiled":24599,"tangible":24600,"##dez":24601,"casper":24602,"elsie":24603,"resin":24604,"tenderness":24605,"rectory":24606,"##ivision":24607,"avail":24608,"sonar":24609,"##mori":24610,"boutique":24611,"##dier":24612,"guerre":24613,"bathed":24614,"upbringing":24615,"vaulted":24616,"sandals":24617,"blessings":24618,"##naut":24619,"##utnant":24620,"1680":24621,"306":24622,"foxes":24623,"pia":24624,"corrosion":24625,"hesitantly":24626,"confederates":24627,"crystalline":24628,"footprints":24629,"shapiro":24630,"tirana":24631,"valentin":24632,"drones":24633,"45th":24634,"microscope":24635,"shipments":24636,"texted":24637,"inquisition":24638,"wry":24639,"guernsey":24640,"unauthorized":24641,"resigning":24642,"760":24643,"ripple":24644,"schubert":24645,"stu":24646,"reassure":24647,"felony":24648,"##ardo":24649,"brittle":24650,"koreans":24651,"##havan":24652,"##ives":24653,"dun":24654,"implicit":24655,"tyres":24656,"##aldi":24657,"##lth":24658,"magnolia":24659,"##ehan":24660,"##puri":24661,"##poulos":24662,"aggressively":24663,"fei":24664,"gr":24665,"familiarity":24666,"##poo":24667,"indicative":24668,"##trust":24669,"fundamentally":24670,"jimmie":24671,"overrun":24672,"395":24673,"anchors":24674,"moans":24675,"##opus":24676,"britannia":24677,"armagh":24678,"##ggle":24679,"purposely":24680,"seizing":24681,"##vao":24682,"bewildered":24683,"mundane":24684,"avoidance":24685,"cosmopolitan":24686,"geometridae":24687,"quartermaster":24688,"caf":24689,"415":24690,"chatter":24691,"engulfed":24692,"gleam":24693,"purge":24694,"##icate":24695,"juliette":24696,"jurisprudence":24697,"guerra":24698,"revisions":24699,"##bn":24700,"casimir":24701,"brew":24702,"##jm":24703,"1749":24704,"clapton":24705,"cloudy":24706,"conde":24707,"hermitage":24708,"278":24709,"simulations":24710,"torches":24711,"vincenzo":24712,"matteo":24713,"##rill":24714,"hidalgo":24715,"booming":24716,"westbound":24717,"accomplishment":24718,"tentacles":24719,"unaffected":24720,"##sius":24721,"annabelle":24722,"flopped":24723,"sloping":24724,"##litz":24725,"dreamer":24726,"interceptor":24727,"vu":24728,"##loh":24729,"consecration":24730,"copying":24731,"messaging":24732,"breaker":24733,"climates":24734,"hospitalized":24735,"1752":24736,"torino":24737,"afternoons":24738,"winfield":24739,"witnessing":24740,"##teacher":24741,"breakers":24742,"choirs":24743,"sawmill":24744,"coldly":24745,"##ege":24746,"sipping":24747,"haste":24748,"uninhabited":24749,"conical":24750,"bibliography":24751,"pamphlets":24752,"severn":24753,"edict":24754,"##oca":24755,"deux":24756,"illnesses":24757,"grips":24758,"##pl":24759,"rehearsals":24760,"sis":24761,"thinkers":24762,"tame":24763,"##keepers":24764,"1690":24765,"acacia":24766,"reformer":24767,"##osed":24768,"##rys":24769,"shuffling":24770,"##iring":24771,"##shima":24772,"eastbound":24773,"ionic":24774,"rhea":24775,"flees":24776,"littered":24777,"##oum":24778,"rocker":24779,"vomiting":24780,"groaning":24781,"champ":24782,"overwhelmingly":24783,"civilizations":24784,"paces":24785,"sloop":24786,"adoptive":24787,"##tish":24788,"skaters":24789,"##vres":24790,"aiding":24791,"mango":24792,"##joy":24793,"nikola":24794,"shriek":24795,"##ignon":24796,"pharmaceuticals":24797,"##mg":24798,"tuna":24799,"calvert":24800,"gustavo":24801,"stocked":24802,"yearbook":24803,"##urai":24804,"##mana":24805,"computed":24806,"subsp":24807,"riff":24808,"hanoi":24809,"kelvin":24810,"hamid":24811,"moors":24812,"pastures":24813,"summons":24814,"jihad":24815,"nectar":24816,"##ctors":24817,"bayou":24818,"untitled":24819,"pleasing":24820,"vastly":24821,"republics":24822,"intellect":24823,"##η":24824,"##ulio":24825,"##tou":24826,"crumbling":24827,"stylistic":24828,"sb":24829,"##ی":24830,"consolation":24831,"frequented":24832,"h₂o":24833,"walden":24834,"widows":24835,"##iens":24836,"404":24837,"##ignment":24838,"chunks":24839,"improves":24840,"288":24841,"grit":24842,"recited":24843,"##dev":24844,"snarl":24845,"sociological":24846,"##arte":24847,"##gul":24848,"inquired":24849,"##held":24850,"bruise":24851,"clube":24852,"consultancy":24853,"homogeneous":24854,"hornets":24855,"multiplication":24856,"pasta":24857,"prick":24858,"savior":24859,"##grin":24860,"##kou":24861,"##phile":24862,"yoon":24863,"##gara":24864,"grimes":24865,"vanishing":24866,"cheering":24867,"reacting":24868,"bn":24869,"distillery":24870,"##quisite":24871,"##vity":24872,"coe":24873,"dockyard":24874,"massif":24875,"##jord":24876,"escorts":24877,"voss":24878,"##valent":24879,"byte":24880,"chopped":24881,"hawke":24882,"illusions":24883,"workings":24884,"floats":24885,"##koto":24886,"##vac":24887,"kv":24888,"annapolis":24889,"madden":24890,"##onus":24891,"alvaro":24892,"noctuidae":24893,"##cum":24894,"##scopic":24895,"avenge":24896,"steamboat":24897,"forte":24898,"illustrates":24899,"erika":24900,"##trip":24901,"570":24902,"dew":24903,"nationalities":24904,"bran":24905,"manifested":24906,"thirsty":24907,"diversified":24908,"muscled":24909,"reborn":24910,"##standing":24911,"arson":24912,"##lessness":24913,"##dran":24914,"##logram":24915,"##boys":24916,"##kushima":24917,"##vious":24918,"willoughby":24919,"##phobia":24920,"286":24921,"alsace":24922,"dashboard":24923,"yuki":24924,"##chai":24925,"granville":24926,"myspace":24927,"publicized":24928,"tricked":24929,"##gang":24930,"adjective":24931,"##ater":24932,"relic":24933,"reorganisation":24934,"enthusiastically":24935,"indications":24936,"saxe":24937,"##lassified":24938,"consolidate":24939,"iec":24940,"padua":24941,"helplessly":24942,"ramps":24943,"renaming":24944,"regulars":24945,"pedestrians":24946,"accents":24947,"convicts":24948,"inaccurate":24949,"lowers":24950,"mana":24951,"##pati":24952,"barrie":24953,"bjp":24954,"outta":24955,"someplace":24956,"berwick":24957,"flanking":24958,"invoked":24959,"marrow":24960,"sparsely":24961,"excerpts":24962,"clothed":24963,"rei":24964,"##ginal":24965,"wept":24966,"##straße":24967,"##vish":24968,"alexa":24969,"excel":24970,"##ptive":24971,"membranes":24972,"aquitaine":24973,"creeks":24974,"cutler":24975,"sheppard":24976,"implementations":24977,"ns":24978,"##dur":24979,"fragrance":24980,"budge":24981,"concordia":24982,"magnesium":24983,"marcelo":24984,"##antes":24985,"gladly":24986,"vibrating":24987,"##rral":24988,"##ggles":24989,"montrose":24990,"##omba":24991,"lew":24992,"seamus":24993,"1630":24994,"cocky":24995,"##ament":24996,"##uen":24997,"bjorn":24998,"##rrick":24999,"fielder":25000,"fluttering":25001,"##lase":25002,"methyl":25003,"kimberley":25004,"mcdowell":25005,"reductions":25006,"barbed":25007,"##jic":25008,"##tonic":25009,"aeronautical":25010,"condensed":25011,"distracting":25012,"##promising":25013,"huffed":25014,"##cala":25015,"##sle":25016,"claudius":25017,"invincible":25018,"missy":25019,"pious":25020,"balthazar":25021,"ci":25022,"##lang":25023,"butte":25024,"combo":25025,"orson":25026,"##dication":25027,"myriad":25028,"1707":25029,"silenced":25030,"##fed":25031,"##rh":25032,"coco":25033,"netball":25034,"yourselves":25035,"##oza":25036,"clarify":25037,"heller":25038,"peg":25039,"durban":25040,"etudes":25041,"offender":25042,"roast":25043,"blackmail":25044,"curvature":25045,"##woods":25046,"vile":25047,"309":25048,"illicit":25049,"suriname":25050,"##linson":25051,"overture":25052,"1685":25053,"bubbling":25054,"gymnast":25055,"tucking":25056,"##mming":25057,"##ouin":25058,"maldives":25059,"##bala":25060,"gurney":25061,"##dda":25062,"##eased":25063,"##oides":25064,"backside":25065,"pinto":25066,"jars":25067,"racehorse":25068,"tending":25069,"##rdial":25070,"baronetcy":25071,"wiener":25072,"duly":25073,"##rke":25074,"barbarian":25075,"cupping":25076,"flawed":25077,"##thesis":25078,"bertha":25079,"pleistocene":25080,"puddle":25081,"swearing":25082,"##nob":25083,"##tically":25084,"fleeting":25085,"prostate":25086,"amulet":25087,"educating":25088,"##mined":25089,"##iti":25090,"##tler":25091,"75th":25092,"jens":25093,"respondents":25094,"analytics":25095,"cavaliers":25096,"papacy":25097,"raju":25098,"##iente":25099,"##ulum":25100,"##tip":25101,"funnel":25102,"271":25103,"disneyland":25104,"##lley":25105,"sociologist":25106,"##iam":25107,"2500":25108,"faulkner":25109,"louvre":25110,"menon":25111,"##dson":25112,"276":25113,"##ower":25114,"afterlife":25115,"mannheim":25116,"peptide":25117,"referees":25118,"comedians":25119,"meaningless":25120,"##anger":25121,"##laise":25122,"fabrics":25123,"hurley":25124,"renal":25125,"sleeps":25126,"##bour":25127,"##icle":25128,"breakout":25129,"kristin":25130,"roadside":25131,"animator":25132,"clover":25133,"disdain":25134,"unsafe":25135,"redesign":25136,"##urity":25137,"firth":25138,"barnsley":25139,"portage":25140,"reset":25141,"narrows":25142,"268":25143,"commandos":25144,"expansive":25145,"speechless":25146,"tubular":25147,"##lux":25148,"essendon":25149,"eyelashes":25150,"smashwords":25151,"##yad":25152,"##bang":25153,"##claim":25154,"craved":25155,"sprinted":25156,"chet":25157,"somme":25158,"astor":25159,"wrocław":25160,"orton":25161,"266":25162,"bane":25163,"##erving":25164,"##uing":25165,"mischief":25166,"##amps":25167,"##sund":25168,"scaling":25169,"terre":25170,"##xious":25171,"impairment":25172,"offenses":25173,"undermine":25174,"moi":25175,"soy":25176,"contiguous":25177,"arcadia":25178,"inuit":25179,"seam":25180,"##tops":25181,"macbeth":25182,"rebelled":25183,"##icative":25184,"##iot":25185,"590":25186,"elaborated":25187,"frs":25188,"uniformed":25189,"##dberg":25190,"259":25191,"powerless":25192,"priscilla":25193,"stimulated":25194,"980":25195,"qc":25196,"arboretum":25197,"frustrating":25198,"trieste":25199,"bullock":25200,"##nified":25201,"enriched":25202,"glistening":25203,"intern":25204,"##adia":25205,"locus":25206,"nouvelle":25207,"ollie":25208,"ike":25209,"lash":25210,"starboard":25211,"ee":25212,"tapestry":25213,"headlined":25214,"hove":25215,"rigged":25216,"##vite":25217,"pollock":25218,"##yme":25219,"thrive":25220,"clustered":25221,"cas":25222,"roi":25223,"gleamed":25224,"olympiad":25225,"##lino":25226,"pressured":25227,"regimes":25228,"##hosis":25229,"##lick":25230,"ripley":25231,"##ophone":25232,"kickoff":25233,"gallon":25234,"rockwell":25235,"##arable":25236,"crusader":25237,"glue":25238,"revolutions":25239,"scrambling":25240,"1714":25241,"grover":25242,"##jure":25243,"englishman":25244,"aztec":25245,"263":25246,"contemplating":25247,"coven":25248,"ipad":25249,"preach":25250,"triumphant":25251,"tufts":25252,"##esian":25253,"rotational":25254,"##phus":25255,"328":25256,"falkland":25257,"##brates":25258,"strewn":25259,"clarissa":25260,"rejoin":25261,"environmentally":25262,"glint":25263,"banded":25264,"drenched":25265,"moat":25266,"albanians":25267,"johor":25268,"rr":25269,"maestro":25270,"malley":25271,"nouveau":25272,"shaded":25273,"taxonomy":25274,"v6":25275,"adhere":25276,"bunk":25277,"airfields":25278,"##ritan":25279,"1741":25280,"encompass":25281,"remington":25282,"tran":25283,"##erative":25284,"amelie":25285,"mazda":25286,"friar":25287,"morals":25288,"passions":25289,"##zai":25290,"breadth":25291,"vis":25292,"##hae":25293,"argus":25294,"burnham":25295,"caressing":25296,"insider":25297,"rudd":25298,"##imov":25299,"##mini":25300,"##rso":25301,"italianate":25302,"murderous":25303,"textual":25304,"wainwright":25305,"armada":25306,"bam":25307,"weave":25308,"timer":25309,"##taken":25310,"##nh":25311,"fra":25312,"##crest":25313,"ardent":25314,"salazar":25315,"taps":25316,"tunis":25317,"##ntino":25318,"allegro":25319,"gland":25320,"philanthropic":25321,"##chester":25322,"implication":25323,"##optera":25324,"esq":25325,"judas":25326,"noticeably":25327,"wynn":25328,"##dara":25329,"inched":25330,"indexed":25331,"crises":25332,"villiers":25333,"bandit":25334,"royalties":25335,"patterned":25336,"cupboard":25337,"interspersed":25338,"accessory":25339,"isla":25340,"kendrick":25341,"entourage":25342,"stitches":25343,"##esthesia":25344,"headwaters":25345,"##ior":25346,"interlude":25347,"distraught":25348,"draught":25349,"1727":25350,"##basket":25351,"biased":25352,"sy":25353,"transient":25354,"triad":25355,"subgenus":25356,"adapting":25357,"kidd":25358,"shortstop":25359,"##umatic":25360,"dimly":25361,"spiked":25362,"mcleod":25363,"reprint":25364,"nellie":25365,"pretoria":25366,"windmill":25367,"##cek":25368,"singled":25369,"##mps":25370,"273":25371,"reunite":25372,"##orous":25373,"747":25374,"bankers":25375,"outlying":25376,"##omp":25377,"##ports":25378,"##tream":25379,"apologies":25380,"cosmetics":25381,"patsy":25382,"##deh":25383,"##ocks":25384,"##yson":25385,"bender":25386,"nantes":25387,"serene":25388,"##nad":25389,"lucha":25390,"mmm":25391,"323":25392,"##cius":25393,"##gli":25394,"cmll":25395,"coinage":25396,"nestor":25397,"juarez":25398,"##rook":25399,"smeared":25400,"sprayed":25401,"twitching":25402,"sterile":25403,"irina":25404,"embodied":25405,"juveniles":25406,"enveloped":25407,"miscellaneous":25408,"cancers":25409,"dq":25410,"gulped":25411,"luisa":25412,"crested":25413,"swat":25414,"donegal":25415,"ref":25416,"##anov":25417,"##acker":25418,"hearst":25419,"mercantile":25420,"##lika":25421,"doorbell":25422,"ua":25423,"vicki":25424,"##alla":25425,"##som":25426,"bilbao":25427,"psychologists":25428,"stryker":25429,"sw":25430,"horsemen":25431,"turkmenistan":25432,"wits":25433,"##national":25434,"anson":25435,"mathew":25436,"screenings":25437,"##umb":25438,"rihanna":25439,"##agne":25440,"##nessy":25441,"aisles":25442,"##iani":25443,"##osphere":25444,"hines":25445,"kenton":25446,"saskatoon":25447,"tasha":25448,"truncated":25449,"##champ":25450,"##itan":25451,"mildred":25452,"advises":25453,"fredrik":25454,"interpreting":25455,"inhibitors":25456,"##athi":25457,"spectroscopy":25458,"##hab":25459,"##kong":25460,"karim":25461,"panda":25462,"##oia":25463,"##nail":25464,"##vc":25465,"conqueror":25466,"kgb":25467,"leukemia":25468,"##dity":25469,"arrivals":25470,"cheered":25471,"pisa":25472,"phosphorus":25473,"shielded":25474,"##riated":25475,"mammal":25476,"unitarian":25477,"urgently":25478,"chopin":25479,"sanitary":25480,"##mission":25481,"spicy":25482,"drugged":25483,"hinges":25484,"##tort":25485,"tipping":25486,"trier":25487,"impoverished":25488,"westchester":25489,"##caster":25490,"267":25491,"epoch":25492,"nonstop":25493,"##gman":25494,"##khov":25495,"aromatic":25496,"centrally":25497,"cerro":25498,"##tively":25499,"##vio":25500,"billions":25501,"modulation":25502,"sedimentary":25503,"283":25504,"facilitating":25505,"outrageous":25506,"goldstein":25507,"##eak":25508,"##kt":25509,"ld":25510,"maitland":25511,"penultimate":25512,"pollard":25513,"##dance":25514,"fleets":25515,"spaceship":25516,"vertebrae":25517,"##nig":25518,"alcoholism":25519,"als":25520,"recital":25521,"##bham":25522,"##ference":25523,"##omics":25524,"m2":25525,"##bm":25526,"trois":25527,"##tropical":25528,"##в":25529,"commemorates":25530,"##meric":25531,"marge":25532,"##raction":25533,"1643":25534,"670":25535,"cosmetic":25536,"ravaged":25537,"##ige":25538,"catastrophe":25539,"eng":25540,"##shida":25541,"albrecht":25542,"arterial":25543,"bellamy":25544,"decor":25545,"harmon":25546,"##rde":25547,"bulbs":25548,"synchronized":25549,"vito":25550,"easiest":25551,"shetland":25552,"shielding":25553,"wnba":25554,"##glers":25555,"##ssar":25556,"##riam":25557,"brianna":25558,"cumbria":25559,"##aceous":25560,"##rard":25561,"cores":25562,"thayer":25563,"##nsk":25564,"brood":25565,"hilltop":25566,"luminous":25567,"carts":25568,"keynote":25569,"larkin":25570,"logos":25571,"##cta":25572,"##ا":25573,"##mund":25574,"##quay":25575,"lilith":25576,"tinted":25577,"277":25578,"wrestle":25579,"mobilization":25580,"##uses":25581,"sequential":25582,"siam":25583,"bloomfield":25584,"takahashi":25585,"274":25586,"##ieving":25587,"presenters":25588,"ringo":25589,"blazed":25590,"witty":25591,"##oven":25592,"##ignant":25593,"devastation":25594,"haydn":25595,"harmed":25596,"newt":25597,"therese":25598,"##peed":25599,"gershwin":25600,"molina":25601,"rabbis":25602,"sudanese":25603,"001":25604,"innate":25605,"restarted":25606,"##sack":25607,"##fus":25608,"slices":25609,"wb":25610,"##shah":25611,"enroll":25612,"hypothetical":25613,"hysterical":25614,"1743":25615,"fabio":25616,"indefinite":25617,"warped":25618,"##hg":25619,"exchanging":25620,"525":25621,"unsuitable":25622,"##sboro":25623,"gallo":25624,"1603":25625,"bret":25626,"cobalt":25627,"homemade":25628,"##hunter":25629,"mx":25630,"operatives":25631,"##dhar":25632,"terraces":25633,"durable":25634,"latch":25635,"pens":25636,"whorls":25637,"##ctuated":25638,"##eaux":25639,"billing":25640,"ligament":25641,"succumbed":25642,"##gly":25643,"regulators":25644,"spawn":25645,"##brick":25646,"##stead":25647,"filmfare":25648,"rochelle":25649,"##nzo":25650,"1725":25651,"circumstance":25652,"saber":25653,"supplements":25654,"##nsky":25655,"##tson":25656,"crowe":25657,"wellesley":25658,"carrot":25659,"##9th":25660,"##movable":25661,"primate":25662,"drury":25663,"sincerely":25664,"topical":25665,"##mad":25666,"##rao":25667,"callahan":25668,"kyiv":25669,"smarter":25670,"tits":25671,"undo":25672,"##yeh":25673,"announcements":25674,"anthologies":25675,"barrio":25676,"nebula":25677,"##islaus":25678,"##shaft":25679,"##tyn":25680,"bodyguards":25681,"2021":25682,"assassinate":25683,"barns":25684,"emmett":25685,"scully":25686,"##mah":25687,"##yd":25688,"##eland":25689,"##tino":25690,"##itarian":25691,"demoted":25692,"gorman":25693,"lashed":25694,"prized":25695,"adventist":25696,"writ":25697,"##gui":25698,"alla":25699,"invertebrates":25700,"##ausen":25701,"1641":25702,"amman":25703,"1742":25704,"align":25705,"healy":25706,"redistribution":25707,"##gf":25708,"##rize":25709,"insulation":25710,"##drop":25711,"adherents":25712,"hezbollah":25713,"vitro":25714,"ferns":25715,"yanking":25716,"269":25717,"php":25718,"registering":25719,"uppsala":25720,"cheerleading":25721,"confines":25722,"mischievous":25723,"tully":25724,"##ross":25725,"49th":25726,"docked":25727,"roam":25728,"stipulated":25729,"pumpkin":25730,"##bry":25731,"prompt":25732,"##ezer":25733,"blindly":25734,"shuddering":25735,"craftsmen":25736,"frail":25737,"scented":25738,"katharine":25739,"scramble":25740,"shaggy":25741,"sponge":25742,"helix":25743,"zaragoza":25744,"279":25745,"##52":25746,"43rd":25747,"backlash":25748,"fontaine":25749,"seizures":25750,"posse":25751,"cowan":25752,"nonfiction":25753,"telenovela":25754,"wwii":25755,"hammered":25756,"undone":25757,"##gpur":25758,"encircled":25759,"irs":25760,"##ivation":25761,"artefacts":25762,"oneself":25763,"searing":25764,"smallpox":25765,"##belle":25766,"##osaurus":25767,"shandong":25768,"breached":25769,"upland":25770,"blushing":25771,"rankin":25772,"infinitely":25773,"psyche":25774,"tolerated":25775,"docking":25776,"evicted":25777,"##col":25778,"unmarked":25779,"##lving":25780,"gnome":25781,"lettering":25782,"litres":25783,"musique":25784,"##oint":25785,"benevolent":25786,"##jal":25787,"blackened":25788,"##anna":25789,"mccall":25790,"racers":25791,"tingle":25792,"##ocene":25793,"##orestation":25794,"introductions":25795,"radically":25796,"292":25797,"##hiff":25798,"##باد":25799,"1610":25800,"1739":25801,"munchen":25802,"plead":25803,"##nka":25804,"condo":25805,"scissors":25806,"##sight":25807,"##tens":25808,"apprehension":25809,"##cey":25810,"##yin":25811,"hallmark":25812,"watering":25813,"formulas":25814,"sequels":25815,"##llas":25816,"aggravated":25817,"bae":25818,"commencing":25819,"##building":25820,"enfield":25821,"prohibits":25822,"marne":25823,"vedic":25824,"civilized":25825,"euclidean":25826,"jagger":25827,"beforehand":25828,"blasts":25829,"dumont":25830,"##arney":25831,"##nem":25832,"740":25833,"conversions":25834,"hierarchical":25835,"rios":25836,"simulator":25837,"##dya":25838,"##lellan":25839,"hedges":25840,"oleg":25841,"thrusts":25842,"shadowed":25843,"darby":25844,"maximize":25845,"1744":25846,"gregorian":25847,"##nded":25848,"##routed":25849,"sham":25850,"unspecified":25851,"##hog":25852,"emory":25853,"factual":25854,"##smo":25855,"##tp":25856,"fooled":25857,"##rger":25858,"ortega":25859,"wellness":25860,"marlon":25861,"##oton":25862,"##urance":25863,"casket":25864,"keating":25865,"ley":25866,"enclave":25867,"##ayan":25868,"char":25869,"influencing":25870,"jia":25871,"##chenko":25872,"412":25873,"ammonia":25874,"erebidae":25875,"incompatible":25876,"violins":25877,"cornered":25878,"##arat":25879,"grooves":25880,"astronauts":25881,"columbian":25882,"rampant":25883,"fabrication":25884,"kyushu":25885,"mahmud":25886,"vanish":25887,"##dern":25888,"mesopotamia":25889,"##lete":25890,"ict":25891,"##rgen":25892,"caspian":25893,"kenji":25894,"pitted":25895,"##vered":25896,"999":25897,"grimace":25898,"roanoke":25899,"tchaikovsky":25900,"twinned":25901,"##analysis":25902,"##awan":25903,"xinjiang":25904,"arias":25905,"clemson":25906,"kazakh":25907,"sizable":25908,"1662":25909,"##khand":25910,"##vard":25911,"plunge":25912,"tatum":25913,"vittorio":25914,"##nden":25915,"cholera":25916,"##dana":25917,"##oper":25918,"bracing":25919,"indifference":25920,"projectile":25921,"superliga":25922,"##chee":25923,"realises":25924,"upgrading":25925,"299":25926,"porte":25927,"retribution":25928,"##vies":25929,"nk":25930,"stil":25931,"##resses":25932,"ama":25933,"bureaucracy":25934,"blackberry":25935,"bosch":25936,"testosterone":25937,"collapses":25938,"greer":25939,"##pathic":25940,"ioc":25941,"fifties":25942,"malls":25943,"##erved":25944,"bao":25945,"baskets":25946,"adolescents":25947,"siegfried":25948,"##osity":25949,"##tosis":25950,"mantra":25951,"detecting":25952,"existent":25953,"fledgling":25954,"##cchi":25955,"dissatisfied":25956,"gan":25957,"telecommunication":25958,"mingled":25959,"sobbed":25960,"6000":25961,"controversies":25962,"outdated":25963,"taxis":25964,"##raus":25965,"fright":25966,"slams":25967,"##lham":25968,"##fect":25969,"##tten":25970,"detectors":25971,"fetal":25972,"tanned":25973,"##uw":25974,"fray":25975,"goth":25976,"olympian":25977,"skipping":25978,"mandates":25979,"scratches":25980,"sheng":25981,"unspoken":25982,"hyundai":25983,"tracey":25984,"hotspur":25985,"restrictive":25986,"##buch":25987,"americana":25988,"mundo":25989,"##bari":25990,"burroughs":25991,"diva":25992,"vulcan":25993,"##6th":25994,"distinctions":25995,"thumping":25996,"##ngen":25997,"mikey":25998,"sheds":25999,"fide":26000,"rescues":26001,"springsteen":26002,"vested":26003,"valuation":26004,"##ece":26005,"##ely":26006,"pinnacle":26007,"rake":26008,"sylvie":26009,"##edo":26010,"almond":26011,"quivering":26012,"##irus":26013,"alteration":26014,"faltered":26015,"##wad":26016,"51st":26017,"hydra":26018,"ticked":26019,"##kato":26020,"recommends":26021,"##dicated":26022,"antigua":26023,"arjun":26024,"stagecoach":26025,"wilfred":26026,"trickle":26027,"pronouns":26028,"##pon":26029,"aryan":26030,"nighttime":26031,"##anian":26032,"gall":26033,"pea":26034,"stitch":26035,"##hei":26036,"leung":26037,"milos":26038,"##dini":26039,"eritrea":26040,"nexus":26041,"starved":26042,"snowfall":26043,"kant":26044,"parasitic":26045,"cot":26046,"discus":26047,"hana":26048,"strikers":26049,"appleton":26050,"kitchens":26051,"##erina":26052,"##partisan":26053,"##itha":26054,"##vius":26055,"disclose":26056,"metis":26057,"##channel":26058,"1701":26059,"tesla":26060,"##vera":26061,"fitch":26062,"1735":26063,"blooded":26064,"##tila":26065,"decimal":26066,"##tang":26067,"##bai":26068,"cyclones":26069,"eun":26070,"bottled":26071,"peas":26072,"pensacola":26073,"basha":26074,"bolivian":26075,"crabs":26076,"boil":26077,"lanterns":26078,"partridge":26079,"roofed":26080,"1645":26081,"necks":26082,"##phila":26083,"opined":26084,"patting":26085,"##kla":26086,"##lland":26087,"chuckles":26088,"volta":26089,"whereupon":26090,"##nche":26091,"devout":26092,"euroleague":26093,"suicidal":26094,"##dee":26095,"inherently":26096,"involuntary":26097,"knitting":26098,"nasser":26099,"##hide":26100,"puppets":26101,"colourful":26102,"courageous":26103,"southend":26104,"stills":26105,"miraculous":26106,"hodgson":26107,"richer":26108,"rochdale":26109,"ethernet":26110,"greta":26111,"uniting":26112,"prism":26113,"umm":26114,"##haya":26115,"##itical":26116,"##utation":26117,"deterioration":26118,"pointe":26119,"prowess":26120,"##ropriation":26121,"lids":26122,"scranton":26123,"billings":26124,"subcontinent":26125,"##koff":26126,"##scope":26127,"brute":26128,"kellogg":26129,"psalms":26130,"degraded":26131,"##vez":26132,"stanisław":26133,"##ructured":26134,"ferreira":26135,"pun":26136,"astonishing":26137,"gunnar":26138,"##yat":26139,"arya":26140,"prc":26141,"gottfried":26142,"##tight":26143,"excursion":26144,"##ographer":26145,"dina":26146,"##quil":26147,"##nare":26148,"huffington":26149,"illustrious":26150,"wilbur":26151,"gundam":26152,"verandah":26153,"##zard":26154,"naacp":26155,"##odle":26156,"constructive":26157,"fjord":26158,"kade":26159,"##naud":26160,"generosity":26161,"thrilling":26162,"baseline":26163,"cayman":26164,"frankish":26165,"plastics":26166,"accommodations":26167,"zoological":26168,"##fting":26169,"cedric":26170,"qb":26171,"motorized":26172,"##dome":26173,"##otted":26174,"squealed":26175,"tackled":26176,"canucks":26177,"budgets":26178,"situ":26179,"asthma":26180,"dail":26181,"gabled":26182,"grasslands":26183,"whimpered":26184,"writhing":26185,"judgments":26186,"##65":26187,"minnie":26188,"pv":26189,"##carbon":26190,"bananas":26191,"grille":26192,"domes":26193,"monique":26194,"odin":26195,"maguire":26196,"markham":26197,"tierney":26198,"##estra":26199,"##chua":26200,"libel":26201,"poke":26202,"speedy":26203,"atrium":26204,"laval":26205,"notwithstanding":26206,"##edly":26207,"fai":26208,"kala":26209,"##sur":26210,"robb":26211,"##sma":26212,"listings":26213,"luz":26214,"supplementary":26215,"tianjin":26216,"##acing":26217,"enzo":26218,"jd":26219,"ric":26220,"scanner":26221,"croats":26222,"transcribed":26223,"##49":26224,"arden":26225,"cv":26226,"##hair":26227,"##raphy":26228,"##lver":26229,"##uy":26230,"357":26231,"seventies":26232,"staggering":26233,"alam":26234,"horticultural":26235,"hs":26236,"regression":26237,"timbers":26238,"blasting":26239,"##ounded":26240,"montagu":26241,"manipulating":26242,"##cit":26243,"catalytic":26244,"1550":26245,"troopers":26246,"##meo":26247,"condemnation":26248,"fitzpatrick":26249,"##oire":26250,"##roved":26251,"inexperienced":26252,"1670":26253,"castes":26254,"##lative":26255,"outing":26256,"314":26257,"dubois":26258,"flicking":26259,"quarrel":26260,"ste":26261,"learners":26262,"1625":26263,"iq":26264,"whistled":26265,"##class":26266,"282":26267,"classify":26268,"tariffs":26269,"temperament":26270,"355":26271,"folly":26272,"liszt":26273,"##yles":26274,"immersed":26275,"jordanian":26276,"ceasefire":26277,"apparel":26278,"extras":26279,"maru":26280,"fished":26281,"##bio":26282,"harta":26283,"stockport":26284,"assortment":26285,"craftsman":26286,"paralysis":26287,"transmitters":26288,"##cola":26289,"blindness":26290,"##wk":26291,"fatally":26292,"proficiency":26293,"solemnly":26294,"##orno":26295,"repairing":26296,"amore":26297,"groceries":26298,"ultraviolet":26299,"##chase":26300,"schoolhouse":26301,"##tua":26302,"resurgence":26303,"nailed":26304,"##otype":26305,"##×":26306,"ruse":26307,"saliva":26308,"diagrams":26309,"##tructing":26310,"albans":26311,"rann":26312,"thirties":26313,"1b":26314,"antennas":26315,"hilarious":26316,"cougars":26317,"paddington":26318,"stats":26319,"##eger":26320,"breakaway":26321,"ipod":26322,"reza":26323,"authorship":26324,"prohibiting":26325,"scoffed":26326,"##etz":26327,"##ttle":26328,"conscription":26329,"defected":26330,"trondheim":26331,"##fires":26332,"ivanov":26333,"keenan":26334,"##adan":26335,"##ciful":26336,"##fb":26337,"##slow":26338,"locating":26339,"##ials":26340,"##tford":26341,"cadiz":26342,"basalt":26343,"blankly":26344,"interned":26345,"rags":26346,"rattling":26347,"##tick":26348,"carpathian":26349,"reassured":26350,"sync":26351,"bum":26352,"guildford":26353,"iss":26354,"staunch":26355,"##onga":26356,"astronomers":26357,"sera":26358,"sofie":26359,"emergencies":26360,"susquehanna":26361,"##heard":26362,"duc":26363,"mastery":26364,"vh1":26365,"williamsburg":26366,"bayer":26367,"buckled":26368,"craving":26369,"##khan":26370,"##rdes":26371,"bloomington":26372,"##write":26373,"alton":26374,"barbecue":26375,"##bians":26376,"justine":26377,"##hri":26378,"##ndt":26379,"delightful":26380,"smartphone":26381,"newtown":26382,"photon":26383,"retrieval":26384,"peugeot":26385,"hissing":26386,"##monium":26387,"##orough":26388,"flavors":26389,"lighted":26390,"relaunched":26391,"tainted":26392,"##games":26393,"##lysis":26394,"anarchy":26395,"microscopic":26396,"hopping":26397,"adept":26398,"evade":26399,"evie":26400,"##beau":26401,"inhibit":26402,"sinn":26403,"adjustable":26404,"hurst":26405,"intuition":26406,"wilton":26407,"cisco":26408,"44th":26409,"lawful":26410,"lowlands":26411,"stockings":26412,"thierry":26413,"##dalen":26414,"##hila":26415,"##nai":26416,"fates":26417,"prank":26418,"tb":26419,"maison":26420,"lobbied":26421,"provocative":26422,"1724":26423,"4a":26424,"utopia":26425,"##qual":26426,"carbonate":26427,"gujarati":26428,"purcell":26429,"##rford":26430,"curtiss":26431,"##mei":26432,"overgrown":26433,"arenas":26434,"mediation":26435,"swallows":26436,"##rnik":26437,"respectful":26438,"turnbull":26439,"##hedron":26440,"##hope":26441,"alyssa":26442,"ozone":26443,"##ʻi":26444,"ami":26445,"gestapo":26446,"johansson":26447,"snooker":26448,"canteen":26449,"cuff":26450,"declines":26451,"empathy":26452,"stigma":26453,"##ags":26454,"##iner":26455,"##raine":26456,"taxpayers":26457,"gui":26458,"volga":26459,"##wright":26460,"##copic":26461,"lifespan":26462,"overcame":26463,"tattooed":26464,"enactment":26465,"giggles":26466,"##ador":26467,"##camp":26468,"barrington":26469,"bribe":26470,"obligatory":26471,"orbiting":26472,"peng":26473,"##enas":26474,"elusive":26475,"sucker":26476,"##vating":26477,"cong":26478,"hardship":26479,"empowered":26480,"anticipating":26481,"estrada":26482,"cryptic":26483,"greasy":26484,"detainees":26485,"planck":26486,"sudbury":26487,"plaid":26488,"dod":26489,"marriott":26490,"kayla":26491,"##ears":26492,"##vb":26493,"##zd":26494,"mortally":26495,"##hein":26496,"cognition":26497,"radha":26498,"319":26499,"liechtenstein":26500,"meade":26501,"richly":26502,"argyle":26503,"harpsichord":26504,"liberalism":26505,"trumpets":26506,"lauded":26507,"tyrant":26508,"salsa":26509,"tiled":26510,"lear":26511,"promoters":26512,"reused":26513,"slicing":26514,"trident":26515,"##chuk":26516,"##gami":26517,"##lka":26518,"cantor":26519,"checkpoint":26520,"##points":26521,"gaul":26522,"leger":26523,"mammalian":26524,"##tov":26525,"##aar":26526,"##schaft":26527,"doha":26528,"frenchman":26529,"nirvana":26530,"##vino":26531,"delgado":26532,"headlining":26533,"##eron":26534,"##iography":26535,"jug":26536,"tko":26537,"1649":26538,"naga":26539,"intersections":26540,"##jia":26541,"benfica":26542,"nawab":26543,"##suka":26544,"ashford":26545,"gulp":26546,"##deck":26547,"##vill":26548,"##rug":26549,"brentford":26550,"frazier":26551,"pleasures":26552,"dunne":26553,"potsdam":26554,"shenzhen":26555,"dentistry":26556,"##tec":26557,"flanagan":26558,"##dorff":26559,"##hear":26560,"chorale":26561,"dinah":26562,"prem":26563,"quezon":26564,"##rogated":26565,"relinquished":26566,"sutra":26567,"terri":26568,"##pani":26569,"flaps":26570,"##rissa":26571,"poly":26572,"##rnet":26573,"homme":26574,"aback":26575,"##eki":26576,"linger":26577,"womb":26578,"##kson":26579,"##lewood":26580,"doorstep":26581,"orthodoxy":26582,"threaded":26583,"westfield":26584,"##rval":26585,"dioceses":26586,"fridays":26587,"subsided":26588,"##gata":26589,"loyalists":26590,"##biotic":26591,"##ettes":26592,"letterman":26593,"lunatic":26594,"prelate":26595,"tenderly":26596,"invariably":26597,"souza":26598,"thug":26599,"winslow":26600,"##otide":26601,"furlongs":26602,"gogh":26603,"jeopardy":26604,"##runa":26605,"pegasus":26606,"##umble":26607,"humiliated":26608,"standalone":26609,"tagged":26610,"##roller":26611,"freshmen":26612,"klan":26613,"##bright":26614,"attaining":26615,"initiating":26616,"transatlantic":26617,"logged":26618,"viz":26619,"##uance":26620,"1723":26621,"combatants":26622,"intervening":26623,"stephane":26624,"chieftain":26625,"despised":26626,"grazed":26627,"317":26628,"cdc":26629,"galveston":26630,"godzilla":26631,"macro":26632,"simulate":26633,"##planes":26634,"parades":26635,"##esses":26636,"960":26637,"##ductive":26638,"##unes":26639,"equator":26640,"overdose":26641,"##cans":26642,"##hosh":26643,"##lifting":26644,"joshi":26645,"epstein":26646,"sonora":26647,"treacherous":26648,"aquatics":26649,"manchu":26650,"responsive":26651,"##sation":26652,"supervisory":26653,"##christ":26654,"##llins":26655,"##ibar":26656,"##balance":26657,"##uso":26658,"kimball":26659,"karlsruhe":26660,"mab":26661,"##emy":26662,"ignores":26663,"phonetic":26664,"reuters":26665,"spaghetti":26666,"820":26667,"almighty":26668,"danzig":26669,"rumbling":26670,"tombstone":26671,"designations":26672,"lured":26673,"outset":26674,"##felt":26675,"supermarkets":26676,"##wt":26677,"grupo":26678,"kei":26679,"kraft":26680,"susanna":26681,"##blood":26682,"comprehension":26683,"genealogy":26684,"##aghan":26685,"##verted":26686,"redding":26687,"##ythe":26688,"1722":26689,"bowing":26690,"##pore":26691,"##roi":26692,"lest":26693,"sharpened":26694,"fulbright":26695,"valkyrie":26696,"sikhs":26697,"##unds":26698,"swans":26699,"bouquet":26700,"merritt":26701,"##tage":26702,"##venting":26703,"commuted":26704,"redhead":26705,"clerks":26706,"leasing":26707,"cesare":26708,"dea":26709,"hazy":26710,"##vances":26711,"fledged":26712,"greenfield":26713,"servicemen":26714,"##gical":26715,"armando":26716,"blackout":26717,"dt":26718,"sagged":26719,"downloadable":26720,"intra":26721,"potion":26722,"pods":26723,"##4th":26724,"##mism":26725,"xp":26726,"attendants":26727,"gambia":26728,"stale":26729,"##ntine":26730,"plump":26731,"asteroids":26732,"rediscovered":26733,"buds":26734,"flea":26735,"hive":26736,"##neas":26737,"1737":26738,"classifications":26739,"debuts":26740,"##eles":26741,"olympus":26742,"scala":26743,"##eurs":26744,"##gno":26745,"##mute":26746,"hummed":26747,"sigismund":26748,"visuals":26749,"wiggled":26750,"await":26751,"pilasters":26752,"clench":26753,"sulfate":26754,"##ances":26755,"bellevue":26756,"enigma":26757,"trainee":26758,"snort":26759,"##sw":26760,"clouded":26761,"denim":26762,"##rank":26763,"##rder":26764,"churning":26765,"hartman":26766,"lodges":26767,"riches":26768,"sima":26769,"##missible":26770,"accountable":26771,"socrates":26772,"regulates":26773,"mueller":26774,"##cr":26775,"1702":26776,"avoids":26777,"solids":26778,"himalayas":26779,"nutrient":26780,"pup":26781,"##jevic":26782,"squat":26783,"fades":26784,"nec":26785,"##lates":26786,"##pina":26787,"##rona":26788,"##ου":26789,"privateer":26790,"tequila":26791,"##gative":26792,"##mpton":26793,"apt":26794,"hornet":26795,"immortals":26796,"##dou":26797,"asturias":26798,"cleansing":26799,"dario":26800,"##rries":26801,"##anta":26802,"etymology":26803,"servicing":26804,"zhejiang":26805,"##venor":26806,"##nx":26807,"horned":26808,"erasmus":26809,"rayon":26810,"relocating":26811,"£10":26812,"##bags":26813,"escalated":26814,"promenade":26815,"stubble":26816,"2010s":26817,"artisans":26818,"axial":26819,"liquids":26820,"mora":26821,"sho":26822,"yoo":26823,"##tsky":26824,"bundles":26825,"oldies":26826,"##nally":26827,"notification":26828,"bastion":26829,"##ths":26830,"sparkle":26831,"##lved":26832,"1728":26833,"leash":26834,"pathogen":26835,"highs":26836,"##hmi":26837,"immature":26838,"880":26839,"gonzaga":26840,"ignatius":26841,"mansions":26842,"monterrey":26843,"sweets":26844,"bryson":26845,"##loe":26846,"polled":26847,"regatta":26848,"brightest":26849,"pei":26850,"rosy":26851,"squid":26852,"hatfield":26853,"payroll":26854,"addict":26855,"meath":26856,"cornerback":26857,"heaviest":26858,"lodging":26859,"##mage":26860,"capcom":26861,"rippled":26862,"##sily":26863,"barnet":26864,"mayhem":26865,"ymca":26866,"snuggled":26867,"rousseau":26868,"##cute":26869,"blanchard":26870,"284":26871,"fragmented":26872,"leighton":26873,"chromosomes":26874,"risking":26875,"##md":26876,"##strel":26877,"##utter":26878,"corinne":26879,"coyotes":26880,"cynical":26881,"hiroshi":26882,"yeomanry":26883,"##ractive":26884,"ebook":26885,"grading":26886,"mandela":26887,"plume":26888,"agustin":26889,"magdalene":26890,"##rkin":26891,"bea":26892,"femme":26893,"trafford":26894,"##coll":26895,"##lun":26896,"##tance":26897,"52nd":26898,"fourier":26899,"upton":26900,"##mental":26901,"camilla":26902,"gust":26903,"iihf":26904,"islamabad":26905,"longevity":26906,"##kala":26907,"feldman":26908,"netting":26909,"##rization":26910,"endeavour":26911,"foraging":26912,"mfa":26913,"orr":26914,"##open":26915,"greyish":26916,"contradiction":26917,"graz":26918,"##ruff":26919,"handicapped":26920,"marlene":26921,"tweed":26922,"oaxaca":26923,"spp":26924,"campos":26925,"miocene":26926,"pri":26927,"configured":26928,"cooks":26929,"pluto":26930,"cozy":26931,"pornographic":26932,"##entes":26933,"70th":26934,"fairness":26935,"glided":26936,"jonny":26937,"lynne":26938,"rounding":26939,"sired":26940,"##emon":26941,"##nist":26942,"remade":26943,"uncover":26944,"##mack":26945,"complied":26946,"lei":26947,"newsweek":26948,"##jured":26949,"##parts":26950,"##enting":26951,"##pg":26952,"293":26953,"finer":26954,"guerrillas":26955,"athenian":26956,"deng":26957,"disused":26958,"stepmother":26959,"accuse":26960,"gingerly":26961,"seduction":26962,"521":26963,"confronting":26964,"##walker":26965,"##going":26966,"gora":26967,"nostalgia":26968,"sabres":26969,"virginity":26970,"wrenched":26971,"##minated":26972,"syndication":26973,"wielding":26974,"eyre":26975,"##56":26976,"##gnon":26977,"##igny":26978,"behaved":26979,"taxpayer":26980,"sweeps":26981,"##growth":26982,"childless":26983,"gallant":26984,"##ywood":26985,"amplified":26986,"geraldine":26987,"scrape":26988,"##ffi":26989,"babylonian":26990,"fresco":26991,"##rdan":26992,"##kney":26993,"##position":26994,"1718":26995,"restricting":26996,"tack":26997,"fukuoka":26998,"osborn":26999,"selector":27000,"partnering":27001,"##dlow":27002,"318":27003,"gnu":27004,"kia":27005,"tak":27006,"whitley":27007,"gables":27008,"##54":27009,"##mania":27010,"mri":27011,"softness":27012,"immersion":27013,"##bots":27014,"##evsky":27015,"1713":27016,"chilling":27017,"insignificant":27018,"pcs":27019,"##uis":27020,"elites":27021,"lina":27022,"purported":27023,"supplemental":27024,"teaming":27025,"##americana":27026,"##dding":27027,"##inton":27028,"proficient":27029,"rouen":27030,"##nage":27031,"##rret":27032,"niccolo":27033,"selects":27034,"##bread":27035,"fluffy":27036,"1621":27037,"gruff":27038,"knotted":27039,"mukherjee":27040,"polgara":27041,"thrash":27042,"nicholls":27043,"secluded":27044,"smoothing":27045,"thru":27046,"corsica":27047,"loaf":27048,"whitaker":27049,"inquiries":27050,"##rrier":27051,"##kam":27052,"indochina":27053,"289":27054,"marlins":27055,"myles":27056,"peking":27057,"##tea":27058,"extracts":27059,"pastry":27060,"superhuman":27061,"connacht":27062,"vogel":27063,"##ditional":27064,"##het":27065,"##udged":27066,"##lash":27067,"gloss":27068,"quarries":27069,"refit":27070,"teaser":27071,"##alic":27072,"##gaon":27073,"20s":27074,"materialized":27075,"sling":27076,"camped":27077,"pickering":27078,"tung":27079,"tracker":27080,"pursuant":27081,"##cide":27082,"cranes":27083,"soc":27084,"##cini":27085,"##typical":27086,"##viere":27087,"anhalt":27088,"overboard":27089,"workout":27090,"chores":27091,"fares":27092,"orphaned":27093,"stains":27094,"##logie":27095,"fenton":27096,"surpassing":27097,"joyah":27098,"triggers":27099,"##itte":27100,"grandmaster":27101,"##lass":27102,"##lists":27103,"clapping":27104,"fraudulent":27105,"ledger":27106,"nagasaki":27107,"##cor":27108,"##nosis":27109,"##tsa":27110,"eucalyptus":27111,"tun":27112,"##icio":27113,"##rney":27114,"##tara":27115,"dax":27116,"heroism":27117,"ina":27118,"wrexham":27119,"onboard":27120,"unsigned":27121,"##dates":27122,"moshe":27123,"galley":27124,"winnie":27125,"droplets":27126,"exiles":27127,"praises":27128,"watered":27129,"noodles":27130,"##aia":27131,"fein":27132,"adi":27133,"leland":27134,"multicultural":27135,"stink":27136,"bingo":27137,"comets":27138,"erskine":27139,"modernized":27140,"canned":27141,"constraint":27142,"domestically":27143,"chemotherapy":27144,"featherweight":27145,"stifled":27146,"##mum":27147,"darkly":27148,"irresistible":27149,"refreshing":27150,"hasty":27151,"isolate":27152,"##oys":27153,"kitchener":27154,"planners":27155,"##wehr":27156,"cages":27157,"yarn":27158,"implant":27159,"toulon":27160,"elects":27161,"childbirth":27162,"yue":27163,"##lind":27164,"##lone":27165,"cn":27166,"rightful":27167,"sportsman":27168,"junctions":27169,"remodeled":27170,"specifies":27171,"##rgh":27172,"291":27173,"##oons":27174,"complimented":27175,"##urgent":27176,"lister":27177,"ot":27178,"##logic":27179,"bequeathed":27180,"cheekbones":27181,"fontana":27182,"gabby":27183,"##dial":27184,"amadeus":27185,"corrugated":27186,"maverick":27187,"resented":27188,"triangles":27189,"##hered":27190,"##usly":27191,"nazareth":27192,"tyrol":27193,"1675":27194,"assent":27195,"poorer":27196,"sectional":27197,"aegean":27198,"##cous":27199,"296":27200,"nylon":27201,"ghanaian":27202,"##egorical":27203,"##weig":27204,"cushions":27205,"forbid":27206,"fusiliers":27207,"obstruction":27208,"somerville":27209,"##scia":27210,"dime":27211,"earrings":27212,"elliptical":27213,"leyte":27214,"oder":27215,"polymers":27216,"timmy":27217,"atm":27218,"midtown":27219,"piloted":27220,"settles":27221,"continual":27222,"externally":27223,"mayfield":27224,"##uh":27225,"enrichment":27226,"henson":27227,"keane":27228,"persians":27229,"1733":27230,"benji":27231,"braden":27232,"pep":27233,"324":27234,"##efe":27235,"contenders":27236,"pepsi":27237,"valet":27238,"##isches":27239,"298":27240,"##asse":27241,"##earing":27242,"goofy":27243,"stroll":27244,"##amen":27245,"authoritarian":27246,"occurrences":27247,"adversary":27248,"ahmedabad":27249,"tangent":27250,"toppled":27251,"dorchester":27252,"1672":27253,"modernism":27254,"marxism":27255,"islamist":27256,"charlemagne":27257,"exponential":27258,"racks":27259,"unicode":27260,"brunette":27261,"mbc":27262,"pic":27263,"skirmish":27264,"##bund":27265,"##lad":27266,"##powered":27267,"##yst":27268,"hoisted":27269,"messina":27270,"shatter":27271,"##ctum":27272,"jedi":27273,"vantage":27274,"##music":27275,"##neil":27276,"clemens":27277,"mahmoud":27278,"corrupted":27279,"authentication":27280,"lowry":27281,"nils":27282,"##washed":27283,"omnibus":27284,"wounding":27285,"jillian":27286,"##itors":27287,"##opped":27288,"serialized":27289,"narcotics":27290,"handheld":27291,"##arm":27292,"##plicity":27293,"intersecting":27294,"stimulating":27295,"##onis":27296,"crate":27297,"fellowships":27298,"hemingway":27299,"casinos":27300,"climatic":27301,"fordham":27302,"copeland":27303,"drip":27304,"beatty":27305,"leaflets":27306,"robber":27307,"brothel":27308,"madeira":27309,"##hedral":27310,"sphinx":27311,"ultrasound":27312,"##vana":27313,"valor":27314,"forbade":27315,"leonid":27316,"villas":27317,"##aldo":27318,"duane":27319,"marquez":27320,"##cytes":27321,"disadvantaged":27322,"forearms":27323,"kawasaki":27324,"reacts":27325,"consular":27326,"lax":27327,"uncles":27328,"uphold":27329,"##hopper":27330,"concepcion":27331,"dorsey":27332,"lass":27333,"##izan":27334,"arching":27335,"passageway":27336,"1708":27337,"researches":27338,"tia":27339,"internationals":27340,"##graphs":27341,"##opers":27342,"distinguishes":27343,"javanese":27344,"divert":27345,"##uven":27346,"plotted":27347,"##listic":27348,"##rwin":27349,"##erik":27350,"##tify":27351,"affirmative":27352,"signifies":27353,"validation":27354,"##bson":27355,"kari":27356,"felicity":27357,"georgina":27358,"zulu":27359,"##eros":27360,"##rained":27361,"##rath":27362,"overcoming":27363,"##dot":27364,"argyll":27365,"##rbin":27366,"1734":27367,"chiba":27368,"ratification":27369,"windy":27370,"earls":27371,"parapet":27372,"##marks":27373,"hunan":27374,"pristine":27375,"astrid":27376,"punta":27377,"##gart":27378,"brodie":27379,"##kota":27380,"##oder":27381,"malaga":27382,"minerva":27383,"rouse":27384,"##phonic":27385,"bellowed":27386,"pagoda":27387,"portals":27388,"reclamation":27389,"##gur":27390,"##odies":27391,"##⁄₄":27392,"parentheses":27393,"quoting":27394,"allergic":27395,"palette":27396,"showcases":27397,"benefactor":27398,"heartland":27399,"nonlinear":27400,"##tness":27401,"bladed":27402,"cheerfully":27403,"scans":27404,"##ety":27405,"##hone":27406,"1666":27407,"girlfriends":27408,"pedersen":27409,"hiram":27410,"sous":27411,"##liche":27412,"##nator":27413,"1683":27414,"##nery":27415,"##orio":27416,"##umen":27417,"bobo":27418,"primaries":27419,"smiley":27420,"##cb":27421,"unearthed":27422,"uniformly":27423,"fis":27424,"metadata":27425,"1635":27426,"ind":27427,"##oted":27428,"recoil":27429,"##titles":27430,"##tura":27431,"##ια":27432,"406":27433,"hilbert":27434,"jamestown":27435,"mcmillan":27436,"tulane":27437,"seychelles":27438,"##frid":27439,"antics":27440,"coli":27441,"fated":27442,"stucco":27443,"##grants":27444,"1654":27445,"bulky":27446,"accolades":27447,"arrays":27448,"caledonian":27449,"carnage":27450,"optimism":27451,"puebla":27452,"##tative":27453,"##cave":27454,"enforcing":27455,"rotherham":27456,"seo":27457,"dunlop":27458,"aeronautics":27459,"chimed":27460,"incline":27461,"zoning":27462,"archduke":27463,"hellenistic":27464,"##oses":27465,"##sions":27466,"candi":27467,"thong":27468,"##ople":27469,"magnate":27470,"rustic":27471,"##rsk":27472,"projective":27473,"slant":27474,"##offs":27475,"danes":27476,"hollis":27477,"vocalists":27478,"##ammed":27479,"congenital":27480,"contend":27481,"gesellschaft":27482,"##ocating":27483,"##pressive":27484,"douglass":27485,"quieter":27486,"##cm":27487,"##kshi":27488,"howled":27489,"salim":27490,"spontaneously":27491,"townsville":27492,"buena":27493,"southport":27494,"##bold":27495,"kato":27496,"1638":27497,"faerie":27498,"stiffly":27499,"##vus":27500,"##rled":27501,"297":27502,"flawless":27503,"realising":27504,"taboo":27505,"##7th":27506,"bytes":27507,"straightening":27508,"356":27509,"jena":27510,"##hid":27511,"##rmin":27512,"cartwright":27513,"berber":27514,"bertram":27515,"soloists":27516,"411":27517,"noses":27518,"417":27519,"coping":27520,"fission":27521,"hardin":27522,"inca":27523,"##cen":27524,"1717":27525,"mobilized":27526,"vhf":27527,"##raf":27528,"biscuits":27529,"curate":27530,"##85":27531,"##anial":27532,"331":27533,"gaunt":27534,"neighbourhoods":27535,"1540":27536,"##abas":27537,"blanca":27538,"bypassed":27539,"sockets":27540,"behold":27541,"coincidentally":27542,"##bane":27543,"nara":27544,"shave":27545,"splinter":27546,"terrific":27547,"##arion":27548,"##erian":27549,"commonplace":27550,"juris":27551,"redwood":27552,"waistband":27553,"boxed":27554,"caitlin":27555,"fingerprints":27556,"jennie":27557,"naturalized":27558,"##ired":27559,"balfour":27560,"craters":27561,"jody":27562,"bungalow":27563,"hugely":27564,"quilt":27565,"glitter":27566,"pigeons":27567,"undertaker":27568,"bulging":27569,"constrained":27570,"goo":27571,"##sil":27572,"##akh":27573,"assimilation":27574,"reworked":27575,"##person":27576,"persuasion":27577,"##pants":27578,"felicia":27579,"##cliff":27580,"##ulent":27581,"1732":27582,"explodes":27583,"##dun":27584,"##inium":27585,"##zic":27586,"lyman":27587,"vulture":27588,"hog":27589,"overlook":27590,"begs":27591,"northwards":27592,"ow":27593,"spoil":27594,"##urer":27595,"fatima":27596,"favorably":27597,"accumulate":27598,"sargent":27599,"sorority":27600,"corresponded":27601,"dispersal":27602,"kochi":27603,"toned":27604,"##imi":27605,"##lita":27606,"internacional":27607,"newfound":27608,"##agger":27609,"##lynn":27610,"##rigue":27611,"booths":27612,"peanuts":27613,"##eborg":27614,"medicare":27615,"muriel":27616,"nur":27617,"##uram":27618,"crates":27619,"millennia":27620,"pajamas":27621,"worsened":27622,"##breakers":27623,"jimi":27624,"vanuatu":27625,"yawned":27626,"##udeau":27627,"carousel":27628,"##hony":27629,"hurdle":27630,"##ccus":27631,"##mounted":27632,"##pod":27633,"rv":27634,"##eche":27635,"airship":27636,"ambiguity":27637,"compulsion":27638,"recapture":27639,"##claiming":27640,"arthritis":27641,"##osomal":27642,"1667":27643,"asserting":27644,"ngc":27645,"sniffing":27646,"dade":27647,"discontent":27648,"glendale":27649,"ported":27650,"##amina":27651,"defamation":27652,"rammed":27653,"##scent":27654,"fling":27655,"livingstone":27656,"##fleet":27657,"875":27658,"##ppy":27659,"apocalyptic":27660,"comrade":27661,"lcd":27662,"##lowe":27663,"cessna":27664,"eine":27665,"persecuted":27666,"subsistence":27667,"demi":27668,"hoop":27669,"reliefs":27670,"710":27671,"coptic":27672,"progressing":27673,"stemmed":27674,"perpetrators":27675,"1665":27676,"priestess":27677,"##nio":27678,"dobson":27679,"ebony":27680,"rooster":27681,"itf":27682,"tortricidae":27683,"##bbon":27684,"##jian":27685,"cleanup":27686,"##jean":27687,"##øy":27688,"1721":27689,"eighties":27690,"taxonomic":27691,"holiness":27692,"##hearted":27693,"##spar":27694,"antilles":27695,"showcasing":27696,"stabilized":27697,"##nb":27698,"gia":27699,"mascara":27700,"michelangelo":27701,"dawned":27702,"##uria":27703,"##vinsky":27704,"extinguished":27705,"fitz":27706,"grotesque":27707,"£100":27708,"##fera":27709,"##loid":27710,"##mous":27711,"barges":27712,"neue":27713,"throbbed":27714,"cipher":27715,"johnnie":27716,"##a1":27717,"##mpt":27718,"outburst":27719,"##swick":27720,"spearheaded":27721,"administrations":27722,"c1":27723,"heartbreak":27724,"pixels":27725,"pleasantly":27726,"##enay":27727,"lombardy":27728,"plush":27729,"##nsed":27730,"bobbie":27731,"##hly":27732,"reapers":27733,"tremor":27734,"xiang":27735,"minogue":27736,"substantive":27737,"hitch":27738,"barak":27739,"##wyl":27740,"kwan":27741,"##encia":27742,"910":27743,"obscene":27744,"elegance":27745,"indus":27746,"surfer":27747,"bribery":27748,"conserve":27749,"##hyllum":27750,"##masters":27751,"horatio":27752,"##fat":27753,"apes":27754,"rebound":27755,"psychotic":27756,"##pour":27757,"iteration":27758,"##mium":27759,"##vani":27760,"botanic":27761,"horribly":27762,"antiques":27763,"dispose":27764,"paxton":27765,"##hli":27766,"##wg":27767,"timeless":27768,"1704":27769,"disregard":27770,"engraver":27771,"hounds":27772,"##bau":27773,"##version":27774,"looted":27775,"uno":27776,"facilitates":27777,"groans":27778,"masjid":27779,"rutland":27780,"antibody":27781,"disqualification":27782,"decatur":27783,"footballers":27784,"quake":27785,"slacks":27786,"48th":27787,"rein":27788,"scribe":27789,"stabilize":27790,"commits":27791,"exemplary":27792,"tho":27793,"##hort":27794,"##chison":27795,"pantry":27796,"traversed":27797,"##hiti":27798,"disrepair":27799,"identifiable":27800,"vibrated":27801,"baccalaureate":27802,"##nnis":27803,"csa":27804,"interviewing":27805,"##iensis":27806,"##raße":27807,"greaves":27808,"wealthiest":27809,"343":27810,"classed":27811,"jogged":27812,"£5":27813,"##58":27814,"##atal":27815,"illuminating":27816,"knicks":27817,"respecting":27818,"##uno":27819,"scrubbed":27820,"##iji":27821,"##dles":27822,"kruger":27823,"moods":27824,"growls":27825,"raider":27826,"silvia":27827,"chefs":27828,"kam":27829,"vr":27830,"cree":27831,"percival":27832,"##terol":27833,"gunter":27834,"counterattack":27835,"defiant":27836,"henan":27837,"ze":27838,"##rasia":27839,"##riety":27840,"equivalence":27841,"submissions":27842,"##fra":27843,"##thor":27844,"bautista":27845,"mechanically":27846,"##heater":27847,"cornice":27848,"herbal":27849,"templar":27850,"##mering":27851,"outputs":27852,"ruining":27853,"ligand":27854,"renumbered":27855,"extravagant":27856,"mika":27857,"blockbuster":27858,"eta":27859,"insurrection":27860,"##ilia":27861,"darkening":27862,"ferocious":27863,"pianos":27864,"strife":27865,"kinship":27866,"##aer":27867,"melee":27868,"##anor":27869,"##iste":27870,"##may":27871,"##oue":27872,"decidedly":27873,"weep":27874,"##jad":27875,"##missive":27876,"##ppel":27877,"354":27878,"puget":27879,"unease":27880,"##gnant":27881,"1629":27882,"hammering":27883,"kassel":27884,"ob":27885,"wessex":27886,"##lga":27887,"bromwich":27888,"egan":27889,"paranoia":27890,"utilization":27891,"##atable":27892,"##idad":27893,"contradictory":27894,"provoke":27895,"##ols":27896,"##ouring":27897,"##tangled":27898,"knesset":27899,"##very":27900,"##lette":27901,"plumbing":27902,"##sden":27903,"##¹":27904,"greensboro":27905,"occult":27906,"sniff":27907,"338":27908,"zev":27909,"beaming":27910,"gamer":27911,"haggard":27912,"mahal":27913,"##olt":27914,"##pins":27915,"mendes":27916,"utmost":27917,"briefing":27918,"gunnery":27919,"##gut":27920,"##pher":27921,"##zh":27922,"##rok":27923,"1679":27924,"khalifa":27925,"sonya":27926,"##boot":27927,"principals":27928,"urbana":27929,"wiring":27930,"##liffe":27931,"##minating":27932,"##rrado":27933,"dahl":27934,"nyu":27935,"skepticism":27936,"np":27937,"townspeople":27938,"ithaca":27939,"lobster":27940,"somethin":27941,"##fur":27942,"##arina":27943,"##−1":27944,"freighter":27945,"zimmerman":27946,"biceps":27947,"contractual":27948,"##herton":27949,"amend":27950,"hurrying":27951,"subconscious":27952,"##anal":27953,"336":27954,"meng":27955,"clermont":27956,"spawning":27957,"##eia":27958,"##lub":27959,"dignitaries":27960,"impetus":27961,"snacks":27962,"spotting":27963,"twigs":27964,"##bilis":27965,"##cz":27966,"##ouk":27967,"libertadores":27968,"nic":27969,"skylar":27970,"##aina":27971,"##firm":27972,"gustave":27973,"asean":27974,"##anum":27975,"dieter":27976,"legislatures":27977,"flirt":27978,"bromley":27979,"trolls":27980,"umar":27981,"##bbies":27982,"##tyle":27983,"blah":27984,"parc":27985,"bridgeport":27986,"crank":27987,"negligence":27988,"##nction":27989,"46th":27990,"constantin":27991,"molded":27992,"bandages":27993,"seriousness":27994,"00pm":27995,"siegel":27996,"carpets":27997,"compartments":27998,"upbeat":27999,"statehood":28000,"##dner":28001,"##edging":28002,"marko":28003,"730":28004,"platt":28005,"##hane":28006,"paving":28007,"##iy":28008,"1738":28009,"abbess":28010,"impatience":28011,"limousine":28012,"nbl":28013,"##talk":28014,"441":28015,"lucille":28016,"mojo":28017,"nightfall":28018,"robbers":28019,"##nais":28020,"karel":28021,"brisk":28022,"calves":28023,"replicate":28024,"ascribed":28025,"telescopes":28026,"##olf":28027,"intimidated":28028,"##reen":28029,"ballast":28030,"specialization":28031,"##sit":28032,"aerodynamic":28033,"caliphate":28034,"rainer":28035,"visionary":28036,"##arded":28037,"epsilon":28038,"##aday":28039,"##onte":28040,"aggregation":28041,"auditory":28042,"boosted":28043,"reunification":28044,"kathmandu":28045,"loco":28046,"robyn":28047,"402":28048,"acknowledges":28049,"appointing":28050,"humanoid":28051,"newell":28052,"redeveloped":28053,"restraints":28054,"##tained":28055,"barbarians":28056,"chopper":28057,"1609":28058,"italiana":28059,"##lez":28060,"##lho":28061,"investigates":28062,"wrestlemania":28063,"##anies":28064,"##bib":28065,"690":28066,"##falls":28067,"creaked":28068,"dragoons":28069,"gravely":28070,"minions":28071,"stupidity":28072,"volley":28073,"##harat":28074,"##week":28075,"musik":28076,"##eries":28077,"##uously":28078,"fungal":28079,"massimo":28080,"semantics":28081,"malvern":28082,"##ahl":28083,"##pee":28084,"discourage":28085,"embryo":28086,"imperialism":28087,"1910s":28088,"profoundly":28089,"##ddled":28090,"jiangsu":28091,"sparkled":28092,"stat":28093,"##holz":28094,"sweatshirt":28095,"tobin":28096,"##iction":28097,"sneered":28098,"##cheon":28099,"##oit":28100,"brit":28101,"causal":28102,"smyth":28103,"##neuve":28104,"diffuse":28105,"perrin":28106,"silvio":28107,"##ipes":28108,"##recht":28109,"detonated":28110,"iqbal":28111,"selma":28112,"##nism":28113,"##zumi":28114,"roasted":28115,"##riders":28116,"tay":28117,"##ados":28118,"##mament":28119,"##mut":28120,"##rud":28121,"840":28122,"completes":28123,"nipples":28124,"cfa":28125,"flavour":28126,"hirsch":28127,"##laus":28128,"calderon":28129,"sneakers":28130,"moravian":28131,"##ksha":28132,"1622":28133,"rq":28134,"294":28135,"##imeters":28136,"bodo":28137,"##isance":28138,"##pre":28139,"##ronia":28140,"anatomical":28141,"excerpt":28142,"##lke":28143,"dh":28144,"kunst":28145,"##tablished":28146,"##scoe":28147,"biomass":28148,"panted":28149,"unharmed":28150,"gael":28151,"housemates":28152,"montpellier":28153,"##59":28154,"coa":28155,"rodents":28156,"tonic":28157,"hickory":28158,"singleton":28159,"##taro":28160,"451":28161,"1719":28162,"aldo":28163,"breaststroke":28164,"dempsey":28165,"och":28166,"rocco":28167,"##cuit":28168,"merton":28169,"dissemination":28170,"midsummer":28171,"serials":28172,"##idi":28173,"haji":28174,"polynomials":28175,"##rdon":28176,"gs":28177,"enoch":28178,"prematurely":28179,"shutter":28180,"taunton":28181,"£3":28182,"##grating":28183,"##inates":28184,"archangel":28185,"harassed":28186,"##asco":28187,"326":28188,"archway":28189,"dazzling":28190,"##ecin":28191,"1736":28192,"sumo":28193,"wat":28194,"##kovich":28195,"1086":28196,"honneur":28197,"##ently":28198,"##nostic":28199,"##ttal":28200,"##idon":28201,"1605":28202,"403":28203,"1716":28204,"blogger":28205,"rents":28206,"##gnan":28207,"hires":28208,"##ikh":28209,"##dant":28210,"howie":28211,"##rons":28212,"handler":28213,"retracted":28214,"shocks":28215,"1632":28216,"arun":28217,"duluth":28218,"kepler":28219,"trumpeter":28220,"##lary":28221,"peeking":28222,"seasoned":28223,"trooper":28224,"##mara":28225,"laszlo":28226,"##iciencies":28227,"##rti":28228,"heterosexual":28229,"##inatory":28230,"##ssion":28231,"indira":28232,"jogging":28233,"##inga":28234,"##lism":28235,"beit":28236,"dissatisfaction":28237,"malice":28238,"##ately":28239,"nedra":28240,"peeling":28241,"##rgeon":28242,"47th":28243,"stadiums":28244,"475":28245,"vertigo":28246,"##ains":28247,"iced":28248,"restroom":28249,"##plify":28250,"##tub":28251,"illustrating":28252,"pear":28253,"##chner":28254,"##sibility":28255,"inorganic":28256,"rappers":28257,"receipts":28258,"watery":28259,"##kura":28260,"lucinda":28261,"##oulos":28262,"reintroduced":28263,"##8th":28264,"##tched":28265,"gracefully":28266,"saxons":28267,"nutritional":28268,"wastewater":28269,"rained":28270,"favourites":28271,"bedrock":28272,"fisted":28273,"hallways":28274,"likeness":28275,"upscale":28276,"##lateral":28277,"1580":28278,"blinds":28279,"prequel":28280,"##pps":28281,"##tama":28282,"deter":28283,"humiliating":28284,"restraining":28285,"tn":28286,"vents":28287,"1659":28288,"laundering":28289,"recess":28290,"rosary":28291,"tractors":28292,"coulter":28293,"federer":28294,"##ifiers":28295,"##plin":28296,"persistence":28297,"##quitable":28298,"geschichte":28299,"pendulum":28300,"quakers":28301,"##beam":28302,"bassett":28303,"pictorial":28304,"buffet":28305,"koln":28306,"##sitor":28307,"drills":28308,"reciprocal":28309,"shooters":28310,"##57":28311,"##cton":28312,"##tees":28313,"converge":28314,"pip":28315,"dmitri":28316,"donnelly":28317,"yamamoto":28318,"aqua":28319,"azores":28320,"demographics":28321,"hypnotic":28322,"spitfire":28323,"suspend":28324,"wryly":28325,"roderick":28326,"##rran":28327,"sebastien":28328,"##asurable":28329,"mavericks":28330,"##fles":28331,"##200":28332,"himalayan":28333,"prodigy":28334,"##iance":28335,"transvaal":28336,"demonstrators":28337,"handcuffs":28338,"dodged":28339,"mcnamara":28340,"sublime":28341,"1726":28342,"crazed":28343,"##efined":28344,"##till":28345,"ivo":28346,"pondered":28347,"reconciled":28348,"shrill":28349,"sava":28350,"##duk":28351,"bal":28352,"cad":28353,"heresy":28354,"jaipur":28355,"goran":28356,"##nished":28357,"341":28358,"lux":28359,"shelly":28360,"whitehall":28361,"##hre":28362,"israelis":28363,"peacekeeping":28364,"##wled":28365,"1703":28366,"demetrius":28367,"ousted":28368,"##arians":28369,"##zos":28370,"beale":28371,"anwar":28372,"backstroke":28373,"raged":28374,"shrinking":28375,"cremated":28376,"##yck":28377,"benign":28378,"towing":28379,"wadi":28380,"darmstadt":28381,"landfill":28382,"parana":28383,"soothe":28384,"colleen":28385,"sidewalks":28386,"mayfair":28387,"tumble":28388,"hepatitis":28389,"ferrer":28390,"superstructure":28391,"##gingly":28392,"##urse":28393,"##wee":28394,"anthropological":28395,"translators":28396,"##mies":28397,"closeness":28398,"hooves":28399,"##pw":28400,"mondays":28401,"##roll":28402,"##vita":28403,"landscaping":28404,"##urized":28405,"purification":28406,"sock":28407,"thorns":28408,"thwarted":28409,"jalan":28410,"tiberius":28411,"##taka":28412,"saline":28413,"##rito":28414,"confidently":28415,"khyber":28416,"sculptors":28417,"##ij":28418,"brahms":28419,"hammersmith":28420,"inspectors":28421,"battista":28422,"fivb":28423,"fragmentation":28424,"hackney":28425,"##uls":28426,"arresting":28427,"exercising":28428,"antoinette":28429,"bedfordshire":28430,"##zily":28431,"dyed":28432,"##hema":28433,"1656":28434,"racetrack":28435,"variability":28436,"##tique":28437,"1655":28438,"austrians":28439,"deteriorating":28440,"madman":28441,"theorists":28442,"aix":28443,"lehman":28444,"weathered":28445,"1731":28446,"decreed":28447,"eruptions":28448,"1729":28449,"flaw":28450,"quinlan":28451,"sorbonne":28452,"flutes":28453,"nunez":28454,"1711":28455,"adored":28456,"downwards":28457,"fable":28458,"rasped":28459,"1712":28460,"moritz":28461,"mouthful":28462,"renegade":28463,"shivers":28464,"stunts":28465,"dysfunction":28466,"restrain":28467,"translit":28468,"327":28469,"pancakes":28470,"##avio":28471,"##cision":28472,"##tray":28473,"351":28474,"vial":28475,"##lden":28476,"bain":28477,"##maid":28478,"##oxide":28479,"chihuahua":28480,"malacca":28481,"vimes":28482,"##rba":28483,"##rnier":28484,"1664":28485,"donnie":28486,"plaques":28487,"##ually":28488,"337":28489,"bangs":28490,"floppy":28491,"huntsville":28492,"loretta":28493,"nikolay":28494,"##otte":28495,"eater":28496,"handgun":28497,"ubiquitous":28498,"##hett":28499,"eras":28500,"zodiac":28501,"1634":28502,"##omorphic":28503,"1820s":28504,"##zog":28505,"cochran":28506,"##bula":28507,"##lithic":28508,"warring":28509,"##rada":28510,"dalai":28511,"excused":28512,"blazers":28513,"mcconnell":28514,"reeling":28515,"bot":28516,"este":28517,"##abi":28518,"geese":28519,"hoax":28520,"taxon":28521,"##bla":28522,"guitarists":28523,"##icon":28524,"condemning":28525,"hunts":28526,"inversion":28527,"moffat":28528,"taekwondo":28529,"##lvis":28530,"1624":28531,"stammered":28532,"##rest":28533,"##rzy":28534,"sousa":28535,"fundraiser":28536,"marylebone":28537,"navigable":28538,"uptown":28539,"cabbage":28540,"daniela":28541,"salman":28542,"shitty":28543,"whimper":28544,"##kian":28545,"##utive":28546,"programmers":28547,"protections":28548,"rm":28549,"##rmi":28550,"##rued":28551,"forceful":28552,"##enes":28553,"fuss":28554,"##tao":28555,"##wash":28556,"brat":28557,"oppressive":28558,"reykjavik":28559,"spartak":28560,"ticking":28561,"##inkles":28562,"##kiewicz":28563,"adolph":28564,"horst":28565,"maui":28566,"protege":28567,"straighten":28568,"cpc":28569,"landau":28570,"concourse":28571,"clements":28572,"resultant":28573,"##ando":28574,"imaginative":28575,"joo":28576,"reactivated":28577,"##rem":28578,"##ffled":28579,"##uising":28580,"consultative":28581,"##guide":28582,"flop":28583,"kaitlyn":28584,"mergers":28585,"parenting":28586,"somber":28587,"##vron":28588,"supervise":28589,"vidhan":28590,"##imum":28591,"courtship":28592,"exemplified":28593,"harmonies":28594,"medallist":28595,"refining":28596,"##rrow":28597,"##ка":28598,"amara":28599,"##hum":28600,"780":28601,"goalscorer":28602,"sited":28603,"overshadowed":28604,"rohan":28605,"displeasure":28606,"secretive":28607,"multiplied":28608,"osman":28609,"##orth":28610,"engravings":28611,"padre":28612,"##kali":28613,"##veda":28614,"miniatures":28615,"mis":28616,"##yala":28617,"clap":28618,"pali":28619,"rook":28620,"##cana":28621,"1692":28622,"57th":28623,"antennae":28624,"astro":28625,"oskar":28626,"1628":28627,"bulldog":28628,"crotch":28629,"hackett":28630,"yucatan":28631,"##sure":28632,"amplifiers":28633,"brno":28634,"ferrara":28635,"migrating":28636,"##gree":28637,"thanking":28638,"turing":28639,"##eza":28640,"mccann":28641,"ting":28642,"andersson":28643,"onslaught":28644,"gaines":28645,"ganga":28646,"incense":28647,"standardization":28648,"##mation":28649,"sentai":28650,"scuba":28651,"stuffing":28652,"turquoise":28653,"waivers":28654,"alloys":28655,"##vitt":28656,"regaining":28657,"vaults":28658,"##clops":28659,"##gizing":28660,"digger":28661,"furry":28662,"memorabilia":28663,"probing":28664,"##iad":28665,"payton":28666,"rec":28667,"deutschland":28668,"filippo":28669,"opaque":28670,"seamen":28671,"zenith":28672,"afrikaans":28673,"##filtration":28674,"disciplined":28675,"inspirational":28676,"##merie":28677,"banco":28678,"confuse":28679,"grafton":28680,"tod":28681,"##dgets":28682,"championed":28683,"simi":28684,"anomaly":28685,"biplane":28686,"##ceptive":28687,"electrode":28688,"##para":28689,"1697":28690,"cleavage":28691,"crossbow":28692,"swirl":28693,"informant":28694,"##lars":28695,"##osta":28696,"afi":28697,"bonfire":28698,"spec":28699,"##oux":28700,"lakeside":28701,"slump":28702,"##culus":28703,"##lais":28704,"##qvist":28705,"##rrigan":28706,"1016":28707,"facades":28708,"borg":28709,"inwardly":28710,"cervical":28711,"xl":28712,"pointedly":28713,"050":28714,"stabilization":28715,"##odon":28716,"chests":28717,"1699":28718,"hacked":28719,"ctv":28720,"orthogonal":28721,"suzy":28722,"##lastic":28723,"gaulle":28724,"jacobite":28725,"rearview":28726,"##cam":28727,"##erted":28728,"ashby":28729,"##drik":28730,"##igate":28731,"##mise":28732,"##zbek":28733,"affectionately":28734,"canine":28735,"disperse":28736,"latham":28737,"##istles":28738,"##ivar":28739,"spielberg":28740,"##orin":28741,"##idium":28742,"ezekiel":28743,"cid":28744,"##sg":28745,"durga":28746,"middletown":28747,"##cina":28748,"customized":28749,"frontiers":28750,"harden":28751,"##etano":28752,"##zzy":28753,"1604":28754,"bolsheviks":28755,"##66":28756,"coloration":28757,"yoko":28758,"##bedo":28759,"briefs":28760,"slabs":28761,"debra":28762,"liquidation":28763,"plumage":28764,"##oin":28765,"blossoms":28766,"dementia":28767,"subsidy":28768,"1611":28769,"proctor":28770,"relational":28771,"jerseys":28772,"parochial":28773,"ter":28774,"##ici":28775,"esa":28776,"peshawar":28777,"cavalier":28778,"loren":28779,"cpi":28780,"idiots":28781,"shamrock":28782,"1646":28783,"dutton":28784,"malabar":28785,"mustache":28786,"##endez":28787,"##ocytes":28788,"referencing":28789,"terminates":28790,"marche":28791,"yarmouth":28792,"##sop":28793,"acton":28794,"mated":28795,"seton":28796,"subtly":28797,"baptised":28798,"beige":28799,"extremes":28800,"jolted":28801,"kristina":28802,"telecast":28803,"##actic":28804,"safeguard":28805,"waldo":28806,"##baldi":28807,"##bular":28808,"endeavors":28809,"sloppy":28810,"subterranean":28811,"##ensburg":28812,"##itung":28813,"delicately":28814,"pigment":28815,"tq":28816,"##scu":28817,"1626":28818,"##ound":28819,"collisions":28820,"coveted":28821,"herds":28822,"##personal":28823,"##meister":28824,"##nberger":28825,"chopra":28826,"##ricting":28827,"abnormalities":28828,"defective":28829,"galician":28830,"lucie":28831,"##dilly":28832,"alligator":28833,"likened":28834,"##genase":28835,"burundi":28836,"clears":28837,"complexion":28838,"derelict":28839,"deafening":28840,"diablo":28841,"fingered":28842,"champaign":28843,"dogg":28844,"enlist":28845,"isotope":28846,"labeling":28847,"mrna":28848,"##erre":28849,"brilliance":28850,"marvelous":28851,"##ayo":28852,"1652":28853,"crawley":28854,"ether":28855,"footed":28856,"dwellers":28857,"deserts":28858,"hamish":28859,"rubs":28860,"warlock":28861,"skimmed":28862,"##lizer":28863,"870":28864,"buick":28865,"embark":28866,"heraldic":28867,"irregularities":28868,"##ajan":28869,"kiara":28870,"##kulam":28871,"##ieg":28872,"antigen":28873,"kowalski":28874,"##lge":28875,"oakley":28876,"visitation":28877,"##mbit":28878,"vt":28879,"##suit":28880,"1570":28881,"murderers":28882,"##miento":28883,"##rites":28884,"chimneys":28885,"##sling":28886,"condemn":28887,"custer":28888,"exchequer":28889,"havre":28890,"##ghi":28891,"fluctuations":28892,"##rations":28893,"dfb":28894,"hendricks":28895,"vaccines":28896,"##tarian":28897,"nietzsche":28898,"biking":28899,"juicy":28900,"##duced":28901,"brooding":28902,"scrolling":28903,"selangor":28904,"##ragan":28905,"352":28906,"annum":28907,"boomed":28908,"seminole":28909,"sugarcane":28910,"##dna":28911,"departmental":28912,"dismissing":28913,"innsbruck":28914,"arteries":28915,"ashok":28916,"batavia":28917,"daze":28918,"kun":28919,"overtook":28920,"##rga":28921,"##tlan":28922,"beheaded":28923,"gaddafi":28924,"holm":28925,"electronically":28926,"faulty":28927,"galilee":28928,"fractures":28929,"kobayashi":28930,"##lized":28931,"gunmen":28932,"magma":28933,"aramaic":28934,"mala":28935,"eastenders":28936,"inference":28937,"messengers":28938,"bf":28939,"##qu":28940,"407":28941,"bathrooms":28942,"##vere":28943,"1658":28944,"flashbacks":28945,"ideally":28946,"misunderstood":28947,"##jali":28948,"##weather":28949,"mendez":28950,"##grounds":28951,"505":28952,"uncanny":28953,"##iii":28954,"1709":28955,"friendships":28956,"##nbc":28957,"sacrament":28958,"accommodated":28959,"reiterated":28960,"logistical":28961,"pebbles":28962,"thumped":28963,"##escence":28964,"administering":28965,"decrees":28966,"drafts":28967,"##flight":28968,"##cased":28969,"##tula":28970,"futuristic":28971,"picket":28972,"intimidation":28973,"winthrop":28974,"##fahan":28975,"interfered":28976,"339":28977,"afar":28978,"francoise":28979,"morally":28980,"uta":28981,"cochin":28982,"croft":28983,"dwarfs":28984,"##bruck":28985,"##dents":28986,"##nami":28987,"biker":28988,"##hner":28989,"##meral":28990,"nano":28991,"##isen":28992,"##ometric":28993,"##pres":28994,"##ан":28995,"brightened":28996,"meek":28997,"parcels":28998,"securely":28999,"gunners":29000,"##jhl":29001,"##zko":29002,"agile":29003,"hysteria":29004,"##lten":29005,"##rcus":29006,"bukit":29007,"champs":29008,"chevy":29009,"cuckoo":29010,"leith":29011,"sadler":29012,"theologians":29013,"welded":29014,"##section":29015,"1663":29016,"jj":29017,"plurality":29018,"xander":29019,"##rooms":29020,"##formed":29021,"shredded":29022,"temps":29023,"intimately":29024,"pau":29025,"tormented":29026,"##lok":29027,"##stellar":29028,"1618":29029,"charred":29030,"ems":29031,"essen":29032,"##mmel":29033,"alarms":29034,"spraying":29035,"ascot":29036,"blooms":29037,"twinkle":29038,"##abia":29039,"##apes":29040,"internment":29041,"obsidian":29042,"##chaft":29043,"snoop":29044,"##dav":29045,"##ooping":29046,"malibu":29047,"##tension":29048,"quiver":29049,"##itia":29050,"hays":29051,"mcintosh":29052,"travers":29053,"walsall":29054,"##ffie":29055,"1623":29056,"beverley":29057,"schwarz":29058,"plunging":29059,"structurally":29060,"m3":29061,"rosenthal":29062,"vikram":29063,"##tsk":29064,"770":29065,"ghz":29066,"##onda":29067,"##tiv":29068,"chalmers":29069,"groningen":29070,"pew":29071,"reckon":29072,"unicef":29073,"##rvis":29074,"55th":29075,"##gni":29076,"1651":29077,"sulawesi":29078,"avila":29079,"cai":29080,"metaphysical":29081,"screwing":29082,"turbulence":29083,"##mberg":29084,"augusto":29085,"samba":29086,"56th":29087,"baffled":29088,"momentary":29089,"toxin":29090,"##urian":29091,"##wani":29092,"aachen":29093,"condoms":29094,"dali":29095,"steppe":29096,"##3d":29097,"##app":29098,"##oed":29099,"##year":29100,"adolescence":29101,"dauphin":29102,"electrically":29103,"inaccessible":29104,"microscopy":29105,"nikita":29106,"##ega":29107,"atv":29108,"##cel":29109,"##enter":29110,"##oles":29111,"##oteric":29112,"##ы":29113,"accountants":29114,"punishments":29115,"wrongly":29116,"bribes":29117,"adventurous":29118,"clinch":29119,"flinders":29120,"southland":29121,"##hem":29122,"##kata":29123,"gough":29124,"##ciency":29125,"lads":29126,"soared":29127,"##ה":29128,"undergoes":29129,"deformation":29130,"outlawed":29131,"rubbish":29132,"##arus":29133,"##mussen":29134,"##nidae":29135,"##rzburg":29136,"arcs":29137,"##ingdon":29138,"##tituted":29139,"1695":29140,"wheelbase":29141,"wheeling":29142,"bombardier":29143,"campground":29144,"zebra":29145,"##lices":29146,"##oj":29147,"##bain":29148,"lullaby":29149,"##ecure":29150,"donetsk":29151,"wylie":29152,"grenada":29153,"##arding":29154,"##ης":29155,"squinting":29156,"eireann":29157,"opposes":29158,"##andra":29159,"maximal":29160,"runes":29161,"##broken":29162,"##cuting":29163,"##iface":29164,"##ror":29165,"##rosis":29166,"additive":29167,"britney":29168,"adultery":29169,"triggering":29170,"##drome":29171,"detrimental":29172,"aarhus":29173,"containment":29174,"jc":29175,"swapped":29176,"vichy":29177,"##ioms":29178,"madly":29179,"##oric":29180,"##rag":29181,"brant":29182,"##ckey":29183,"##trix":29184,"1560":29185,"1612":29186,"broughton":29187,"rustling":29188,"##stems":29189,"##uder":29190,"asbestos":29191,"mentoring":29192,"##nivorous":29193,"finley":29194,"leaps":29195,"##isan":29196,"apical":29197,"pry":29198,"slits":29199,"substitutes":29200,"##dict":29201,"intuitive":29202,"fantasia":29203,"insistent":29204,"unreasonable":29205,"##igen":29206,"##vna":29207,"domed":29208,"hannover":29209,"margot":29210,"ponder":29211,"##zziness":29212,"impromptu":29213,"jian":29214,"lc":29215,"rampage":29216,"stemming":29217,"##eft":29218,"andrey":29219,"gerais":29220,"whichever":29221,"amnesia":29222,"appropriated":29223,"anzac":29224,"clicks":29225,"modifying":29226,"ultimatum":29227,"cambrian":29228,"maids":29229,"verve":29230,"yellowstone":29231,"##mbs":29232,"conservatoire":29233,"##scribe":29234,"adherence":29235,"dinners":29236,"spectra":29237,"imperfect":29238,"mysteriously":29239,"sidekick":29240,"tatar":29241,"tuba":29242,"##aks":29243,"##ifolia":29244,"distrust":29245,"##athan":29246,"##zle":29247,"c2":29248,"ronin":29249,"zac":29250,"##pse":29251,"celaena":29252,"instrumentalist":29253,"scents":29254,"skopje":29255,"##mbling":29256,"comical":29257,"compensated":29258,"vidal":29259,"condor":29260,"intersect":29261,"jingle":29262,"wavelengths":29263,"##urrent":29264,"mcqueen":29265,"##izzly":29266,"carp":29267,"weasel":29268,"422":29269,"kanye":29270,"militias":29271,"postdoctoral":29272,"eugen":29273,"gunslinger":29274,"##ɛ":29275,"faux":29276,"hospice":29277,"##for":29278,"appalled":29279,"derivation":29280,"dwarves":29281,"##elis":29282,"dilapidated":29283,"##folk":29284,"astoria":29285,"philology":29286,"##lwyn":29287,"##otho":29288,"##saka":29289,"inducing":29290,"philanthropy":29291,"##bf":29292,"##itative":29293,"geek":29294,"markedly":29295,"sql":29296,"##yce":29297,"bessie":29298,"indices":29299,"rn":29300,"##flict":29301,"495":29302,"frowns":29303,"resolving":29304,"weightlifting":29305,"tugs":29306,"cleric":29307,"contentious":29308,"1653":29309,"mania":29310,"rms":29311,"##miya":29312,"##reate":29313,"##ruck":29314,"##tucket":29315,"bien":29316,"eels":29317,"marek":29318,"##ayton":29319,"##cence":29320,"discreet":29321,"unofficially":29322,"##ife":29323,"leaks":29324,"##bber":29325,"1705":29326,"332":29327,"dung":29328,"compressor":29329,"hillsborough":29330,"pandit":29331,"shillings":29332,"distal":29333,"##skin":29334,"381":29335,"##tat":29336,"##you":29337,"nosed":29338,"##nir":29339,"mangrove":29340,"undeveloped":29341,"##idia":29342,"textures":29343,"##inho":29344,"##500":29345,"##rise":29346,"ae":29347,"irritating":29348,"nay":29349,"amazingly":29350,"bancroft":29351,"apologetic":29352,"compassionate":29353,"kata":29354,"symphonies":29355,"##lovic":29356,"airspace":29357,"##lch":29358,"930":29359,"gifford":29360,"precautions":29361,"fulfillment":29362,"sevilla":29363,"vulgar":29364,"martinique":29365,"##urities":29366,"looting":29367,"piccolo":29368,"tidy":29369,"##dermott":29370,"quadrant":29371,"armchair":29372,"incomes":29373,"mathematicians":29374,"stampede":29375,"nilsson":29376,"##inking":29377,"##scan":29378,"foo":29379,"quarterfinal":29380,"##ostal":29381,"shang":29382,"shouldered":29383,"squirrels":29384,"##owe":29385,"344":29386,"vinegar":29387,"##bner":29388,"##rchy":29389,"##systems":29390,"delaying":29391,"##trics":29392,"ars":29393,"dwyer":29394,"rhapsody":29395,"sponsoring":29396,"##gration":29397,"bipolar":29398,"cinder":29399,"starters":29400,"##olio":29401,"##urst":29402,"421":29403,"signage":29404,"##nty":29405,"aground":29406,"figurative":29407,"mons":29408,"acquaintances":29409,"duets":29410,"erroneously":29411,"soyuz":29412,"elliptic":29413,"recreated":29414,"##cultural":29415,"##quette":29416,"##ssed":29417,"##tma":29418,"##zcz":29419,"moderator":29420,"scares":29421,"##itaire":29422,"##stones":29423,"##udence":29424,"juniper":29425,"sighting":29426,"##just":29427,"##nsen":29428,"britten":29429,"calabria":29430,"ry":29431,"bop":29432,"cramer":29433,"forsyth":29434,"stillness":29435,"##л":29436,"airmen":29437,"gathers":29438,"unfit":29439,"##umber":29440,"##upt":29441,"taunting":29442,"##rip":29443,"seeker":29444,"streamlined":29445,"##bution":29446,"holster":29447,"schumann":29448,"tread":29449,"vox":29450,"##gano":29451,"##onzo":29452,"strive":29453,"dil":29454,"reforming":29455,"covent":29456,"newbury":29457,"predicting":29458,"##orro":29459,"decorate":29460,"tre":29461,"##puted":29462,"andover":29463,"ie":29464,"asahi":29465,"dept":29466,"dunkirk":29467,"gills":29468,"##tori":29469,"buren":29470,"huskies":29471,"##stis":29472,"##stov":29473,"abstracts":29474,"bets":29475,"loosen":29476,"##opa":29477,"1682":29478,"yearning":29479,"##glio":29480,"##sir":29481,"berman":29482,"effortlessly":29483,"enamel":29484,"napoli":29485,"persist":29486,"##peration":29487,"##uez":29488,"attache":29489,"elisa":29490,"b1":29491,"invitations":29492,"##kic":29493,"accelerating":29494,"reindeer":29495,"boardwalk":29496,"clutches":29497,"nelly":29498,"polka":29499,"starbucks":29500,"##kei":29501,"adamant":29502,"huey":29503,"lough":29504,"unbroken":29505,"adventurer":29506,"embroidery":29507,"inspecting":29508,"stanza":29509,"##ducted":29510,"naia":29511,"taluka":29512,"##pone":29513,"##roids":29514,"chases":29515,"deprivation":29516,"florian":29517,"##jing":29518,"##ppet":29519,"earthly":29520,"##lib":29521,"##ssee":29522,"colossal":29523,"foreigner":29524,"vet":29525,"freaks":29526,"patrice":29527,"rosewood":29528,"triassic":29529,"upstate":29530,"##pkins":29531,"dominates":29532,"ata":29533,"chants":29534,"ks":29535,"vo":29536,"##400":29537,"##bley":29538,"##raya":29539,"##rmed":29540,"555":29541,"agra":29542,"infiltrate":29543,"##ailing":29544,"##ilation":29545,"##tzer":29546,"##uppe":29547,"##werk":29548,"binoculars":29549,"enthusiast":29550,"fujian":29551,"squeak":29552,"##avs":29553,"abolitionist":29554,"almeida":29555,"boredom":29556,"hampstead":29557,"marsden":29558,"rations":29559,"##ands":29560,"inflated":29561,"334":29562,"bonuses":29563,"rosalie":29564,"patna":29565,"##rco":29566,"329":29567,"detachments":29568,"penitentiary":29569,"54th":29570,"flourishing":29571,"woolf":29572,"##dion":29573,"##etched":29574,"papyrus":29575,"##lster":29576,"##nsor":29577,"##toy":29578,"bobbed":29579,"dismounted":29580,"endelle":29581,"inhuman":29582,"motorola":29583,"tbs":29584,"wince":29585,"wreath":29586,"##ticus":29587,"hideout":29588,"inspections":29589,"sanjay":29590,"disgrace":29591,"infused":29592,"pudding":29593,"stalks":29594,"##urbed":29595,"arsenic":29596,"leases":29597,"##hyl":29598,"##rrard":29599,"collarbone":29600,"##waite":29601,"##wil":29602,"dowry":29603,"##bant":29604,"##edance":29605,"genealogical":29606,"nitrate":29607,"salamanca":29608,"scandals":29609,"thyroid":29610,"necessitated":29611,"##!":29612,"##\"":29613,"###":29614,"##$":29615,"##%":29616,"##&":29617,"##'":29618,"##(":29619,"##)":29620,"##*":29621,"##+":29622,"##,":29623,"##-":29624,"##.":29625,"##/":29626,"##:":29627,"##;":29628,"##<":29629,"##=":29630,"##>":29631,"##?":29632,"##@":29633,"##[":29634,"##\\":29635,"##]":29636,"##^":29637,"##_":29638,"##`":29639,"##{":29640,"##|":29641,"##}":29642,"##~":29643,"##¡":29644,"##¢":29645,"##£":29646,"##¤":29647,"##¥":29648,"##¦":29649,"##§":29650,"##¨":29651,"##©":29652,"##ª":29653,"##«":29654,"##¬":29655,"##®":29656,"##±":29657,"##´":29658,"##µ":29659,"##¶":29660,"##·":29661,"##º":29662,"##»":29663,"##¼":29664,"##¾":29665,"##¿":29666,"##æ":29667,"##ð":29668,"##÷":29669,"##þ":29670,"##đ":29671,"##ħ":29672,"##ŋ":29673,"##œ":29674,"##ƒ":29675,"##ɐ":29676,"##ɑ":29677,"##ɒ":29678,"##ɔ":29679,"##ɕ":29680,"##ə":29681,"##ɡ":29682,"##ɣ":29683,"##ɨ":29684,"##ɪ":29685,"##ɫ":29686,"##ɬ":29687,"##ɯ":29688,"##ɲ":29689,"##ɴ":29690,"##ɹ":29691,"##ɾ":29692,"##ʀ":29693,"##ʁ":29694,"##ʂ":29695,"##ʃ":29696,"##ʉ":29697,"##ʊ":29698,"##ʋ":29699,"##ʌ":29700,"##ʎ":29701,"##ʐ":29702,"##ʑ":29703,"##ʒ":29704,"##ʔ":29705,"##ʰ":29706,"##ʲ":29707,"##ʳ":29708,"##ʷ":29709,"##ʸ":29710,"##ʻ":29711,"##ʼ":29712,"##ʾ":29713,"##ʿ":29714,"##ˈ":29715,"##ˡ":29716,"##ˢ":29717,"##ˣ":29718,"##ˤ":29719,"##β":29720,"##γ":29721,"##δ":29722,"##ε":29723,"##ζ":29724,"##θ":29725,"##κ":29726,"##λ":29727,"##μ":29728,"##ξ":29729,"##ο":29730,"##π":29731,"##ρ":29732,"##σ":29733,"##τ":29734,"##υ":29735,"##φ":29736,"##χ":29737,"##ψ":29738,"##ω":29739,"##б":29740,"##г":29741,"##д":29742,"##ж":29743,"##з":29744,"##м":29745,"##п":29746,"##с":29747,"##у":29748,"##ф":29749,"##х":29750,"##ц":29751,"##ч":29752,"##ш":29753,"##щ":29754,"##ъ":29755,"##э":29756,"##ю":29757,"##ђ":29758,"##є":29759,"##і":29760,"##ј":29761,"##љ":29762,"##њ":29763,"##ћ":29764,"##ӏ":29765,"##ա":29766,"##բ":29767,"##գ":29768,"##դ":29769,"##ե":29770,"##թ":29771,"##ի":29772,"##լ":29773,"##կ":29774,"##հ":29775,"##մ":29776,"##յ":29777,"##ն":29778,"##ո":29779,"##պ":29780,"##ս":29781,"##վ":29782,"##տ":29783,"##ր":29784,"##ւ":29785,"##ք":29786,"##־":29787,"##א":29788,"##ב":29789,"##ג":29790,"##ד":29791,"##ו":29792,"##ז":29793,"##ח":29794,"##ט":29795,"##י":29796,"##ך":29797,"##כ":29798,"##ל":29799,"##ם":29800,"##מ":29801,"##ן":29802,"##נ":29803,"##ס":29804,"##ע":29805,"##ף":29806,"##פ":29807,"##ץ":29808,"##צ":29809,"##ק":29810,"##ר":29811,"##ש":29812,"##ת":29813,"##،":29814,"##ء":29815,"##ب":29816,"##ت":29817,"##ث":29818,"##ج":29819,"##ح":29820,"##خ":29821,"##ذ":29822,"##ز":29823,"##س":29824,"##ش":29825,"##ص":29826,"##ض":29827,"##ط":29828,"##ظ":29829,"##ع":29830,"##غ":29831,"##ـ":29832,"##ف":29833,"##ق":29834,"##ك":29835,"##و":29836,"##ى":29837,"##ٹ":29838,"##پ":29839,"##چ":29840,"##ک":29841,"##گ":29842,"##ں":29843,"##ھ":29844,"##ہ":29845,"##ے":29846,"##अ":29847,"##आ":29848,"##उ":29849,"##ए":29850,"##क":29851,"##ख":29852,"##ग":29853,"##च":29854,"##ज":29855,"##ट":29856,"##ड":29857,"##ण":29858,"##त":29859,"##थ":29860,"##द":29861,"##ध":29862,"##न":29863,"##प":29864,"##ब":29865,"##भ":29866,"##म":29867,"##य":29868,"##र":29869,"##ल":29870,"##व":29871,"##श":29872,"##ष":29873,"##स":29874,"##ह":29875,"##ा":29876,"##ि":29877,"##ी":29878,"##ो":29879,"##।":29880,"##॥":29881,"##ং":29882,"##অ":29883,"##আ":29884,"##ই":29885,"##উ":29886,"##এ":29887,"##ও":29888,"##ক":29889,"##খ":29890,"##গ":29891,"##চ":29892,"##ছ":29893,"##জ":29894,"##ট":29895,"##ড":29896,"##ণ":29897,"##ত":29898,"##থ":29899,"##দ":29900,"##ধ":29901,"##ন":29902,"##প":29903,"##ব":29904,"##ভ":29905,"##ম":29906,"##য":29907,"##র":29908,"##ল":29909,"##শ":29910,"##ষ":29911,"##স":29912,"##হ":29913,"##া":29914,"##ি":29915,"##ী":29916,"##ে":29917,"##க":29918,"##ச":29919,"##ட":29920,"##த":29921,"##ந":29922,"##ன":29923,"##ப":29924,"##ம":29925,"##ய":29926,"##ர":29927,"##ல":29928,"##ள":29929,"##வ":29930,"##ா":29931,"##ி":29932,"##ு":29933,"##ே":29934,"##ை":29935,"##ನ":29936,"##ರ":29937,"##ಾ":29938,"##ක":29939,"##ය":29940,"##ර":29941,"##ල":29942,"##ව":29943,"##ා":29944,"##ก":29945,"##ง":29946,"##ต":29947,"##ท":29948,"##น":29949,"##พ":29950,"##ม":29951,"##ย":29952,"##ร":29953,"##ล":29954,"##ว":29955,"##ส":29956,"##อ":29957,"##า":29958,"##เ":29959,"##་":29960,"##།":29961,"##ག":29962,"##ང":29963,"##ད":29964,"##ན":29965,"##པ":29966,"##བ":29967,"##མ":29968,"##འ":29969,"##ར":29970,"##ལ":29971,"##ས":29972,"##မ":29973,"##ა":29974,"##ბ":29975,"##გ":29976,"##დ":29977,"##ე":29978,"##ვ":29979,"##თ":29980,"##ი":29981,"##კ":29982,"##ლ":29983,"##მ":29984,"##ნ":29985,"##ო":29986,"##რ":29987,"##ს":29988,"##ტ":29989,"##უ":29990,"##ᄀ":29991,"##ᄂ":29992,"##ᄃ":29993,"##ᄅ":29994,"##ᄆ":29995,"##ᄇ":29996,"##ᄉ":29997,"##ᄊ":29998,"##ᄋ":29999,"##ᄌ":30000,"##ᄎ":30001,"##ᄏ":30002,"##ᄐ":30003,"##ᄑ":30004,"##ᄒ":30005,"##ᅡ":30006,"##ᅢ":30007,"##ᅥ":30008,"##ᅦ":30009,"##ᅧ":30010,"##ᅩ":30011,"##ᅪ":30012,"##ᅭ":30013,"##ᅮ":30014,"##ᅯ":30015,"##ᅲ":30016,"##ᅳ":30017,"##ᅴ":30018,"##ᅵ":30019,"##ᆨ":30020,"##ᆫ":30021,"##ᆯ":30022,"##ᆷ":30023,"##ᆸ":30024,"##ᆼ":30025,"##ᴬ":30026,"##ᴮ":30027,"##ᴰ":30028,"##ᴵ":30029,"##ᴺ":30030,"##ᵀ":30031,"##ᵃ":30032,"##ᵇ":30033,"##ᵈ":30034,"##ᵉ":30035,"##ᵍ":30036,"##ᵏ":30037,"##ᵐ":30038,"##ᵒ":30039,"##ᵖ":30040,"##ᵗ":30041,"##ᵘ":30042,"##ᵣ":30043,"##ᵤ":30044,"##ᵥ":30045,"##ᶜ":30046,"##ᶠ":30047,"##‐":30048,"##‑":30049,"##‒":30050,"##–":30051,"##—":30052,"##―":30053,"##‖":30054,"##‘":30055,"##’":30056,"##‚":30057,"##“":30058,"##”":30059,"##„":30060,"##†":30061,"##‡":30062,"##•":30063,"##…":30064,"##‰":30065,"##′":30066,"##″":30067,"##›":30068,"##‿":30069,"##⁄":30070,"##⁰":30071,"##ⁱ":30072,"##⁴":30073,"##⁵":30074,"##⁶":30075,"##⁷":30076,"##⁸":30077,"##⁹":30078,"##⁻":30079,"##ⁿ":30080,"##₅":30081,"##₆":30082,"##₇":30083,"##₈":30084,"##₉":30085,"##₊":30086,"##₍":30087,"##₎":30088,"##ₐ":30089,"##ₑ":30090,"##ₒ":30091,"##ₓ":30092,"##ₕ":30093,"##ₖ":30094,"##ₗ":30095,"##ₘ":30096,"##ₚ":30097,"##ₛ":30098,"##ₜ":30099,"##₤":30100,"##₩":30101,"##€":30102,"##₱":30103,"##₹":30104,"##ℓ":30105,"##№":30106,"##ℝ":30107,"##™":30108,"##⅓":30109,"##⅔":30110,"##←":30111,"##↑":30112,"##→":30113,"##↓":30114,"##↔":30115,"##↦":30116,"##⇄":30117,"##⇌":30118,"##⇒":30119,"##∂":30120,"##∅":30121,"##∆":30122,"##∇":30123,"##∈":30124,"##∗":30125,"##∘":30126,"##√":30127,"##∞":30128,"##∧":30129,"##∨":30130,"##∩":30131,"##∪":30132,"##≈":30133,"##≡":30134,"##≤":30135,"##≥":30136,"##⊂":30137,"##⊆":30138,"##⊕":30139,"##⊗":30140,"##⋅":30141,"##─":30142,"##│":30143,"##■":30144,"##▪":30145,"##●":30146,"##★":30147,"##☆":30148,"##☉":30149,"##♠":30150,"##♣":30151,"##♥":30152,"##♦":30153,"##♯":30154,"##⟨":30155,"##⟩":30156,"##ⱼ":30157,"##⺩":30158,"##⺼":30159,"##⽥":30160,"##、":30161,"##。":30162,"##〈":30163,"##〉":30164,"##《":30165,"##》":30166,"##「":30167,"##」":30168,"##『":30169,"##』":30170,"##〜":30171,"##あ":30172,"##い":30173,"##う":30174,"##え":30175,"##お":30176,"##か":30177,"##き":30178,"##く":30179,"##け":30180,"##こ":30181,"##さ":30182,"##し":30183,"##す":30184,"##せ":30185,"##そ":30186,"##た":30187,"##ち":30188,"##っ":30189,"##つ":30190,"##て":30191,"##と":30192,"##な":30193,"##に":30194,"##ぬ":30195,"##ね":30196,"##の":30197,"##は":30198,"##ひ":30199,"##ふ":30200,"##へ":30201,"##ほ":30202,"##ま":30203,"##み":30204,"##む":30205,"##め":30206,"##も":30207,"##や":30208,"##ゆ":30209,"##よ":30210,"##ら":30211,"##り":30212,"##る":30213,"##れ":30214,"##ろ":30215,"##を":30216,"##ん":30217,"##ァ":30218,"##ア":30219,"##ィ":30220,"##イ":30221,"##ウ":30222,"##ェ":30223,"##エ":30224,"##オ":30225,"##カ":30226,"##キ":30227,"##ク":30228,"##ケ":30229,"##コ":30230,"##サ":30231,"##シ":30232,"##ス":30233,"##セ":30234,"##タ":30235,"##チ":30236,"##ッ":30237,"##ツ":30238,"##テ":30239,"##ト":30240,"##ナ":30241,"##ニ":30242,"##ノ":30243,"##ハ":30244,"##ヒ":30245,"##フ":30246,"##ヘ":30247,"##ホ":30248,"##マ":30249,"##ミ":30250,"##ム":30251,"##メ":30252,"##モ":30253,"##ャ":30254,"##ュ":30255,"##ョ":30256,"##ラ":30257,"##リ":30258,"##ル":30259,"##レ":30260,"##ロ":30261,"##ワ":30262,"##ン":30263,"##・":30264,"##ー":30265,"##一":30266,"##三":30267,"##上":30268,"##下":30269,"##不":30270,"##世":30271,"##中":30272,"##主":30273,"##久":30274,"##之":30275,"##也":30276,"##事":30277,"##二":30278,"##五":30279,"##井":30280,"##京":30281,"##人":30282,"##亻":30283,"##仁":30284,"##介":30285,"##代":30286,"##仮":30287,"##伊":30288,"##会":30289,"##佐":30290,"##侍":30291,"##保":30292,"##信":30293,"##健":30294,"##元":30295,"##光":30296,"##八":30297,"##公":30298,"##内":30299,"##出":30300,"##分":30301,"##前":30302,"##劉":30303,"##力":30304,"##加":30305,"##勝":30306,"##北":30307,"##区":30308,"##十":30309,"##千":30310,"##南":30311,"##博":30312,"##原":30313,"##口":30314,"##古":30315,"##史":30316,"##司":30317,"##合":30318,"##吉":30319,"##同":30320,"##名":30321,"##和":30322,"##囗":30323,"##四":30324,"##国":30325,"##國":30326,"##土":30327,"##地":30328,"##坂":30329,"##城":30330,"##堂":30331,"##場":30332,"##士":30333,"##夏":30334,"##外":30335,"##大":30336,"##天":30337,"##太":30338,"##夫":30339,"##奈":30340,"##女":30341,"##子":30342,"##学":30343,"##宀":30344,"##宇":30345,"##安":30346,"##宗":30347,"##定":30348,"##宣":30349,"##宮":30350,"##家":30351,"##宿":30352,"##寺":30353,"##將":30354,"##小":30355,"##尚":30356,"##山":30357,"##岡":30358,"##島":30359,"##崎":30360,"##川":30361,"##州":30362,"##巿":30363,"##帝":30364,"##平":30365,"##年":30366,"##幸":30367,"##广":30368,"##弘":30369,"##張":30370,"##彳":30371,"##後":30372,"##御":30373,"##德":30374,"##心":30375,"##忄":30376,"##志":30377,"##忠":30378,"##愛":30379,"##成":30380,"##我":30381,"##戦":30382,"##戸":30383,"##手":30384,"##扌":30385,"##政":30386,"##文":30387,"##新":30388,"##方":30389,"##日":30390,"##明":30391,"##星":30392,"##春":30393,"##昭":30394,"##智":30395,"##曲":30396,"##書":30397,"##月":30398,"##有":30399,"##朝":30400,"##木":30401,"##本":30402,"##李":30403,"##村":30404,"##東":30405,"##松":30406,"##林":30407,"##森":30408,"##楊":30409,"##樹":30410,"##橋":30411,"##歌":30412,"##止":30413,"##正":30414,"##武":30415,"##比":30416,"##氏":30417,"##民":30418,"##水":30419,"##氵":30420,"##氷":30421,"##永":30422,"##江":30423,"##沢":30424,"##河":30425,"##治":30426,"##法":30427,"##海":30428,"##清":30429,"##漢":30430,"##瀬":30431,"##火":30432,"##版":30433,"##犬":30434,"##王":30435,"##生":30436,"##田":30437,"##男":30438,"##疒":30439,"##発":30440,"##白":30441,"##的":30442,"##皇":30443,"##目":30444,"##相":30445,"##省":30446,"##真":30447,"##石":30448,"##示":30449,"##社":30450,"##神":30451,"##福":30452,"##禾":30453,"##秀":30454,"##秋":30455,"##空":30456,"##立":30457,"##章":30458,"##竹":30459,"##糹":30460,"##美":30461,"##義":30462,"##耳":30463,"##良":30464,"##艹":30465,"##花":30466,"##英":30467,"##華":30468,"##葉":30469,"##藤":30470,"##行":30471,"##街":30472,"##西":30473,"##見":30474,"##訁":30475,"##語":30476,"##谷":30477,"##貝":30478,"##貴":30479,"##車":30480,"##軍":30481,"##辶":30482,"##道":30483,"##郎":30484,"##郡":30485,"##部":30486,"##都":30487,"##里":30488,"##野":30489,"##金":30490,"##鈴":30491,"##镇":30492,"##長":30493,"##門":30494,"##間":30495,"##阝":30496,"##阿":30497,"##陳":30498,"##陽":30499,"##雄":30500,"##青":30501,"##面":30502,"##風":30503,"##食":30504,"##香":30505,"##馬":30506,"##高":30507,"##龍":30508,"##龸":30509,"##fi":30510,"##fl":30511,"##！":30512,"##（":30513,"##）":30514,"##，":30515,"##－":30516,"##．":30517,"##／":30518,"##：":30519,"##？":30520,"##～":30521}}} \ No newline at end of file diff --git a/benchmarks/benchmark_nl.py b/benchmarks/benchmark_nl.py index 8239eaa..b5d9d42 100644 --- a/benchmarks/benchmark_nl.py +++ b/benchmarks/benchmark_nl.py @@ -16,6 +16,10 @@ import os import gc import hashlib +import uuid +import shutil +import subprocess +from pathlib import Path try: import pandas as pd @@ -28,6 +32,100 @@ pq = None +KAGGLE_WIKIPEDIA_HANDLE = "jjinho/wikipedia-20230701" +DEFAULT_WIKIPEDIA_CACHE = Path.home() / ".cache" / "cuemap" / "benchmarks" / "wikipedia-20230701" + + +def _parquet_files(path: Path) -> List[Path]: + """Return parquet files below a dataset path, including nested downloads.""" + if not path.exists(): + return [] + if path.is_file(): + return [path] if path.suffix.lower() == ".parquet" else [] + return sorted(path.rglob("*.parquet")) + + +def ensure_wikipedia_dataset(path: Optional[str]) -> str: + """Resolve an explicit parquet path or download the release fixture on demand. + + The benchmark intentionally does not vendor Wikipedia data. When no path is + supplied, the public Kaggle fixture used for the release run is downloaded + into a reusable local cache. KaggleHub is preferred, with the Kaggle CLI as + a fallback for environments that already have it configured. + """ + if path and path.strip(): + return os.path.expanduser(path) + + cache_root = Path( + os.environ.get("CUEMAP_BENCHMARK_WIKIPEDIA_DIR", str(DEFAULT_WIKIPEDIA_CACHE)) + ).expanduser() + cache_root.mkdir(parents=True, exist_ok=True) + + parquet_files = _parquet_files(cache_root) + if parquet_files: + print( + f"Using cached Kaggle Wikipedia fixture at {cache_root} " + f"({len(parquet_files):,} parquet files)." + ) + return str(cache_root) + + print( + f"No --wikipedia-path supplied; downloading Kaggle dataset " + f"{KAGGLE_WIKIPEDIA_HANDLE} to {cache_root}..." + ) + errors = [] + + try: + import kagglehub + except ImportError: + errors.append( + "kagglehub is not installed (install it with `python -m pip install kagglehub`)" + ) + else: + try: + kagglehub.dataset_download( + KAGGLE_WIKIPEDIA_HANDLE, + output_dir=str(cache_root), + ) + except Exception as exc: + errors.append(f"kagglehub download failed: {exc}") + + parquet_files = _parquet_files(cache_root) + if not parquet_files: + kaggle_executable = shutil.which("kaggle") + if kaggle_executable: + try: + subprocess.run( + [ + kaggle_executable, + "datasets", + "download", + KAGGLE_WIKIPEDIA_HANDLE, + "--path", + str(cache_root), + "--unzip", + ], + check=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + errors.append(f"Kaggle CLI download failed: {exc}") + else: + errors.append("the `kaggle` CLI is not installed") + + parquet_files = _parquet_files(cache_root) + if not parquet_files: + detail = "; ".join(errors) + raise RuntimeError( + "Could not download the Kaggle Wikipedia fixture. " + "Install kagglehub (`python -m pip install kagglehub`) and configure " + "Kaggle access if prompted, or pass --wikipedia-path to a local parquet " + f"dataset. Details: {detail}" + ) + + print(f"Downloaded {len(parquet_files):,} parquet files to {cache_root}.") + return str(cache_root) + + @dataclass class BenchmarkResult: """Results from a benchmark run.""" @@ -84,8 +182,7 @@ def __init__(self, path: str, max_texts: int = 50000, file_limit: int = 10, batc if (pd or pq) and os.path.exists(path): try: if os.path.isdir(path): - import glob - files = glob.glob(os.path.join(path, "*.parquet")) + files = [str(file) for file in Path(path).rglob("*.parquet")] if not files: print(f" ! No parquet files found in {path}") return @@ -225,7 +322,10 @@ def __init__( query_sample_size: int = 10000, include_metadata: bool = False, batch_writes: bool = False, + semantic_mode: str = "lexical", ): + if semantic_mode not in {"lexical", "semantic", "hybrid"}: + raise ValueError("semantic_mode must be lexical, semantic, or hybrid") self.python_url = python_url self.rust_url = rust_url self.project_id = project_id @@ -239,6 +339,7 @@ def __init__( self.query_sample_size = max(1, query_sample_size) self.include_metadata = include_metadata self.batch_writes = batch_writes + self.semantic_mode = semantic_mode self.synthetic_counter = 0 def _get_headers(self) -> dict: @@ -247,6 +348,17 @@ def _get_headers(self) -> dict: if self.project_id: headers["X-Project-ID"] = self.project_id return headers + + async def _read_json_response(self, response: aiohttp.ClientResponse, operation: str) -> Any: + """Read a JSON response and fail loudly on HTTP or payload errors.""" + body = await response.text() + if not 200 <= response.status < 300: + detail = body[:500].replace("\n", " ") + raise RuntimeError(f"{operation} failed with HTTP {response.status}: {detail}") + try: + return json.loads(body) + except json.JSONDecodeError as exc: + raise RuntimeError(f"{operation} returned invalid JSON") from exc async def generate_memory_content(self, idx: int, embedded_cues: List[str]) -> str: """Generate memory content with embedded cues for extraction.""" @@ -298,9 +410,8 @@ async def seed_data( } async with session.post(f"{url}/memories", json=payload, headers=self._get_headers()) as resp: - if resp.status == 200: - data = await resp.json() - memory_ids.append(data["id"]) + data = await self._read_json_response(resp, "memory ingestion") + memory_ids.append(data["id"]) return memory_ids @@ -344,7 +455,7 @@ async def benchmark_writes(self, session: aiohttp.ClientSession, url: str, size: json={"memories": payloads, "minimal_response": True}, headers=self._get_headers(), ) as resp: - await resp.json() + await self._read_json_response(resp, "batch memory ingestion") op_end = time.time() per_memory_ms = ((op_end - op_start) * 1000) / max(1, len(payloads)) latencies.extend([per_memory_ms] * len(payloads)) @@ -352,7 +463,7 @@ async def benchmark_writes(self, session: aiohttp.ClientSession, url: str, size: for payload in payloads: op_start = time.time() async with session.post(f"{url}/memories", json=payload, headers=self._get_headers()) as resp: - await resp.json() + await self._read_json_response(resp, "memory ingestion") op_end = time.time() latencies.append((op_end - op_start) * 1000) @@ -409,6 +520,10 @@ async def benchmark_reads( payload = { "query_text": query_text, "cues": [], # No explicit cues + # Keep the release latency benchmark on the sparse lexical + # path. Rust defaults to hybrid recall when this field is + # omitted, which would include encoder/reranking work. + "semantic_mode": self.semantic_mode, "limit": 5, "trace_timing": trace_timing, } @@ -420,7 +535,6 @@ async def benchmark_reads( "disable_cuebridge_artifacts": True, "depth": 1, "expansion_depth": 1, - "cuepacks": [], "parent_fusion": "off", "ordered_reconstruction": "off", "evidence_coverage": "off", @@ -437,7 +551,7 @@ async def benchmark_reads( for payload in payloads: op_start = time.time() async with session.post(f"{url}/recall", json=payload, headers=self._get_headers()) as resp: - body = await resp.json() + body = await self._read_json_response(resp, "memory recall") op_end = time.time() latencies.append((op_end - op_start) * 1000) # Convert to ms @@ -580,7 +694,7 @@ def print_results(self, results: List[BenchmarkResult]): print(f"Dataset Size: {result.dataset_size:,}") print(f"Time: {result.total_time:.2f}s") print(f"Throughput: {result.throughput:.0f} ops/s") - print(f"Latency (ms): Avg={result.avg_latency:.2f}, P50={result.p50_latency:.2f}, P99={result.p99_latency:.2f}") + print(f"Latency (ms): Avg={result.avg_latency:.2f}, P50={result.p50_latency:.2f}, P95={result.p95_latency:.2f}") if result.timing_summary: timing_items = [ item for item in result.timing_summary.items() @@ -590,21 +704,21 @@ def print_results(self, results: List[BenchmarkResult]): item for item in result.timing_summary.items() if item not in timing_items ] - print("Timing breakdown (avg / p99 ms):") + print("Timing breakdown (avg / p95 ms):") for key, stats in sorted( timing_items, key=lambda item: item[1]["avg"], reverse=True )[:20]: - print(f" {key}: {stats['avg']:.3f} / {stats['p99']:.3f} ms") + print(f" {key}: {stats['avg']:.3f} / {stats['p95']:.3f} ms") if counter_items: - print("Trace counters (avg / p99):") + print("Trace counters (avg / p95):") for key, stats in sorted( counter_items, key=lambda item: item[1]["avg"], reverse=True )[:12]: - print(f" {key}: {stats['avg']:.3f} / {stats['p99']:.3f}") + print(f" {key}: {stats['avg']:.3f} / {stats['p95']:.3f}") print("-" * 40) def save_results(self, results: List[BenchmarkResult], filename: str = "benchmark_nl_results.json"): @@ -639,7 +753,7 @@ async def main(): parser = argparse.ArgumentParser(description='CueMap NL Benchmark: Python vs Rust') parser.add_argument('--sizes', type=str, help='Comma-separated list of sizes (e.g., 10000,100000)') parser.add_argument('--project-id', type=str, default="nl_test",help='Project ID for multi-tenant instance') - parser.add_argument('--wikipedia-path', type=str, default=os.path.expanduser('~/Downloads/wikipedia/'), help='Path to Wikipedia parquet file or directory') + parser.add_argument('--wikipedia-path', type=str, default=None, help='Path to a local Wikipedia parquet file or directory; omitted downloads the release Kaggle fixture') parser.add_argument('--wait-for-jobs', action='store_true', help='Wait for background jobs to complete before running recall benchmarks') parser.add_argument('--trace-timing', action='store_true', help='Request and aggregate /recall timing breakdowns from the Rust server') parser.add_argument('--wiki-reservoir-size', type=int, default=50000, help='Maximum unique sampled Wikipedia snippets to keep in RAM') @@ -648,11 +762,13 @@ async def main(): parser.add_argument('--query-sample-size', type=int, default=10000, help='Maximum ingested texts retained for recall query generation') parser.add_argument('--include-metadata', action='store_true', help='Include per-memory benchmark metadata during writes') parser.add_argument('--batch-writes', action='store_true', help='Write each payload buffer through /memories/batch instead of one POST per memory') + parser.add_argument('--semantic-mode', choices=('lexical', 'semantic', 'hybrid'), default='lexical', help='Rust recall mode; lexical is the release sparse-core benchmark default') args = parser.parse_args() print("CueMap NL Benchmark: Python vs Rust") print("Testing Natural Language Extraction & Query Resolution") print("="*80) + print("Release sparse-core mode: start Rust with CUEMAP_SEMANTIC_ENCODER_ENABLED=false") if args.project_id: print(f"Running in Multi-Tenant mode for project: {args.project_id}") @@ -670,10 +786,11 @@ async def main(): print(f"\nRunning benchmark for sizes: {sizes_to_run}") + wikipedia_path = ensure_wikipedia_dataset(args.wikipedia_path) requested_unique_writes = sum(sizes_to_run) effective_wiki_reservoir_size = args.wiki_reservoir_size effective_wiki_file_limit = args.wiki_file_limit - if args.wikipedia_path and effective_wiki_reservoir_size < requested_unique_writes: + if wikipedia_path and effective_wiki_reservoir_size < requested_unique_writes: effective_wiki_reservoir_size = requested_unique_writes print( f"Auto-increasing Wikipedia unique reservoir from " @@ -691,17 +808,19 @@ async def main(): python_url="http://localhost:8000", rust_url="http://localhost:8080", project_id=args.project_id, - wiki_path=args.wikipedia_path, + wiki_path=wikipedia_path, wiki_reservoir_size=effective_wiki_reservoir_size, wiki_file_limit=effective_wiki_file_limit, payload_buffer_size=args.payload_buffer_size, query_sample_size=args.query_sample_size, include_metadata=args.include_metadata, batch_writes=args.batch_writes, + semantic_mode=args.semantic_mode, ) try: results = [] + run_tag = f"{int(time.time())}_{os.getpid()}_{uuid.uuid4().hex[:6]}" async with aiohttp.ClientSession() as session: # Health check print("\nChecking server health...") @@ -717,9 +836,20 @@ async def main(): print("✓ Rust server is healthy\n") for idx, size in enumerate(sizes_to_run): + # Keep every requested scale isolated. Otherwise a 1M pass + # after a 100K pass would benchmark 1.1M memories in the same + # project, and rerunning the command could reuse old state. + base_project_id = args.project_id or "nl_test" + suffix = f"_{run_tag}_{size}" + prefix_length = 64 - len(suffix) + if prefix_length < 3: + raise ValueError("--project-id is too long for a run-scoped benchmark project") + benchmark.project_id = f"{base_project_id[:prefix_length]}{suffix}" print(f"\n{'='*60}") print(f"Benchmarking with {size:,} memories [{idx+1}/{len(sizes_to_run)}]") print(f"{'='*60}") + print(f" Project: {benchmark.project_id}") + print(f" Rust recall mode: {benchmark.semantic_mode}") num_queries = 1000 if size <= 10000 else 500 @@ -751,7 +881,9 @@ async def main(): # Wait for background jobs before recall benchmarks (if flag set) if args.wait_for_jobs: await benchmark.wait_for_jobs(session, benchmark.rust_url) - except Exception as e: print(f" ✗ Failed: {e}") + except Exception as e: + print(f" ✗ Failed: {e}") + raise if py_healthy: print(f"\n[Python] Read (NL) benchmark ({num_queries} queries)...") @@ -763,7 +895,7 @@ async def main(): ingested_texts=py_texts ) results.append(res) - print(f" ✓ Completed in {res.total_time:.2f}s (P99: {res.p99_latency:.2f}ms)") + print(f" ✓ Completed in {res.total_time:.2f}s (P95: {res.p95_latency:.2f}ms)") except Exception as e: print(f" ✗ Failed: {e}") print(f"[Rust] Read (NL, Lean) benchmark ({num_queries} queries)...") @@ -776,8 +908,10 @@ async def main(): trace_timing=args.trace_timing ) results.append(res) - print(f" ✓ Completed in {res.total_time:.2f}s (P99: {res.p99_latency:.2f}ms)") - except Exception as e: print(f" ✗ Failed: {e}") + print(f" ✓ Completed in {res.total_time:.2f}s (P95: {res.p95_latency:.2f}ms)") + except Exception as e: + print(f" ✗ Failed: {e}") + raise benchmark.print_results(results) benchmark.save_results(results) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..829e9a2 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,17 @@ +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + target: auto + threshold: 1% + patch: + default: + target: auto + threshold: 1% + +comment: + layout: "reach, diff, flags, files" + behavior: default diff --git a/cuepacks/memory-general.toml b/cuepacks/memory-general.toml deleted file mode 100644 index 61897df..0000000 --- a/cuepacks/memory-general.toml +++ /dev/null @@ -1,429 +0,0 @@ -name = "memory-general" -version = "0.1.0" -description = "General deterministic semantic cues for personal memory, chat logs, notes, and assistant answers." -enabled_by_default = true - -[[query_rules]] -id = "memory.ordered_reconstruction" -contains_any = [ - "timeline", "sequence", "in order", "chronological", "chronologically", - "progress", "progression", "evolved", "developed over time", "progressed", - "what order" -] -regex_any = [ - '''(?i)\bhow\s+did\b.{0,120}\b(?:progress|progression|evolve|evolved|develop|developed)\b''' -] -labels = ["ordered_reconstruction"] - -[[query_rules]] -id = "memory.multi_evidence_summary" -contains_any = [ - "summarize", "summary", "main points", "overall", "across the conversation", - "across this conversation", "across the thread", "across this thread", - "what happened across", "recap", "overview", "key points", "key details", - "key features", "comprehensive summary", "comprehensive overview" -] -labels = ["multi_evidence_summary"] - -[[query_rules]] -id = "memory.multi_evidence_collection" -regex_any = [ - '''(?ix)^\s*(?:what|which)\s+(?:[a-z][a-z'-]+\s+){0,5}(?:has|have|did|do|does|are|were|can|should|would)\b.{0,140}\b(?:all|some|several|multiple|different|various|main|favorite|recommended|suggested|options?|examples?|ideas?|done|attended|visited|read|watched|bought|purchased|tried|used|mentioned|participated)\b''', - '''(?ix)^\s*(?:what|which)\s+(?:kinds?|types?|ways|options?|examples?|ideas?)\s+(?:of|for|did|do|are|were)\b''', - '''(?ix)^\s*(?:can|could)\s+you\s+(?:suggest|recommend|list|summarize)\b.{0,140}\b(?:based on|from|across|overall|some|options?|ideas?|examples?)\b''' -] -labels = ["multi_evidence_collection"] -weighted_cues = [ - { cue = "has:list", weight = 1.2 } -] -cue_weight_adjustments = [ - { cue = "things", weight = 0.55 }, - { cue = "ways", weight = 0.55 }, - { cue = "kinds", weight = 0.55 }, - { cue = "types", weight = 0.55 }, - { cue = "options", weight = 0.55 }, - { cue = "ideas", weight = 0.55 }, - { cue = "examples", weight = 0.55 } -] - -[[query_rules]] -id = "memory.source_instruction" -contains_any = [ - "asked me to", "ask me to", "told me to", "tell me to", "requested", - "request", "instructed", "instruction", "instructions", "what did i ask", - "what did i tell", "what did i request" -] -labels = ["source_instruction"] - -[[query_rules]] -id = "memory.instruction_applicable" -regex_any = [ - '''(?ix)^\s*(?:what\s+(?:should|are|is|can|information|tools|options)|how\s+(?:do|should|can|does|did|has|is)|which|when\s+(?:building|creating|implementing|working|using|handling|planning|writing|designing|organizing)\b|if\s+(?:i|we)(?:'m| am|'re| are)?\s+(?:building|creating|implementing|working|using|handling|planning|writing|designing|organizing)\b|can\s+you\s+(?:explain|help|recommend|show|tell)|could\s+you\s+(?:explain|help|recommend|show|tell)|i(?:'m| am)\s+(?:trying|thinking|looking|planning|working|about\s+to)|i\s+(?:need|want)\s+to\b)''' -] -labels = ["instruction_applicable"] - -[[query_rules]] -id = "memory.preference_applicable" -regex_any = [ - '''(?ix)^\s*(?:what\s+(?:should|are|is|can|options|steps|ways)|how\s+(?:do|should|can|would)|which|when\s+(?:building|creating|implementing|working|using|handling|planning|writing|designing|organizing|choosing|editing)\b|if\s+(?:i|we)(?:'m| am|'re| are)?\s+(?:building|creating|implementing|working|using|handling|planning|writing|designing|organizing|choosing|editing)\b|can\s+you\s+(?:explain|help|recommend|show|suggest|walk|tell)|could\s+you\s+(?:explain|help|recommend|show|suggest|walk|tell)|i(?:'m| am)\s+(?:trying|thinking|looking|planning|working|about\s+to)|i\s+(?:need|want)\s+to\b)''' -] -labels = ["preference_applicable"] - -[[memory_rules]] -id = "memory.standing_instruction.always_when" -regex_any = [ - '''(?is)\balways\s+.{3,220}?\bwhen\s+(?:i|we|the user|users?|someone|people)\s+(?:ask|asks|asked|am asking|are asking)\s+about\s+.{3,220}''', - '''(?is)\bwhen\s+(?:i|we|the user|users?|someone|people)\s+(?:ask|asks|asked|am asking|are asking)\s+about\s+.{3,220}?\b(?:always|make sure to|remember to|include|specify|confirm|explain|provide)\b''', - '''(?is)\b(?:make sure to|remember to|please)\s+.{3,220}?\bwhen\s+(?:discussing|covering|answering|responding to|talking about|i\s+ask\s+about|we\s+ask\s+about)\s+.{3,220}''' -] -emits = ["type:standing_instruction", "instruction:conditional", "instruction:always"] - -[[memory_rules]] -id = "memory.preference.explicit" -regex_any = [ - '''(?is)\b(?:i|we)\s+(?:really\s+|usually\s+|generally\s+|strongly\s+)?(?:prefer|like|love|enjoy)\s+.{3,220}''', - '''(?is)\b(?:my|our)\s+preference\s+(?:is|for|would\s+be)\s+.{3,220}''', - '''(?is)\b(?:i|we)\s+would\s+rather\s+.{3,220}''', - '''(?is)\b(?:i|we)\s+(?:do\s+not|don't|dont|dislike|avoid|can't\s+stand|cannot\s+stand)\s+.{3,220}''' -] -emits = ["preference:explicit"] - -[[memory_rules]] -id = "memory.inline_enumeration" -regex_any = [ - '''(?ix)\b(?:including|such as|for example|for instance|like|especially)\b.{0,160}\b[a-z][a-z'-]+\b\s*,\s*\b[a-z][a-z'-]+\b(?:\s*,\s*|\s+(?:and|or)\s+)\b[a-z][a-z'-]+\b''' -] -emits = ["has:list"] - -[[query_rules]] -id = "aquarium.tank_alias" -contains_any = ["aquarium", "aquariums"] -weighted_cues = [ - { cue = "tank", weight = 3.8 } -] - -[[query_rules]] -id = "home.bedroom_furniture_alias" -contains_all = ["bedroom", "furniture"] -weighted_cues = [ - { cue = "dresser", weight = 4.2 }, - { cue = "wardrobe", weight = 3.6 }, - { cue = "nightstand", weight = 3.6 } -] - -[[memory_rules]] -id = "navigation.route" -contains_any = [ - "getting around", "get around", "how to get to", "how do i get to", "how can i get to", - "best way to get", "way to get there", "way to get to", "route to", "route from", - "directions to", "direction to", "navigate", "navigation", "meeting point", - "travel to", "travel from", "transfer to", "transfer at" -] -emits = ["type:navigation", "travel:route"] - -[[memory_rules]] -id = "navigation.transit" -contains_any = [ - "public transport", "public transportation", "public transit", "mass transit", - "take the train", "take a train", "by train", "train from", "train to", "train station", - "subway", "metro", "take the bus", "take a bus", "by bus", "bus from", "bus to", - "bus station", "tram", "ferry", "taxi", "rideshare", "ride share", "airport shuttle" -] -emits = ["type:navigation", "travel:transit"] - -[[memory_rules]] -id = "navigation.station" -contains_any = [ - "station", "airport", "terminal", "platform", "ticket gate", "departure gate", - "arrival gate", "gate when entering", "gate when exiting" -] -emits = ["type:navigation", "travel:station"] - -[[memory_rules]] -id = "navigation.fare" -contains_any = [ - "fare", "ticket", "tickets", "travel time", "ride time", "journey time", - "transfer time", "approximate cost", "cost using" -] -emits = ["type:navigation", "travel:fare"] - -[[memory_rules]] -id = "navigation.pass" -contains_any = [ - "transit card", "transport card", "transportation card", "prepaid card", - "rail pass", "train pass", "travel pass", "metrocard", "metro card" -] -emits = ["type:navigation", "travel:pass"] - -[[memory_rules]] -id = "navigation.named_card_pass" -regex_any = [ - '''(?i)\b(?:using|with|my|our|a|an|the)\s+[A-Z][A-Za-z0-9+.'-]{1,40}\s+card\b''' -] -emits = ["type:navigation", "travel:pass"] - -[[memory_rules]] -id = "navigation.app" -contains_any = [ - "transit app", "transport app", "transportation app", "travel app", "trip app", - "route app", "maps app", "itinerary app", "tripit app", "google maps", "apple maps", - "citymapper", "moovit", "downloaded the app", "downloaded an app" -] -emits = ["type:navigation", "travel:app"] - -[[memory_rules]] -id = "media.music_streaming.regex" -regex_any = [ - '''(?s)\b(?:[Ll]isten(?:ing)?\s+to|[Ss]tream(?:ing)?|[Pp]laying)\b.{0,100}\b(?:[Mm]usic|[Ss]ongs?|[Tt]racks?|[Aa]lbums?|[Aa]rtists?|[Pp]laylists?)\b.{0,80}\bon\s+[A-Z][A-Za-z0-9+.'-]{1,40}\b''', - '''(?s)\b(?:[Mm]usic|[Ss]ongs?|[Tt]racks?|[Aa]lbums?|[Aa]rtists?|[Pp]laylists?)\b.{0,80}\bon\s+[A-Z][A-Za-z0-9+.'-]{1,40}\b.{0,100}\b(?:[Ll]isten(?:ing)?\s+to|[Ss]tream(?:ing)?|[Pp]laying)\b''' -] -emits = ["media:music", "media:music_streaming", "media:streaming", "type:usage"] - -[[memory_rules]] -id = "media.music_streaming.phrase" -contains_any = [ - "music streaming service", "music streaming services", "music streaming platform", - "music streaming platforms" -] -emits = ["media:music", "media:music_streaming", "media:streaming", "type:usage"] - -[[memory_rules]] -id = "media.streaming" -contains_any = [ - "streaming service", "streaming services", "streaming platform", "streaming platforms" -] -emits = ["media:streaming"] - -[[memory_rules]] -id = "media.watching" -contains_any = [ - "documentary", "documentaries", "episode", "episodes", "film", "films", "movie", - "movies", "show", "shows", "specials", "tv show", "tv shows", "watch", "watched", "watching" -] -emits = ["media:watching"] - -[[memory_rules]] -id = "media.usage" -contains_all = ["watch"] -contains_any = [ - "been using", "free trial", "keeping up with", "started a free trial", "started using", - "start using", "subscribed to", "subscription" -] -emits = ["media:streaming", "type:usage"] - -[[memory_rules]] -id = "media.usage_watch_context" -contains_any = [ - "documentary", "documentaries", "episode", "episodes", "film", "films", "movie", - "movies", "show", "shows", "specials", "tv show", "tv shows" -] -contains_all = ["free trial"] -emits = ["media:streaming", "type:usage"] - -[[memory_rules]] -id = "media.long_term_watch_usage" -contains_any = [ - "documentary", "documentaries", "episode", "episodes", "film", "films", "movie", - "movies", "show", "shows", "specials", "tv show", "tv shows", "watch", "watching", "watched" -] -contains_all = ["been using"] -emits = ["media:streaming", "type:usage"] - -[[memory_rules]] -id = "reading.current" -contains_any = ["book", "novel", "memoir", "biography", "reading", "read", "author"] -regex_any = [ - '''(?ix)\b(?:i|we)(?:\s+am|\s+are|'m|'re|’m|’re|\s+have\s+been|\s+has\s+been|'ve\s+been|’ve\s+been)?\s+(?:currently\s+|right\s+now\s+|still\s+)?(?:reading|devouring|rereading|re-reading|halfway\s+through|in\s+the\s+middle\s+of)\b''' -] -emits = ["media:book", "media:book_reading", "reading:current", "temporal:current"] - -[[memory_rules]] -id = "reading.current_quoted_title" -regex_any = [ - '''(?ix)\b(?:i|we)(?:\s+am|\s+are|'m|\s+have\s+been|'ve\s+been)?\s+(?:currently\s+|right\s+now\s+|still\s+)?(?:reading|devouring|rereading|re-reading|halfway\s+through|in\s+the\s+middle\s+of)\b.{0,80}["“][^"”]{3,120}["”]''' -] -emits = ["media:book", "media:book_reading", "reading:current", "temporal:current"] - -[[memory_rules]] -id = "religion.topic" -contains_any = [ - "abbey", "ashram", "baptist", "bible", "buddhist", "cathedral", "catholic", - "chapel", "christian", "church", "convent", "episcopal", "gurdwara", "hindu", - "islamic", "jewish", "lutheran", "methodist", "monastery", "mosque", "muslim", - "orthodox", "parish", "presbyterian", "rabbi", "shrine", "synagogue", "temple" -] -emits = ["topic:religion"] - -[[memory_rules]] -id = "religion.event" -contains_any = [ - "bible study", "church service", "communion service", "eucharist", - "maundy thursday service", "prayer meeting", "sabbath service", "sunday school", - "worship service" -] -emits = ["topic:religion", "activity_domain:religion", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.bus" -contains_any = [ - "bus", "buses", "bus ride", "bus trip", "bus commute", "took a bus", "took the bus", - "taking a bus", "taking the bus", "rode a bus", "rode the bus" -] -emits = ["transport_mode:bus"] - -[[memory_rules]] -id = "transport.bus_event" -contains_any = [ - "bus ride", "bus trip", "bus commute", "took a bus", "took the bus", - "taking a bus", "taking the bus", "rode a bus", "rode the bus", - "got back from a bus", "got back from the bus" -] -emits = ["transport_event:bus", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.train" -contains_any = [ - "train", "trains", "train ride", "train trip", "train commute", "took a train", - "took the train", "taking a train", "taking the train", "rode a train", "rode the train" -] -emits = ["transport_mode:train"] - -[[memory_rules]] -id = "transport.train_event" -contains_any = [ - "train ride", "train trip", "train commute", "took a train", "took the train", - "taking a train", "taking the train", "rode a train", "rode the train", - "got back from a train", "got back from the train" -] -emits = ["transport_event:train", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.plane" -contains_any = [ - "plane", "planes", "flight", "flights", "plane ride", "plane trip", "took a flight", - "took the flight", "boarded a plane", "boarded the plane" -] -emits = ["transport_mode:plane"] - -[[memory_rules]] -id = "transport.plane_event" -contains_any = [ - "plane ride", "plane trip", "flight", "flights", "took a flight", - "took the flight", "boarded a plane", "boarded the plane" -] -emits = ["transport_event:plane", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.car" -contains_any = ["car", "cars", "car ride", "car trip", "car commute"] -emits = ["transport_mode:car"] - -[[memory_rules]] -id = "transport.car_event" -contains_any = ["car ride", "car trip", "car commute", "took a car", "drove a car"] -emits = ["transport_event:car", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.taxi" -contains_any = [ - "taxi", "taxis", "cab", "cabs", "taxi ride", "cab ride", "took a taxi", - "took a cab", "taking a taxi", "taking a cab" -] -emits = ["transport_mode:taxi"] - -[[memory_rules]] -id = "transport.taxi_event" -contains_any = ["taxi ride", "cab ride", "took a taxi", "took a cab", "taking a taxi", "taking a cab"] -emits = ["transport_event:taxi", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.subway" -contains_any = [ - "subway", "subways", "metro", "metros", "subway ride", "metro ride", - "took the subway", "took the metro", "taking the subway", "taking the metro" -] -emits = ["transport_mode:subway"] - -[[memory_rules]] -id = "transport.subway_event" -contains_any = ["subway ride", "metro ride", "took the subway", "took the metro", "taking the subway", "taking the metro"] -emits = ["transport_event:subway", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.tram" -contains_any = ["tram", "trams", "tram ride", "took the tram", "taking the tram"] -emits = ["transport_mode:tram"] - -[[memory_rules]] -id = "transport.tram_event" -contains_any = ["tram ride", "took the tram", "taking the tram"] -emits = ["transport_event:tram", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.ferry" -contains_any = ["ferry", "ferries", "ferry ride", "took the ferry", "taking the ferry"] -emits = ["transport_mode:ferry"] - -[[memory_rules]] -id = "transport.ferry_event" -contains_any = ["ferry ride", "took the ferry", "taking the ferry"] -emits = ["transport_event:ferry", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.bike" -contains_any = ["bike", "bikes", "bicycle", "bicycles", "bike ride", "bicycle ride"] -emits = ["transport_mode:bike"] - -[[memory_rules]] -id = "transport.bike_event" -contains_any = ["bike ride", "bicycle ride", "took a bike", "rode a bike", "rode a bicycle"] -emits = ["transport_event:bike", "type:activity", "type:event"] - -[[memory_rules]] -id = "transport.walk" -contains_any = ["walk", "walking", "walked", "walking tour"] -emits = ["transport_mode:walk"] - -[[memory_rules]] -id = "transport.walk_event" -contains_any = ["walked", "walking tour", "took a walk", "went for a walk"] -emits = ["transport_event:walk", "type:activity", "type:event"] - -[[query_rules]] -id = "navigation.query" -contains_any = [ - "getting around", "get around", "how to get to", "directions to", "route to", - "public transport", "public transportation", "public transit", "transit app", - "transport app", "travel app" -] -labels = ["navigation"] -weighted_cues = [ - { cue = "type:navigation", weight = 2.2 }, - { cue = "travel:route", weight = 1.8 }, - { cue = "travel:transit", weight = 1.8 }, -] -suppress_generic = true - -[[query_rules]] -id = "media.query" -contains_any = [ - "streaming service", "streaming platform", "music streaming", "currently reading", - "what book", "which book", "watching", "watched" -] -labels = ["media"] -weighted_cues = [ - { cue = "media:streaming", weight = 1.8 }, - { cue = "media:music_streaming", weight = 1.8 }, - { cue = "media:book_reading", weight = 1.8 }, -] -suppress_generic = true - -[[query_rules]] -id = "religion.query" -contains_any = ["church service", "bible study", "prayer meeting", "worship service", "sabbath service"] -labels = ["religious_activity"] -weighted_cues = [ - { cue = "activity_domain:religion", weight = 2.0 }, - { cue = "topic:religion", weight = 1.6 }, -] -suppress_generic = true diff --git a/evals/README.md b/evals/README.md index 3182862..a411b5e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,20 +1,22 @@ -# CueMap v0.7 Evaluation Pack +# CueMap v0.7.2 Evaluation Pack -CueMap is a deterministic memory engine with an embedding-free, vector-database-free, LLM-free recall hot path. These reports document the v0.7 retrieval runs we used to calibrate release readiness across LongMemEval, LoCoMo, and BEAM. +CueMap is a deterministic memory engine with an embedding-free, vector-database-free, LLM-free recall hot path. These reports document the v0.7.2 retrieval runs used to calibrate release readiness across LongMemEval, LoCoMo, and BEAM. -The short version: CueMap is already very strong on compact long-memory retrieval, competitive on LoCoMo when adjacent context expansion is enabled, and competitive with reported leading BEAM bands at both 1M and 10M scale. CueBridge's most meaningful v0.7 diagnostic lift showed up on BEAM 128K, where question-oracle artifacts improved Hit@20 by 10 questions while preserving top-20 wins. The next lift is ranking dense multi-evidence sets, event ordering, summarization, and preference-following at larger BEAM scales. +The short version: CueMap is already very strong on compact long-memory retrieval, competitive on LoCoMo when adjacent context expansion is enabled, and shows strong candidate discovery at BEAM 10M scale. The latest raw BEAM 128K, 1M, and 10M runs reach 84.2%, 80.3%, and 67.0% Hit@20, with 4,702, 2,934, and 1,749 average top-20 context tokens respectively. The separate historical CueBridge question-oracle run improved Hit@20 by 10 questions; it is reported independently from the latest raw baselines. + +The latest LoCoMo, LongMemEval, and BEAM runs use the wrappers' default `SEMANTIC_MODE=hybrid`, so their reported retrieval combines lexical and semantic signals. Set `SEMANTIC_MODE=lexical` or `SEMANTIC_MODE=semantic` when intentionally comparing a single retrieval mode. ## Results At A Glance | Benchmark | Mode | Scale | Recall_Any@20 | Recall_Any@100 | Context tokens | Main read | |---|---:|---:|---:|---:|---:|---| -| LongMemEval | Raw | 470 non-abstention Qs | 99.1% | n/a | Compact top-20 recall | Very strong baseline | -| LoCoMo | Raw, expansion-depth 10 | 1,986 Qs | 93.2% | n/a | 10,279 avg | Strong recall with explicit context expansion | -| LoCoMo | Raw, expansion-depth 3 | 1,986 Qs | 89.2% | n/a | 3,362 avg | Lean context setting | -| BEAM | Raw | 128K | 80.3% | 94.4% | Top-100 retrieval | Strong candidate discovery | -| BEAM | CueBridge diagnostic | 128K | 83.7% | 94.6% | Top-100 retrieval | +10 Hit@20 with preserved top-20 wins | -| BEAM | Raw | 1M | 69.9% | 86.2% | Top-100 retrieval | Inside the reported leading 64-72% band | -| BEAM | Raw | 10M | 51.7% | 71.6% | Top-100 retrieval | Inside the reported leading 48-64% band | +| LongMemEval | Raw · hybrid | 470 non-abstention Qs | 96.2% | n/a | 3,914 avg top-20 | Very strong baseline | +| LoCoMo | Raw, expansion-depth 10 | 1,986 Qs | 96.1% | n/a | 10,044 avg | Strong recall with explicit context expansion | +| LoCoMo | Raw, expansion-depth 3 | 1,986 Qs | 94.2% | n/a | 3,184 avg | Lean context setting | +| BEAM | Raw | 128K | 84.2% | 96.3% | 4,702 avg top-20 | Strong candidate discovery | +| BEAM | Historical CueBridge diagnostic | 128K | 83.7% | 94.6% | Previous calibration | +10 Hit@20 in a separate run | +| BEAM | Raw | 1M | 80.3% | 93.1% | 2,934 avg top-20 | Latest raw run | +| BEAM | Raw | 10M | 67.0% | 83.5% | 1,749 avg top-20 | Latest raw run | ## Hot-Path Latency Context @@ -51,10 +53,25 @@ The wrappers call the canonical Python harnesses under `/evals`. If CUEMAP_EVALS_DIR=/path/to/cuemap/evals bash evals/beam/run_beam.sh ``` +Each wrapper pins CLI fallbacks to `rust_engine/target/release/cuemap` and +prints the resolved binary and version before running. Override it explicitly +with `CUEMAP_RUST_BIN=/path/to/cuemap` when comparing another build. + +BEAM defaults to message-level turn ingestion. Use `INGEST_MODE=long-form` only +when intentionally evaluating the segmented `/ingest/content` path. + +The LoCoMo, LongMemEval, and BEAM harnesses report an approximate retrieved +context-token footprint. Each scored question stores `ctx_tokens` in its JSON +record, and the run summary prints Avg/P50/P95/P99/Max plus question-type +breakdowns. For BEAM, `ctx_tokens` is explicitly the top-20 footprint; the full +returned-limit count is available as `ctx_tokens_returned`. The estimate is +tokenizer-independent (word and punctuation counting), so use it for relative +context-budget comparisons rather than exact model billing. + The default URL is `http://127.0.0.1:8080`. Start the Rust engine before running: ```bash -cuemap start +rust_engine/target/release/cuemap start ``` For disposable benchmark projects, keep `DELETE_PROJECTS=1` so temporary `eval_*` projects are deleted after each record. diff --git a/evals/beam/report.md b/evals/beam/report.md index f1b1303..548e812 100644 --- a/evals/beam/report.md +++ b/evals/beam/report.md @@ -1,28 +1,28 @@ # BEAM: Scaling Deterministic Recall To 10M Tokens -`CueMap v0.7` `BEAM 128K / 1M / 10M` `CueBridge lift` `embedding-free` +`CueMap v0.7.2` `BEAM 128K / 1M / 10M` `CueBridge lift` `embedding-free` BEAM is the stress test for scale. CueMap uses deterministic lexical/facet recall instead of an embedding service or vector database, so this benchmark asks the central question: can that architecture stay competitive at 1M and 10M tokens? -The answer in v0.7 is yes. CueMap reaches 69.9% Hit@20 at BEAM 1M and 51.7% Hit@20 at BEAM 10M, which sits inside the reported leading ranges of roughly 64-72% for 1M and 48-64% for 10M. BEAM 128K is also where the current CueBridge diagnostic run produced the most meaningful lift. +The answer in v0.7.2 is yes. The latest raw 128K, 1M, and 10M runs reach 84.2%, 80.3%, and 67.0% Hit@20, with Hit@100 at 96.3%, 93.1%, and 83.5% respectively. All three runs use the wrapper's default **hybrid recall mode** (`SEMANTIC_MODE=hybrid`), message-level turn ingestion, evidence coverage disabled, and ordered reconstruction disabled. ## Headline | Context tier | Questions | Hit@1 | Hit@5 | Hit@10 | Hit@20 | Hit@50 | Hit@100 | |---|---:|---:|---:|---:|---:|---:|---:| -| 128K | 355 | 36.9% | 61.4% | 70.7% | 80.3% | 89.6% | 94.4% | -| 1M | 625 | 28.6% | 50.1% | 61.8% | 69.9% | 80.3% | 86.2% | -| 10M | 176 | 23.3% | 38.1% | 44.3% | 51.7% | 61.9% | 71.6% | +| 128K | 355 | 49.6% | 71.8% | 77.5% | 84.2% | 90.4% | 96.3% | +| 1M | 625 | 38.4% | 63.4% | 74.6% | 80.3% | 90.4% | 93.1% | +| 10M | 176 | 33.5% | 50.0% | 58.5% | 67.0% | 77.8% | 83.5% | -The 10M tier crosses the 50% Hit@20 target while keeping Hit@100 above 70%. That matters because it shows CueMap handles the scale jump and often puts the right memory into the candidate set with embedding-free recall. +The 10M tier crosses the 50% Hit@20 target while keeping Hit@100 above 80%. That matters because it shows CueMap handles the scale jump and often puts the right memory into the candidate set with embedding-free recall. ## Depth Metrics | Context tier | Recall_All@20 | Recall_Frac@20 | NDCG@20 | |---|---:|---:|---:| -| 128K | 46.5% | 60.0% | 44.5% | -| 1M | 25.0% | 42.7% | 33.2% | -| 10M | 17.0% | 28.4% | 23.3% | +| 128K | 54.9% | 69.2% | 54.7% | +| 1M | 34.9% | 54.4% | 41.6% | +| 10M | 23.9% | 38.1% | 29.7% | This is where the next upside is visible. Hit@100 remains much higher than Hit@20, especially at 10M. CueMap is often finding relevant candidates, and a stronger deterministic reranker can move more of them into the top-20 answer context. @@ -30,33 +30,35 @@ This is where the next upside is visible. Hit@100 remains much higher than Hit@2 | Type | Hit@1 | Hit@5 | Hit@10 | Hit@20 | Hit@100 | Read | |---|---:|---:|---:|---:|---:|---| -| Contradiction resolution | 72.5% | 90.0% | 92.5% | 100.0% | 100.0% | Excellent. | -| Event ordering | 10.0% | 35.0% | 57.5% | 70.0% | 100.0% | Candidate discovery is high; ordering is the next lift. | -| Information extraction | 35.0% | 62.5% | 67.5% | 75.0% | 92.5% | Strong base with top-rank upside. | -| Instruction following | 35.0% | 45.0% | 45.0% | 47.5% | 72.5% | Good candidate base for instruction-aware ranking. | -| Knowledge update | 47.5% | 72.5% | 80.0% | 95.0% | 97.5% | Strong. | -| Multi-session reasoning | 42.5% | 65.0% | 82.5% | 92.5% | 100.0% | Good candidate discovery. | -| Preference following | 25.6% | 59.0% | 64.1% | 74.4% | 87.2% | Strong target for semantic preference bridges. | -| Summarization | 11.1% | 33.3% | 52.8% | 69.4% | 100.0% | Candidate discovery high, with set-coverage upside. | -| Temporal reasoning | 50.0% | 87.5% | 92.5% | 97.5% | 100.0% | Strong. | +| Contradiction resolution | 85.0% | 100.0% | 100.0% | 100.0% | 100.0% | Excellent. | +| Event ordering | 17.5% | 57.5% | 67.5% | 85.0% | 97.5% | Candidate discovery is high; ordering is the next lift. | +| Information extraction | 50.0% | 72.5% | 80.0% | 90.0% | 95.0% | Strong base with top-rank upside. | +| Instruction following | 17.5% | 42.5% | 47.5% | 57.5% | 82.5% | Good candidate base for instruction-aware ranking. | +| Knowledge update | 87.5% | 97.5% | 97.5% | 97.5% | 97.5% | Strong. | +| Multi-session reasoning | 57.5% | 87.5% | 97.5% | 100.0% | 100.0% | Good candidate discovery. | +| Preference following | 23.1% | 46.2% | 51.3% | 61.5% | 94.9% | Strong target for semantic preference bridges. | +| Summarization | 13.9% | 38.9% | 52.8% | 63.9% | 100.0% | Candidate discovery high, with set-coverage upside. | +| Temporal reasoning | 90.0% | 100.0% | 100.0% | 100.0% | 100.0% | Strong. | ## 10M Category Breakdown | Type | Hit@20 | Hit@100 | Read | |---|---:|---:|---| -| Contradiction resolution | 90.0% | 100.0% | Robust at 10M. | -| Knowledge update | 80.0% | 90.0% | Strong. | -| Multi-session reasoning | 70.0% | 85.0% | Good candidate discovery with full-evidence upside. | -| Information extraction | 65.0% | 75.0% | Strong base with top-rank upside. | -| Temporal reasoning | 40.0% | 80.0% | Candidate set exists; rank lift is available. | -| Instruction following | 40.0% | 50.0% | Clear target for intent-aware scoring. | -| Event ordering | 30.0% | 65.0% | Clear target for order-aware reranking. | -| Summarization | 31.2% | 43.8% | Clear target for set-coverage reranking. | -| Preference following | 15.0% | 50.0% | Clear target for semantic preference bridges. | - -## CueBridge Diagnostic At 128K - -Question-oracle CueBridge mode improved BEAM 128K Hit@20 from `287/355` to `297/355`. +| Contradiction resolution | 95.0% | 100.0% | Robust at 10M. | +| Knowledge update | 95.0% | 100.0% | Strong. | +| Multi-session reasoning | 85.0% | 100.0% | Good candidate discovery with full-evidence upside. | +| Information extraction | 80.0% | 90.0% | Strong base with top-rank upside. | +| Temporal reasoning | 65.0% | 80.0% | Candidate set exists; rank lift is available. | +| Instruction following | 25.0% | 50.0% | Clear target for intent-aware scoring. | +| Event ordering | 50.0% | 85.0% | Clear target for order-aware reranking. | +| Summarization | 50.0% | 68.8% | Clear target for set-coverage reranking. | +| Preference following | 55.0% | 75.0% | Clear target for semantic preference bridges. | + +## Historical CueBridge Diagnostic At 128K + +This separate question-oracle calibration predates the latest raw 128K run. It +improved Hit@20 from `287/355` to `297/355`; do not compare its raw column +directly with the latest 128K baseline above. | Metric | Raw | CueBridge diagnostic | Delta | |---|---:|---:|---:| @@ -77,9 +79,9 @@ What works now: | Strength | Evidence | |---|---| -| Candidate discovery | 128K Hit@100 is 94.4%; 10M Hit@100 is 71.6%. | +| Candidate discovery | 128K Hit@100 is 96.3%; 1M Hit@100 is 93.1%; 10M Hit@100 is 83.5%. | | Contradictions and updates | These categories stay strong even at 10M. | -| Embedding-free architecture | The engine stays in the reported leading 10M band with deterministic indexing. | +| Embedding-free architecture | The engine preserves strong 10M candidate discovery with deterministic indexing. | Next lift areas: @@ -92,6 +94,24 @@ Next lift areas: The next practical upgrade is a deterministic reranker over the top-100 candidate set: evidence-set coverage, recency/order features, intent-specific scoring, and conservative CueBridge artifacts. +## Retrieved Context Footprint + +Each BEAM JSON result now records `ctx_tokens` for the top-20 memory text for +every scored question, matching the benchmark's primary Hit@20 metric. This is +an approximate, model-agnostic token count (using the same word/punctuation +estimator as the LoCoMo harness), not the model tokenizer. The evaluator also +prints aggregate `Avg`, `P50`, `P95`, `P99`, and `Max` values, plus average/P95 +values by question type. `ctx_tokens_returned` retains the count for the full +returned limit (normally top-100) for diagnostic comparison. When CueBridge +comparison is enabled, the raw baseline counts are retained under +`raw_result.ctx_tokens` and `raw_result.ctx_tokens_returned`. + +| Run | Avg | P50 | P95 | P99 | Max | +|---|---:|---:|---:|---:|---:| +| Latest raw 128K, top-20 | 4,702 | 2,816 | 14,889 | 20,954 | 34,393 | +| Latest raw 1M, top-20 | 2,934 | 1,354 | 11,554 | 20,731 | 29,243 | +| Latest raw 10M, top-20 | 1,749 | 1,220 | 4,170 | 16,802 | 19,733 | + ## Reproduce Start CueMap first: @@ -106,6 +126,10 @@ Raw 128K: CONTEXT=128k bash evals/beam/run_beam.sh ``` +The wrapper defaults to message-level ingestion (`cuemap add` / `/memories`), +which preserves one indexed memory per BEAM turn. To reproduce the older +segmented long-form path explicitly, set `INGEST_MODE=long-form`. + Raw 1M: ```bash @@ -136,7 +160,8 @@ Useful knobs: |---|---:|---| | `CONTEXT` | `128k` | BEAM tier: `128k`, `500k`, `1m`, or `10m`. | | `LIMIT` | `100` | Recall limit; BEAM reports include @50 and @100. | +| `SEMANTIC_MODE` | `hybrid` | Retrieval mode: `lexical`, `semantic`, or `hybrid`. | | `MODE` | `raw` | `raw`, `product-cuebridge`, or `question-oracle`. | | `DELETE_PROJECTS` | `1` | Delete temporary eval projects after each record. | -The wrapper writes fresh output under `evals/beam/results/` by default. The metrics above came from the v0.7 release calibration runs for `128k`, `1m`, and `10m`. +The wrapper writes fresh output under `evals/beam/results/` by default. The figures above come from the latest v0.7.2 raw runs for all three tiers. diff --git a/evals/beam/run_beam.sh b/evals/beam/run_beam.sh old mode 100644 new mode 100755 index 1323eac..c657346 --- a/evals/beam/run_beam.sh +++ b/evals/beam/run_beam.sh @@ -3,9 +3,26 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CUEMAP_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +CUEMAP_ENGINE_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CUEMAP_EVALS_DIR="${CUEMAP_EVALS_DIR:-$CUEMAP_ROOT/evals}" HARNESS="$CUEMAP_EVALS_DIR/test_beam_settled.py" +# Pin every subprocess in this evaluation to the release engine. The +# harness uses both HTTP and CLI fallbacks, so relying on whatever `cuemap` +# happens to be on PATH can silently mix engine versions. +CUEMAP_RUST_BIN="${CUEMAP_RUST_BIN:-$CUEMAP_ENGINE_ROOT/target/release/cuemap}" +if [[ "$CUEMAP_RUST_BIN" != */* ]]; then + CUEMAP_RUST_BIN="$(command -v "$CUEMAP_RUST_BIN" || true)" +fi +if [[ -z "$CUEMAP_RUST_BIN" || ! -x "$CUEMAP_RUST_BIN" ]]; then + echo "CueMap release binary not found or not executable: ${CUEMAP_RUST_BIN:-}" >&2 + echo "Build it with: cargo build --release --manifest-path $CUEMAP_ENGINE_ROOT/Cargo.toml" >&2 + exit 1 +fi +export CUEMAP_RUST_BIN +CUEMAP_BIN_DIR="${CUEMAP_RUST_BIN%/*}" +export PATH="$CUEMAP_BIN_DIR:$PATH" + if [[ ! -f "$HARNESS" ]]; then echo "Missing BEAM harness: $HARNESS" >&2 echo "Set CUEMAP_EVALS_DIR to the directory containing test_beam_settled.py." >&2 @@ -16,23 +33,59 @@ CONTEXT="${CONTEXT:-128k}" CUEMAP_URL="${CUEMAP_URL:-http://127.0.0.1:8080}" LIMIT="${LIMIT:-100}" MODE="${MODE:-raw}" +SEMANTIC_MODE="${SEMANTIC_MODE:-hybrid}" +INGEST_MODE="${INGEST_MODE:-message}" +ORDERED_RECONSTRUCTION="${ORDERED_RECONSTRUCTION:-off}" +EVIDENCE_COVERAGE="${EVIDENCE_COVERAGE:-off}" OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results}" OUTPUT="${OUTPUT:-$OUT_DIR/beam_${CONTEXT}_${MODE}.json}" +TRACE_TIMING="${TRACE_TIMING:-0}" +TIMING_FILE="${TIMING_FILE:-$OUT_DIR/beam_${CONTEXT}_${MODE}_${SEMANTIC_MODE}_timing_$(date +%Y%m%d_%H%M%S).jsonl}" + +case "$SEMANTIC_MODE" in + lexical|semantic|hybrid) + export CUEMAP_SEMANTIC_MODE="$SEMANTIC_MODE" + ;; + *) + echo "Unknown SEMANTIC_MODE=$SEMANTIC_MODE. Use lexical, semantic, or hybrid." >&2 + exit 1 + ;; +esac + +case "$INGEST_MODE" in + message) + ;; + long-form) + ;; + *) + echo "Unknown INGEST_MODE=$INGEST_MODE. Use message or long-form." >&2 + exit 1 + ;; +esac mkdir -p "$OUT_DIR" +if [[ "$TRACE_TIMING" == "1" ]]; then + : >"$TIMING_FILE" + export CUEMAP_TRACE_TIMING=1 + export CUEMAP_TIMING_FILE="$TIMING_FILE" +fi + args=( python "$HARNESS" --context "$CONTEXT" --url "$CUEMAP_URL" --limit "$LIMIT" - --ingest-long-form - --ordered-reconstruction "${ORDERED_RECONSTRUCTION:-auto}" - --evidence-coverage "${EVIDENCE_COVERAGE:-off}" + --ordered-reconstruction "$ORDERED_RECONSTRUCTION" + --evidence-coverage "$EVIDENCE_COVERAGE" --no-auto-reinforce --output "$OUTPUT" ) +if [[ "$INGEST_MODE" == "long-form" ]]; then + args+=(--ingest-long-form) +fi + if [[ "${DELETE_PROJECTS:-1}" == "1" ]]; then args+=(--delete-project-after-record) fi @@ -95,6 +148,22 @@ if [[ "$MODE" != "raw" ]]; then args+=(--cuebridge-max-fix-cases "${CUEBRIDGE_MAX_FIX_CASES:-1000}") fi -echo "Running BEAM $CONTEXT in $MODE mode" +echo "Running BEAM $CONTEXT in $MODE mode with $SEMANTIC_MODE retrieval" +echo "Ingestion mode: $INGEST_MODE" +echo "Ordered reconstruction: $ORDERED_RECONSTRUCTION | Evidence coverage: $EVIDENCE_COVERAGE" +echo "CueMap binary: $CUEMAP_RUST_BIN ($("$CUEMAP_RUST_BIN" --version))" echo "Output: $OUTPUT" -exec "${args[@]}" + +run_status=0 +if "${args[@]}"; then + run_status=0 +else + run_status=$? +fi + +if [[ "$TRACE_TIMING" == "1" && -s "$TIMING_FILE" ]]; then + python "$SCRIPT_DIR/report_timing.py" --input "$TIMING_FILE" + echo "Timing samples: $TIMING_FILE" +fi + +exit "$run_status" diff --git a/evals/locomo/report.md b/evals/locomo/report.md index c7e69cf..9432f36 100644 --- a/evals/locomo/report.md +++ b/evals/locomo/report.md @@ -1,6 +1,6 @@ # LoCoMo: Long Conversation Recall With Low Context -`CueMap v0.7` `conversation memory` `context expansion` +`CueMap v0.7.2` `conversation memory` `context expansion` LoCoMo stresses long conversations where the answer evidence is often near the recalled turn rather than exactly inside it. CueMap handles this best with engine-level expansion depth, which is a real product feature: return the matching memory plus adjacent conversation turns so an answer model receives the local neighborhood. @@ -8,10 +8,13 @@ LoCoMo stresses long conversations where the answer evidence is often near the r | Setting | Questions | Hit@1 | Hit@5 | Hit@10 | Hit@20 | Avg ctx tokens | |---|---:|---:|---:|---:|---:|---:| -| Lean context, expansion-depth 3 | 1,986 | 50.3% | 75.3% | 83.1% | 89.2% | 3,362 | -| Strong context, expansion-depth 10 | 1,986 | 56.8% | 80.5% | 88.2% | 93.2% | 10,279 | +| Lean context, expansion-depth 3 | 1,986 | 62.9% | 85.3% | 90.4% | 94.2% | 3,184 | +| Strong context, expansion-depth 10 | 1,986 | 66.1% | 87.7% | 92.5% | 96.1% | 10,044 | -The strong setting crosses 93% Hit@20 while still using far less context than many leaderboard-style RAG reports. As a reference point, public Agent Memory Benchmark views show some LoCoMo systems in the 14.7K to 36.2K context-token range. CueMap's lean run was 3.4K average tokens, and the stronger run was 10.3K. +The strong setting reaches 96.1% Hit@20 while still using far less context than many leaderboard-style RAG reports. As a reference point, public Agent Memory Benchmark views show some LoCoMo systems in the 14.7K to 36.2K context-token range. CueMap's lean run averages 3.2K tokens, and the stronger run averages 10.0K. + +Both reported runs use the wrapper's default **hybrid recall mode** +(`SEMANTIC_MODE=hybrid`), combining lexical and semantic retrieval signals. ## Strong Setting Metrics @@ -19,26 +22,46 @@ Settings: `limit=20`, `expansion-depth=10`. | Metric | Score | |---|---:| -| Recall_Any@1 | 56.8% | -| Recall_Any@5 | 80.5% | -| Recall_Any@10 | 88.2% | -| Recall_Any@20 | 93.2% | -| Recall_All@5 | 71.7% | -| Recall_All@10 | 79.4% | -| Recall_All@20 | 85.5% | -| Recall_Frac@5 | 75.6% | -| Recall_Frac@10 | 83.8% | -| Recall_Frac@20 | 89.5% | -| NDCG@5 | 66.2% | -| NDCG@10 | 69.2% | -| NDCG@20 | 70.9% | +| Recall_Any@1 | 66.1% | +| Recall_Any@5 | 87.7% | +| Recall_Any@10 | 92.5% | +| Recall_Any@20 | 96.1% | +| Recall_All@5 | 77.8% | +| Recall_All@10 | 83.6% | +| Recall_All@20 | 88.9% | +| Recall_Frac@5 | 82.5% | +| Recall_Frac@10 | 88.1% | +| Recall_Frac@20 | 92.7% | +| NDCG@5 | 74.6% | +| NDCG@10 | 76.6% | +| NDCG@20 | 78.0% | + +## Lean Setting Metrics + +Settings: `limit=20`, `expansion-depth=3`. + +| Metric | Score | +|---|---:| +| Recall_Any@1 | 62.9% | +| Recall_Any@5 | 85.3% | +| Recall_Any@10 | 90.4% | +| Recall_Any@20 | 94.2% | +| Recall_All@5 | 76.0% | +| Recall_All@10 | 81.6% | +| Recall_All@20 | 86.3% | +| Recall_Frac@5 | 80.3% | +| Recall_Frac@10 | 86.0% | +| Recall_Frac@20 | 90.4% | +| NDCG@5 | 72.0% | +| NDCG@10 | 74.0% | +| NDCG@20 | 75.3% | ## Context Footprint | Setting | Avg | P50 | P95 | P99 | Max | |---|---:|---:|---:|---:|---:| -| Expansion-depth 3 | 3,362 | 3,395 | 4,047 | 4,272 | 4,926 | -| Expansion-depth 10 | 10,279 | 10,269 | 12,248 | 12,981 | 13,656 | +| Expansion-depth 3 | 3,184 | 3,242 | 3,950 | 4,244 | 5,689 | +| Expansion-depth 10 | 10,044 | 10,136 | 12,093 | 12,789 | 13,749 | This is the main CueMap story on LoCoMo: competitive recall using compact retrieved neighborhoods instead of full-conversation context. @@ -46,11 +69,11 @@ This is the main CueMap story on LoCoMo: competitive recall using compact retrie | Type | Count | Hit@1 | Hit@5 | Hit@10 | Hit@20 | Read | |---|---:|---:|---:|---:|---:|---| -| Multi-hop | 321 | 52.0% | 74.8% | 83.8% | 90.7% | Solid with local context. | -| Temporal reasoning | 96 | 30.2% | 56.2% | 63.5% | 78.1% | Strong candidate base for time-aware ranking lift. | -| Single-hop | 282 | 39.7% | 69.9% | 83.3% | 89.7% | Good Hit@20 with top-rank upside. | -| Common-sense | 841 | 63.5% | 85.3% | 92.2% | 96.0% | Strong. | -| Adversarial | 446 | 64.3% | 87.4% | 92.4% | 95.3% | Strong retrieval that gives the answer model evidence for abstention. | +| Multi-hop | 321 | 65.1% | 85.7% | 90.3% | 95.0% | Solid with local context. | +| Temporal reasoning | 96 | 29.2% | 54.2% | 64.6% | 80.2% | Strong candidate base for time-aware ranking lift. | +| Single-hop | 282 | 49.6% | 81.2% | 90.1% | 96.8% | Good Hit@20 with top-rank upside. | +| Common-sense | 841 | 73.2% | 91.7% | 95.2% | 97.1% | Strong. | +| Adversarial | 446 | 71.7% | 92.8% | 96.4% | 98.0% | Strong retrieval that gives the answer model evidence for abstention. | ## Product Read @@ -94,6 +117,7 @@ Useful knobs: |---|---:|---| | `EXPANSION_DEPTH` | `10` | How many adjacent conversation turns to attach around recalled memories. | | `LIMIT` | `20` | Recall limit. | +| `SEMANTIC_MODE` | `hybrid` | Retrieval mode: `lexical`, `semantic`, or `hybrid`. | | `DELETE_PROJECTS` | `1` | Delete temporary eval projects after each record. | -The wrapper writes fresh output under `evals/locomo/results/` by default. The metrics above came from the v0.7 release calibration run. +The wrapper writes fresh output under `evals/locomo/results/` by default. The metrics above came from the full 1,986-question v0.7.2 rerun on 2026-08-15. diff --git a/evals/locomo/run_locomo.sh b/evals/locomo/run_locomo.sh index 6d48220..3315473 100644 --- a/evals/locomo/run_locomo.sh +++ b/evals/locomo/run_locomo.sh @@ -3,9 +3,25 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CUEMAP_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +CUEMAP_ENGINE_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CUEMAP_EVALS_DIR="${CUEMAP_EVALS_DIR:-$CUEMAP_ROOT/evals}" HARNESS="$CUEMAP_EVALS_DIR/test_locomo_settled.py" +# Pin the CLI used by the Python harness. Without this, LoCoMo can recall +# through an unrelated globally installed `cuemap` binary. +CUEMAP_RUST_BIN="${CUEMAP_RUST_BIN:-$CUEMAP_ENGINE_ROOT/target/release/cuemap}" +if [[ "$CUEMAP_RUST_BIN" != */* ]]; then + CUEMAP_RUST_BIN="$(command -v "$CUEMAP_RUST_BIN" || true)" +fi +if [[ -z "$CUEMAP_RUST_BIN" || ! -x "$CUEMAP_RUST_BIN" ]]; then + echo "CueMap release binary not found or not executable: ${CUEMAP_RUST_BIN:-}" >&2 + echo "Build it with: cargo build --release --manifest-path $CUEMAP_ENGINE_ROOT/Cargo.toml" >&2 + exit 1 +fi +export CUEMAP_RUST_BIN +CUEMAP_BIN_DIR="${CUEMAP_RUST_BIN%/*}" +export PATH="$CUEMAP_BIN_DIR:$PATH" + if [[ ! -f "$HARNESS" ]]; then echo "Missing LoCoMo harness: $HARNESS" >&2 echo "Set CUEMAP_EVALS_DIR to the directory containing test_locomo_settled.py." >&2 @@ -16,9 +32,20 @@ CUEMAP_URL="${CUEMAP_URL:-http://127.0.0.1:8080}" LIMIT="${LIMIT:-20}" EXPANSION_DEPTH="${EXPANSION_DEPTH:-10}" MODE="${MODE:-raw}" +SEMANTIC_MODE="${SEMANTIC_MODE:-hybrid}" OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results}" OUTPUT="${OUTPUT:-$OUT_DIR/locomo_${MODE}_depth${EXPANSION_DEPTH}.json}" +case "$SEMANTIC_MODE" in + lexical|semantic|hybrid) + export CUEMAP_SEMANTIC_MODE="$SEMANTIC_MODE" + ;; + *) + echo "Unknown SEMANTIC_MODE=$SEMANTIC_MODE. Use lexical, semantic, or hybrid." >&2 + exit 1 + ;; +esac + mkdir -p "$OUT_DIR" args=( @@ -69,5 +96,7 @@ esac echo "Running LoCoMo in $MODE mode" echo "Expansion depth: $EXPANSION_DEPTH" +echo "Retrieval mode: $SEMANTIC_MODE" +echo "CueMap binary: $CUEMAP_RUST_BIN ($("$CUEMAP_RUST_BIN" --version))" echo "Output: $OUTPUT" exec "${args[@]}" diff --git a/evals/longmemeval/fast_longmemeval.py b/evals/longmemeval/fast_longmemeval.py new file mode 100644 index 0000000..a9ea6c8 --- /dev/null +++ b/evals/longmemeval/fast_longmemeval.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Run the canonical LongMemEval harness with BEAM-style ingestion. + +The canonical harness uses ``cuemap add`` once per message. This adapter keeps +its recall and scoring logic intact while replacing only those client calls +with direct HTTP POSTs to ``/ingest/content``, matching the BEAM harness. The +server queues the write and the canonical harness still waits for the project +to settle before recall. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +_HARNESS_ENV = "CUEMAP_LONGMEMEVAL_HARNESS" +_TIMING_MARKER = "--- TIMING ---" + + +def _load_harness(): + harness_path = os.environ.get(_HARNESS_ENV) + if not harness_path: + raise RuntimeError(f"{_HARNESS_ENV} must point to test_longmemeval_settled.py") + + path = Path(harness_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"LongMemEval harness not found: {path}") + + # The canonical harness imports sibling evaluation utilities. When it is + # executed directly, Python adds its parent directory automatically; an + # adapter loaded from rust_engine needs to reproduce that behavior. + harness_parent = str(path.parent) + if harness_parent not in sys.path: + sys.path.insert(0, harness_parent) + + spec = importlib.util.spec_from_file_location("cuemap_longmemeval_harness", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load LongMemEval harness: {path}") + + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _parse_add_command(cmd: list[str]) -> tuple[str, str, dict[str, Any], str, bool] | None: + if len(cmd) < 3 or Path(cmd[0]).name != "cuemap" or cmd[1] != "add": + return None + + try: + project_idx = next(i for i, value in enumerate(cmd) if value in {"-p", "--project"}) + url_idx = cmd.index("--url") + metadata_idx = cmd.index("--metadata") + project_id = cmd[project_idx + 1] + url = cmd[url_idx + 1] + metadata = json.loads(cmd[metadata_idx + 1]) + content = cmd[-1] + except (StopIteration, IndexError, ValueError, json.JSONDecodeError): + return None + + if not isinstance(metadata, dict): + metadata = {} + + return ( + project_id, + url, + metadata, + content, + "--disable-temporal-chunking" in cmd, + ) + + +def _cue_value(value: Any) -> str: + normalized = re.sub(r"[^a-zA-Z0-9]+", "_", str(value).strip().lower()).strip("_") + return normalized or "unknown" + + +def _post_ingest( + *, + url: str, + project_id: str, + message_index: int, + metadata: dict[str, Any], + content: str, + disable_temporal_chunking: bool, + timeout: int = 60, +) -> dict[str, Any]: + source_key = f"longmemeval:{project_id}:message:{message_index}" + structural_cues = ["source_type:chat_message"] + if metadata.get("source_role"): + structural_cues.append(f"source_role:{_cue_value(metadata['source_role'])}") + if metadata.get("source_date"): + structural_cues.append(f"source_date:{_cue_value(metadata['source_date'])}") + + # Use a window larger than the message's sentence count and a max size + # larger than its content so each LongMemEval message remains one memory, + # while still taking the same /ingest/content route as BEAM. + sentence_window = max(1, len(content) + 1) + max_chunk_chars = max(2000, len(content) + 1) + payload = { + "content": content, + "filename": f"longmemeval_{project_id}_{message_index}.txt", + "source_key": source_key, + "metadata": metadata, + "structural_cues": structural_cues, + "segmenter": "sentence_window", + "segment_window_size": sentence_window, + "segment_overlap": 0, + "segment_min_chunk_chars": 1, + "segment_max_chunk_chars": max_chunk_chars, + } + # The ingest/content endpoint does not currently expose + # disable_temporal_chunking. The canonical LongMemEval harness does not + # set the flag in normal runs; keep parsing it only for command shape + # compatibility without changing user metadata. + _ = disable_temporal_chunking + + endpoint = f"{url.rstrip('/')}/ingest/content" + request = Request( + endpoint, + data=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "X-Project-ID": project_id, + }, + method="POST", + ) + + try: + with urlopen(request, timeout=timeout) as response: + body = response.read().decode("utf-8") + if response.status < 200 or response.status >= 300: + raise RuntimeError( + f"POST {endpoint} failed with HTTP {response.status}: {body}" + ) + return json.loads(body) if body else {} + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"POST {endpoint} failed with HTTP {exc.code}: {body}") from exc + except URLError as exc: + raise RuntimeError(f"POST {endpoint} failed: {exc}") from exc + + +def _timing_enabled() -> bool: + return os.environ.get("CUEMAP_TRACE_TIMING") == "1" and bool( + os.environ.get("CUEMAP_TIMING_FILE") + ) + + +def _append_timing_record(stdout: str) -> None: + marker_index = stdout.find(_TIMING_MARKER) + if marker_index < 0: + return + + timing_text = stdout[marker_index + len(_TIMING_MARKER) :].strip() + try: + timing = json.loads(timing_text) + except json.JSONDecodeError: + return + if not isinstance(timing, dict): + return + + timing_path = Path(os.environ["CUEMAP_TIMING_FILE"]).expanduser() + timing_path.parent.mkdir(parents=True, exist_ok=True) + with timing_path.open("a", encoding="utf-8") as handle: + json.dump(timing, handle, separators=(",", ":")) + handle.write("\n") + + +def main() -> None: + harness = _load_harness() + original_run_cmd = harness.run_cmd + message_indices: dict[tuple[str, str], int] = {} + + def fast_run_cmd(cmd: list[str], *, check: bool = False) -> subprocess.CompletedProcess: + parsed = _parse_add_command(cmd) + is_recall = len(cmd) >= 2 and Path(cmd[0]).name == "cuemap" and cmd[1] == "recall" + if parsed is None and not is_recall: + return original_run_cmd(cmd, check=check) + + if parsed is None: + recall_cmd = list(cmd) + if "--semantic-mode" not in recall_cmd: + recall_cmd.extend([ + "--semantic-mode", + os.environ.get("CUEMAP_SEMANTIC_MODE", "hybrid"), + ]) + if _timing_enabled() and "--trace-timing" not in recall_cmd: + recall_cmd.append("--trace-timing") + result = original_run_cmd(recall_cmd, check=check) + if _timing_enabled() and result.returncode == 0: + _append_timing_record(result.stdout) + return result + + project_id, url, metadata, content, disable_temporal_chunking = parsed + key = (url, project_id) + message_index = message_indices.get(key, 0) + message_indices[key] = message_index + 1 + _post_ingest( + url=url, + project_id=project_id, + message_index=message_index, + metadata=metadata, + content=content, + disable_temporal_chunking=disable_temporal_chunking, + ) + return subprocess.CompletedProcess( + cmd, + 0, + stdout="✓ Memory queued\n", + stderr="", + ) + + harness.run_cmd = fast_run_cmd + harness.evaluate() + + +if __name__ == "__main__": + main() diff --git a/evals/longmemeval/report.md b/evals/longmemeval/report.md index 543eaa3..c6830b0 100644 --- a/evals/longmemeval/report.md +++ b/evals/longmemeval/report.md @@ -1,30 +1,45 @@ # LongMemEval: Compact Long-Memory Recall -`CueMap v0.7` `raw lexical recall` `near-saturated Hit@20` +`CueMap v0.7.2` `raw hybrid recall` `near-saturated Hit@20` -LongMemEval is the cleanest showcase for CueMap's raw engine: compact user memories, compact retrieval depth, and a strong deterministic recall path. CueMap reaches near-saturated Hit@20 while keeping the recall path embedding-free and LLM-free. +LongMemEval is the cleanest showcase for CueMap's raw engine: compact user memories, compact retrieval depth, and a strong deterministic recall path. CueMap reaches near-saturated Hit@20 while keeping the recall path embedding-free and LLM-free. The latest run uses the wrapper default `SEMANTIC_MODE=hybrid`, combining lexical and semantic retrieval signals. ## Headline | Run | Questions | Hit@1 | Hit@5 | Hit@10 | Hit@20 | NDCG@20 | |---|---:|---:|---:|---:|---:|---:| -| Raw CueMap | 470 | 55.1% | 92.1% | 98.1% | 99.1% | 74.2% | +| Raw CueMap · hybrid | 470 | 57.9% | 90.2% | 94.9% | 96.2% | 75.1% | The important signal is that the raw engine is already close to saturated at Hit@20. LongMemEval is the raw-recall showcase; BEAM 128K is the stronger CueBridge-lift showcase. ## Quality Metrics -| Metric | Raw | +| Metric | Raw hybrid | |---|---:| -| Recall_All@5 | 65.3% | -| Recall_All@10 | 83.0% | -| Recall_All@20 | 90.4% | -| Recall_Frac@5 | 78.5% | -| Recall_Frac@10 | 91.2% | -| Recall_Frac@20 | 95.7% | -| NDCG@5 | 67.4% | -| NDCG@10 | 72.6% | -| NDCG@20 | 74.2% | +| Recall_All@5 | 73.0% | +| Recall_All@10 | 82.1% | +| Recall_All@20 | 88.1% | +| Recall_Frac@5 | 81.8% | +| Recall_Frac@10 | 88.8% | +| Recall_Frac@20 | 92.9% | +| NDCG@5 | 70.8% | +| NDCG@10 | 73.7% | +| NDCG@20 | 75.1% | + +## Question-Type Breakdown + +The run excluded 30 abstention cases before scoring. The remaining 470 +questions break down as follows; context figures are approximate top-20 token +footprints for each type. + +| Question type | Questions | Hit@1 | Hit@5 | Hit@10 | Hit@20 | Avg ctx | P95 ctx | +|---|---:|---:|---:|---:|---:|---:|---:| +| Single-session user | 64 | 75.0% | 98.4% | 98.4% | 98.4% | 4,338 | 6,965 | +| Multi-session | 121 | 62.8% | 90.1% | 95.9% | 95.9% | 3,969 | 6,027 | +| Single-session preference | 30 | 30.0% | 66.7% | 86.7% | 86.7% | 3,543 | 5,338 | +| Temporal reasoning | 127 | 59.1% | 88.2% | 91.3% | 91.3% | 3,799 | 5,560 | +| Knowledge update | 72 | 61.1% | 98.6% | 100.0% | 100.0% | 4,266 | 6,645 | +| Single-session assistant | 56 | 35.7% | 87.5% | 94.6% | 94.6% | 3,318 | 5,941 | ## Product Read @@ -38,6 +53,18 @@ Next lift areas: | Preference questions | Add more semantic paraphrase bridges through CueBridge. | | Top-1 ranking | Convert more of the excellent Hit@20 performance into first-result precision. | +## Retrieved Context Footprint + +Each LongMemEval JSON result now records `ctx_tokens` for the selected recall +attempt and `raw_ctx_tokens` for the raw baseline. These are approximate, +model-agnostic counts of the returned memory text, using the same +word/punctuation estimator as LoCoMo. The evaluator prints aggregate `Avg`, +`P50`, `P95`, `P99`, and `Max` values, plus average/P95 values by question type. + +| Run | Avg | P50 | P95 | P99 | Max | +|---|---:|---:|---:|---:|---:| +| Latest raw hybrid, top-20 | 3,914 | 3,771 | 6,263 | 7,250 | 10,812 | + ## Reproduce Start CueMap first: @@ -58,7 +85,8 @@ Useful knobs: |---|---:|---| | `LIMIT` | `20` | Recall limit used for scoring. | | `VARIANT` | `core` | LongMemEval variant. | +| `SEMANTIC_MODE` | `hybrid` | Retrieval mode: `lexical`, `semantic`, or `hybrid`. | | `DELETE_PROJECTS` | `1` | Delete temporary eval projects after each record. | | `MODE` | `raw` | `raw`, `question-oracle`, or `product-cuebridge`; BEAM is the recommended CueBridge showcase. | -The wrapper writes fresh output under `evals/longmemeval/results/` by default. The metrics above came from the v0.7 release calibration run. +The wrapper writes fresh output under `evals/longmemeval/results/` by default. The metrics above came from the latest v0.7.2 raw hybrid run: 470 scored questions with 30 abstention cases excluded. diff --git a/evals/longmemeval/run_longmemeval.sh b/evals/longmemeval/run_longmemeval.sh index 116811f..0e0615c 100644 --- a/evals/longmemeval/run_longmemeval.sh +++ b/evals/longmemeval/run_longmemeval.sh @@ -3,9 +3,24 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CUEMAP_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +CUEMAP_ENGINE_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" CUEMAP_EVALS_DIR="${CUEMAP_EVALS_DIR:-$CUEMAP_ROOT/evals}" HARNESS="$CUEMAP_EVALS_DIR/test_longmemeval_settled.py" +# Pin both the adapter and canonical harness CLI calls to the release engine. +CUEMAP_RUST_BIN="${CUEMAP_RUST_BIN:-$CUEMAP_ENGINE_ROOT/target/release/cuemap}" +if [[ "$CUEMAP_RUST_BIN" != */* ]]; then + CUEMAP_RUST_BIN="$(command -v "$CUEMAP_RUST_BIN" || true)" +fi +if [[ -z "$CUEMAP_RUST_BIN" || ! -x "$CUEMAP_RUST_BIN" ]]; then + echo "CueMap release binary not found or not executable: ${CUEMAP_RUST_BIN:-}" >&2 + echo "Build it with: cargo build --release --manifest-path $CUEMAP_ENGINE_ROOT/Cargo.toml" >&2 + exit 1 +fi +export CUEMAP_RUST_BIN +CUEMAP_BIN_DIR="${CUEMAP_RUST_BIN%/*}" +export PATH="$CUEMAP_BIN_DIR:$PATH" + if [[ ! -f "$HARNESS" ]]; then echo "Missing LongMemEval harness: $HARNESS" >&2 echo "Set CUEMAP_EVALS_DIR to the directory containing test_longmemeval_settled.py." >&2 @@ -15,13 +30,33 @@ fi CUEMAP_URL="${CUEMAP_URL:-http://127.0.0.1:8080}" LIMIT="${LIMIT:-20}" MODE="${MODE:-raw}" +SEMANTIC_MODE="${SEMANTIC_MODE:-hybrid}" OUT_DIR="${OUT_DIR:-$SCRIPT_DIR/results}" OUTPUT="${OUTPUT:-$OUT_DIR/longmemeval_${MODE}.json}" +TRACE_TIMING="${TRACE_TIMING:-0}" +TIMING_FILE="${TIMING_FILE:-$OUT_DIR/longmemeval_${MODE}_${SEMANTIC_MODE}_timing_$(date +%Y%m%d_%H%M%S).jsonl}" + +case "$SEMANTIC_MODE" in + lexical|semantic|hybrid) + export CUEMAP_SEMANTIC_MODE="$SEMANTIC_MODE" + ;; + *) + echo "Unknown SEMANTIC_MODE=$SEMANTIC_MODE. Use lexical, semantic, or hybrid." >&2 + exit 1 + ;; +esac mkdir -p "$OUT_DIR" +if [[ "$TRACE_TIMING" == "1" ]]; then + : >"$TIMING_FILE" + export CUEMAP_TRACE_TIMING=1 + export CUEMAP_TIMING_FILE="$TIMING_FILE" +fi + args=( - python "$HARNESS" + python + "${SCRIPT_DIR}/fast_longmemeval.py" --url "$CUEMAP_URL" --limit "$LIMIT" --variant "${VARIANT:-core}" @@ -29,6 +64,21 @@ args=( --output "$OUTPUT" ) +if [[ "${FAST_INGEST:-1}" == "1" ]]; then + export CUEMAP_LONGMEMEVAL_HARNESS="$HARNESS" + echo "Ingestion transport: direct /ingest/content (BEAM-compatible)" +else + args=( + python "$HARNESS" + --url "$CUEMAP_URL" + --limit "$LIMIT" + --variant "${VARIANT:-core}" + --no-auto-reinforce + --output "$OUTPUT" + ) + echo "Ingestion transport: legacy per-memory cuemap add" +fi + if [[ "${DELETE_PROJECTS:-1}" == "1" ]]; then args+=(--delete-project-after-record) fi @@ -76,5 +126,20 @@ case "$MODE" in esac echo "Running LongMemEval in $MODE mode" +echo "Retrieval mode: $SEMANTIC_MODE" +echo "CueMap binary: $CUEMAP_RUST_BIN ($("$CUEMAP_RUST_BIN" --version))" echo "Output: $OUTPUT" -exec "${args[@]}" + +run_status=0 +if "${args[@]}"; then + run_status=0 +else + run_status=$? +fi + +if [[ "$TRACE_TIMING" == "1" && -s "$TIMING_FILE" ]]; then + python "$CUEMAP_ROOT/evals/beam/report_timing.py" --input "$TIMING_FILE" + echo "Timing samples: $TIMING_FILE" +fi + +exit "$run_status" diff --git a/scripts/build-npm-native-packages.sh b/scripts/build-npm-native-packages.sh new file mode 100755 index 0000000..83636e6 --- /dev/null +++ b/scripts/build-npm-native-packages.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DIST_DIR="${DIST_DIR:-${ROOT_DIR}/dist/npm-native}" +VERSION="${VERSION:-$(awk -F '"' '/^version = "/ { print $2; exit }' "${ROOT_DIR}/Cargo.toml")}" +TOKENIZER_URL="${TOKENIZER_URL:-https://cuemap.dev/assets/en_tokenizer.bin.gz}" +TOKENIZER_SHA256="${TOKENIZER_SHA256:-f54fd31ec463f8646d0239bb531a64e0210ed1ae02bf5e3b42aeeb9bff8305ba}" + +if [[ "$(uname -s)" != "Darwin" ]]; then + echo "This release builder requires macOS to produce both Darwin binaries" >&2 + exit 1 +fi + +if [[ -z "${VERSION}" ]]; then + echo "Could not read the package version from Cargo.toml" >&2 + exit 1 +fi + +for command in cargo curl docker gzip node npm rustup shasum; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +if ! rustup target list --installed | grep -qx "x86_64-apple-darwin"; then + rustup target add x86_64-apple-darwin +fi + +rm -rf "${DIST_DIR}" +mkdir -p "${DIST_DIR}/binaries" "${DIST_DIR}/packages" "${DIST_DIR}/tarballs" "${DIST_DIR}/tokenizer" + +echo "Downloading checksum-pinned tokenizer" +curl -fsSL --retry 3 "${TOKENIZER_URL}" -o "${DIST_DIR}/tokenizer/en_tokenizer.bin.gz" +actual_tokenizer_sha="$(shasum -a 256 "${DIST_DIR}/tokenizer/en_tokenizer.bin.gz" | awk '{print $1}')" +if [[ "${actual_tokenizer_sha}" != "${TOKENIZER_SHA256}" ]]; then + echo "Tokenizer checksum mismatch: expected ${TOKENIZER_SHA256}, got ${actual_tokenizer_sha}" >&2 + exit 1 +fi +gzip -dc "${DIST_DIR}/tokenizer/en_tokenizer.bin.gz" > "${DIST_DIR}/tokenizer/en_tokenizer.bin" + +echo "Building Darwin ARM64 binary" +cargo build --manifest-path "${ROOT_DIR}/Cargo.toml" --locked --release --target aarch64-apple-darwin +cp "${ROOT_DIR}/target/aarch64-apple-darwin/release/cuemap" "${DIST_DIR}/binaries/cuemap-darwin-arm64" + +echo "Building Darwin x64 binary" +cargo build --manifest-path "${ROOT_DIR}/Cargo.toml" --locked --release --target x86_64-apple-darwin +cp "${ROOT_DIR}/target/x86_64-apple-darwin/release/cuemap" "${DIST_DIR}/binaries/cuemap-darwin-x64" + +echo "Building Linux x64 binary on Debian Trixie" +linux_output="${DIST_DIR}/linux-x64-output" +docker buildx build \ + --platform linux/amd64 \ + --target native-binary \ + --output "type=local,dest=${linux_output}" \ + "${ROOT_DIR}" +cp "${linux_output}/cuemap" "${DIST_DIR}/binaries/cuemap-linux-x64" +rm -rf "${linux_output}" + +write_package_json() { + local output_path="$1" + local package_name="$2" + local os_name="$3" + local cpu_name="$4" + + node -e ' + const fs = require("node:fs"); + const [output, name, version, os, cpu] = process.argv.slice(1); + const manifest = { + name, + version, + description: `Pre-compiled CueMap Engine for ${os} ${cpu}`, + engines: { node: ">=18" }, + os: [os], + cpu: [cpu], + bin: { cuemap: "bin/cuemap" }, + files: ["bin", "assets", "README.md", "LICENSE"], + repository: { type: "git", url: "https://github.com/cuemap-dev/cuemap.git" }, + author: "Kaan Demirel", + license: "BSL-1.1", + publishConfig: { access: "public" }, + }; + if (os === "linux") manifest.libc = ["glibc"]; + fs.writeFileSync(output, `${JSON.stringify(manifest, null, 2)}\n`); + ' "${output_path}" "${package_name}" "${VERSION}" "${os_name}" "${cpu_name}" +} + +stage_package() { + local platform="$1" + local os_name="$2" + local cpu_name="$3" + local binary_path="${DIST_DIR}/binaries/cuemap-${platform}" + local package_name="@cuemap-dev/engine-${platform}" + local package_dir="${DIST_DIR}/packages/engine-${platform}" + + mkdir -p "${package_dir}/bin" "${package_dir}/assets" + cp "${ROOT_DIR}/scripts/npm-native-wrapper.cjs" "${package_dir}/bin/cuemap" + cp "${binary_path}" "${package_dir}/bin/cuemap-native" + cp "${DIST_DIR}/tokenizer/en_tokenizer.bin" "${package_dir}/assets/en_tokenizer.bin" + cp "${ROOT_DIR}/scripts/npm-native-README.md" "${package_dir}/README.md" + cp "${ROOT_DIR}/LICENSE" "${package_dir}/LICENSE" + chmod 0755 "${package_dir}/bin/cuemap" "${package_dir}/bin/cuemap-native" + write_package_json "${package_dir}/package.json" "${package_name}" "${os_name}" "${cpu_name}" + + npm pack "${package_dir}" --pack-destination "${DIST_DIR}/tarballs" +} + +stage_package "darwin-arm64" "darwin" "arm64" +stage_package "darwin-x64" "darwin" "x64" +stage_package "linux-x64" "linux" "x64" + +( + cd "${DIST_DIR}/tarballs" + shasum -a 256 ./*.tgz > SHA256SUMS +) + +echo +echo "Native npm packages are ready in ${DIST_DIR}/tarballs" diff --git a/scripts/npm-native-README.md b/scripts/npm-native-README.md new file mode 100644 index 0000000..514fda4 --- /dev/null +++ b/scripts/npm-native-README.md @@ -0,0 +1,5 @@ +# CueMap native engine + +This package contains the CueMap engine binary and English tokenizer for one operating-system and CPU combination. It is normally installed automatically by `cuemap-mcp`. + +The `cuemap` command launches the packaged binary and configures its tokenizer asset. Set `TOKENIZER_PATH` only when you intentionally want to use a different compiled tokenizer. diff --git a/scripts/npm-native-wrapper.cjs b/scripts/npm-native-wrapper.cjs new file mode 100755 index 0000000..851b27f --- /dev/null +++ b/scripts/npm-native-wrapper.cjs @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +const { spawn } = require("node:child_process"); +const path = require("node:path"); + +const packageRoot = path.resolve(__dirname, ".."); +const binaryName = process.platform === "win32" ? "cuemap-native.exe" : "cuemap-native"; +const binaryPath = path.join(__dirname, binaryName); +const tokenizerPath = path.join(packageRoot, "assets", "en_tokenizer.bin"); + +const child = spawn(binaryPath, process.argv.slice(2), { + stdio: "inherit", + env: { + ...process.env, + TOKENIZER_PATH: process.env.TOKENIZER_PATH || tokenizerPath, + }, +}); + +child.on("error", (error) => { + console.error(`Failed to start CueMap engine: ${error.message}`); + process.exit(1); +}); + +for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => { + if (!child.killed) child.kill(signal); + }); +} + +child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 1); +}); diff --git a/scripts/verify-npm-native-packages.sh b/scripts/verify-npm-native-packages.sh new file mode 100755 index 0000000..f89bfd9 --- /dev/null +++ b/scripts/verify-npm-native-packages.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DIST_DIR="${DIST_DIR:-${ROOT_DIR}/dist/npm-native}" +VERIFY_DIR="${DIST_DIR}/verify" + +rm -rf "${VERIFY_DIR}" +mkdir -p "${VERIFY_DIR}" + +( + cd "${DIST_DIR}/tarballs" + shasum -a 256 -c SHA256SUMS +) + +for platform in darwin-arm64 darwin-x64 linux-x64; do + tarball="$(find "${DIST_DIR}/tarballs" -maxdepth 1 -name "cuemap-dev-engine-${platform}-*.tgz" -print -quit)" + if [[ -z "${tarball}" ]]; then + echo "Missing tarball for ${platform}" >&2 + exit 1 + fi + + package_dir="${VERIFY_DIR}/${platform}" + mkdir -p "${package_dir}" + tar -xzf "${tarball}" -C "${package_dir}" + + test -x "${package_dir}/package/bin/cuemap" + test -x "${package_dir}/package/bin/cuemap-native" + test -s "${package_dir}/package/assets/en_tokenizer.bin" +done + +if [[ "$(uname -s)" == "Darwin" ]]; then + case "$(uname -m)" in + arm64) + node "${ROOT_DIR}/scripts/verify-npm-native-runtime.cjs" \ + "${VERIFY_DIR}/darwin-arm64/package/bin/cuemap" darwin-arm64 + if arch -x86_64 /usr/bin/true >/dev/null 2>&1; then + node "${ROOT_DIR}/scripts/verify-npm-native-runtime.cjs" \ + "${VERIFY_DIR}/darwin-x64/package/bin/cuemap" darwin-x64 + fi + ;; + x86_64) + node "${ROOT_DIR}/scripts/verify-npm-native-runtime.cjs" \ + "${VERIFY_DIR}/darwin-x64/package/bin/cuemap" darwin-x64 + ;; + *) + echo "Unsupported macOS architecture: $(uname -m)" >&2 + exit 1 + ;; + esac +fi + +docker run --rm --platform linux/amd64 \ + -v "${VERIFY_DIR}/linux-x64/package:/package:ro" \ + -v "${ROOT_DIR}/scripts/verify-npm-native-runtime.cjs:/verify-runtime.cjs:ro" \ + node:20-trixie-slim \ + node /verify-runtime.cjs /package/bin/cuemap linux-x64 + +echo "Native package structure, checksums, tokenizer, ingestion, and recall checks passed" diff --git a/scripts/verify-npm-native-runtime.cjs b/scripts/verify-npm-native-runtime.cjs new file mode 100755 index 0000000..c01d553 --- /dev/null +++ b/scripts/verify-npm-native-runtime.cjs @@ -0,0 +1,135 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const net = require("node:net"); +const os = require("node:os"); +const path = require("node:path"); +const { spawn } = require("node:child_process"); + +const executable = process.argv[2]; +const platformLabel = process.argv[3] || path.basename(path.dirname(path.dirname(executable || "package"))); +if (!executable) { + console.error("Usage: verify-npm-native-runtime.cjs "); + process.exit(2); +} + +const memory = "Maya switched from coffee to mint tea after the April deploy."; +const project = `npm-package-smoke-${process.pid}`; +const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "cuemap-npm-runtime-")); +let child; +let output = ""; + +function findFreePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +async function request(url, options = {}) { + const response = await fetch(url, options); + const body = await response.text(); + if (!response.ok) { + throw new Error(`${options.method || "GET"} ${url} returned ${response.status}: ${body}`); + } + return body ? JSON.parse(body) : null; +} + +async function waitUntilReady(baseUrl) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`CueMap exited before becoming ready.\n${output}`); + } + try { + await request(`${baseUrl}/stats`, { + headers: { "X-Project-ID": project }, + }); + return; + } catch { + await new Promise((resolve) => setTimeout(resolve, 200)); + } + } + throw new Error(`CueMap did not become ready within 30 seconds.\n${output}`); +} + +async function stopChild() { + if (!child || child.exitCode !== null) return; + child.kill("SIGINT"); + await Promise.race([ + new Promise((resolve) => child.once("exit", resolve)), + new Promise((resolve) => setTimeout(resolve, 5_000)), + ]); + if (child.exitCode === null) child.kill("SIGKILL"); +} + +async function main() { + const port = await findFreePort(); + const baseUrl = `http://127.0.0.1:${port}`; + child = spawn( + executable, + [ + "start", + "--port", + String(port), + "--data-dir", + dataDir, + "--disable-snapshots", + "--disable-bg-jobs", + ], + { + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env, TOKENIZER_PATH: "" }, + }, + ); + for (const stream of [child.stdout, child.stderr]) { + stream.on("data", (chunk) => { + output = `${output}${chunk}`.slice(-20_000); + }); + } + + await waitUntilReady(baseUrl); + const ingest = await request(`${baseUrl}/memories`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Project-ID": project, + }, + body: JSON.stringify({ content: memory }), + }); + if (!ingest.cues.includes("switch") || ingest.cues.includes("switched")) { + throw new Error(`Packaged tokenizer did not lemmatize correctly: ${JSON.stringify(ingest.cues)}`); + } + + const recall = await request(`${baseUrl}/recall`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Project-ID": project, + }, + body: JSON.stringify({ + query_text: "What did Maya switch to after the April deploy?", + limit: 5, + }), + }); + if (!recall.results.some((result) => result.content === memory)) { + throw new Error(`Packaged recall did not return the stored memory: ${JSON.stringify(recall)}`); + } + + console.log(`Runtime smoke passed: ${platformLabel}`); +} + +main() + .catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }) + .finally(async () => { + await stopChild(); + fs.rmSync(dataDir, { recursive: true, force: true }); + }); diff --git a/src/agent/chunker.rs b/src/agent/chunker.rs index 4a7ec9e..a7f5f71 100644 --- a/src/agent/chunker.rs +++ b/src/agent/chunker.rs @@ -256,6 +256,8 @@ struct TextBlock { text: String, start_line: usize, end_line: usize, + kind: BlockKind, + language: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1585,9 +1587,12 @@ impl Chunker { Self::chunk_text_with_config(content, &SegmenterConfig::default()) } - /// Chunk longform text by logical blocks instead of sliding sentence windows. - /// This keeps paragraph/list/code structure together for long assistant turns, - /// tickets, notes, and documents while still splitting oversized blocks. + /// Chunk longform text by logical blocks. + /// + /// Plain prose inside a logical block uses the configured sentence window, + /// while headings, lists, tables, and fenced/code-like blocks retain their + /// logical structure. Code blocks are parsed with the existing Tree-sitter + /// chunkers so they receive AST-derived cues rather than generic text cues. pub fn chunk_text_logical_blocks(content: &str, config: &SegmenterConfig) -> Vec { let blocks = Self::logical_text_blocks(content); if blocks.is_empty() { @@ -1596,99 +1601,316 @@ impl Chunker { let line_count = content.lines().count().max(1); let mut chunks = Vec::new(); - let mut pending = String::new(); - let mut pending_start = 1usize; - let mut pending_end = 1usize; - - let flush_pending = |chunks: &mut Vec, - pending: &mut String, - pending_start: &mut usize, - pending_end: &mut usize| { - let trimmed = pending.trim(); - if trimmed.is_empty() { - return; - } - chunks.push(Chunk { - content: trimmed.to_string(), - start_line: *pending_start, - end_line: *pending_end, - context: format!("block:{}", chunks.len()), - structural_cues: vec![ - "lang:text".to_string(), - "type:logical_block".to_string(), - format!("block:{}", chunks.len()), - ], - category: ChunkCategory::Prose, - }); - pending.clear(); - }; - - for block in blocks { + for (block_idx, block) in blocks.into_iter().enumerate() { let block_text = block.text.trim(); if block_text.is_empty() { continue; } - if block_text.len() > config.max_chunk_chars { - flush_pending( - &mut chunks, - &mut pending, - &mut pending_start, - &mut pending_end, - ); + let inferred_language = block + .language + .clone() + .or_else(|| Self::infer_code_language(block_text)); + let is_code = block.kind == BlockKind::CodeFence + || (block.kind == BlockKind::Plain + && inferred_language.is_some() + && Self::looks_like_unfenced_code(block_text)); + + if is_code { + chunks.extend(Self::chunk_logical_code_block( + &block, + block_idx, + inferred_language.as_deref(), + config.max_chunk_chars, + line_count, + )); + continue; + } + + let use_sentence_windows = block.kind == BlockKind::Plain; + if use_sentence_windows { let mut split_config = config.clone(); split_config.overlap = split_config.overlap.min(split_config.window_size / 2); - for split in Self::chunk_text_with_config(block_text, &split_config) { - chunks.push(Chunk { - content: split.content, - start_line: block.start_line.max(1), - end_line: block.end_line.min(line_count), - context: format!("block:{}:{}", block.start_line, split.context), - structural_cues: vec![ - "lang:text".to_string(), - "type:logical_block_split".to_string(), - format!("block:{}", chunks.len()), - ], - category: ChunkCategory::Prose, - }); + let splits = Self::chunk_text_with_config(block_text, &split_config); + if splits.len() > 1 || block_text.len() > config.max_chunk_chars { + for split in splits { + let mut cues = split.structural_cues; + cues.push("type:logical_block".to_string()); + cues.push("type:logical_block_split".to_string()); + cues.push(format!("block:{}", block_idx)); + cues.sort(); + cues.dedup(); + chunks.push(Chunk { + content: split.content, + start_line: block.start_line + .saturating_add(split.start_line.saturating_sub(1)) + .max(1), + end_line: block.start_line + .saturating_add(split.end_line.saturating_sub(1)) + .min(line_count), + context: format!("block:{}:{}", block_idx, split.context), + structural_cues: cues, + category: ChunkCategory::Prose, + }); + } + continue; } - continue; } - let pending_len = pending.len(); - let sep_len = if pending.is_empty() { 0 } else { 2 }; - if pending_len + sep_len + block_text.len() > config.max_chunk_chars { - flush_pending( - &mut chunks, - &mut pending, - &mut pending_start, - &mut pending_end, - ); + let mut cues = vec![ + "lang:text".to_string(), + "type:logical_block".to_string(), + format!("block:{}", block_idx), + ]; + cues.push(format!( + "block_kind:{}", + match block.kind { + BlockKind::Plain => "plain", + BlockKind::Heading => "heading", + BlockKind::List => "list", + BlockKind::Table => "table", + BlockKind::CodeFence => "code", + } + )); + chunks.push(Chunk { + content: block_text.to_string(), + start_line: block.start_line.max(1), + end_line: block.end_line.min(line_count), + context: format!("block:{}", block_idx), + structural_cues: cues, + category: ChunkCategory::Prose, + }); + } + + if chunks.is_empty() { + return Self::chunk_text_with_config(content, config); + } + + chunks + } + + /// Route a logical code block through the same Tree-sitter chunkers used by + /// file ingestion. A small code block remains one logical chunk, but its + /// chunk carries the union of AST cues found inside it. Oversized blocks use + /// the AST chunks so the configured maximum remains meaningful. + fn chunk_logical_code_block( + block: &TextBlock, + block_idx: usize, + language: Option<&str>, + max_chunk_chars: usize, + line_count: usize, + ) -> Vec { + let (code, code_start_line) = Self::code_fence_body(&block.text, block.start_line); + let code = code.trim(); + if code.is_empty() { + return Vec::new(); + } + + let normalized_language = language + .map(Self::normalize_code_language) + .filter(|value| !value.is_empty()); + let language = normalized_language.as_deref(); + let parser_chunks = Self::chunk_code_by_language(code, language); + + if code.len() <= max_chunk_chars { + let mut cues = vec![ + "type:logical_block".to_string(), + "type:logical_code_block".to_string(), + format!("block:{}", block_idx), + ]; + cues.push(format!( + "lang:{}", + language.unwrap_or("code") + )); + for parser_chunk in &parser_chunks { + cues.extend(parser_chunk.structural_cues.iter().cloned()); } + cues.sort(); + cues.dedup(); - if pending.is_empty() { - pending_start = block.start_line.max(1); - pending_end = block.end_line.max(pending_start); - pending.push_str(block_text); - } else { - pending.push_str("\n\n"); - pending.push_str(block_text); - pending_end = block.end_line.max(pending_end); + return vec![Chunk { + content: code.to_string(), + start_line: code_start_line.max(1), + end_line: (code_start_line + .saturating_add(code.lines().count().saturating_sub(1))) + .min(line_count), + context: format!("block:{}:code", block_idx), + structural_cues: cues, + category: if Self::is_structured_language(language) { + ChunkCategory::Structured + } else { + ChunkCategory::Code + }, + }]; + } + + if !parser_chunks.is_empty() { + return parser_chunks + .into_iter() + .enumerate() + .map(|(part_idx, mut chunk)| { + chunk.start_line = code_start_line + .saturating_add(chunk.start_line.saturating_sub(1)) + .max(1); + chunk.end_line = code_start_line + .saturating_add(chunk.end_line.saturating_sub(1)) + .min(line_count); + chunk.context = format!("block:{}:{}", block_idx, chunk.context); + chunk.structural_cues.push("type:logical_block".to_string()); + chunk.structural_cues.push(format!("block:{}", block_idx)); + chunk + .structural_cues + .push(format!("code_part:{}", part_idx)); + chunk.structural_cues.sort(); + chunk.structural_cues.dedup(); + chunk + }) + .collect(); + } + + let mut cues = vec![ + "type:logical_block".to_string(), + "type:logical_code_block".to_string(), + format!("block:{}", block_idx), + format!("lang:{}", language.unwrap_or("code")), + ]; + cues.sort(); + cues.dedup(); + let lines: Vec<&str> = code.lines().collect(); + lines + .chunks(20) + .enumerate() + .map(|(part_idx, part)| Chunk { + content: part.join("\n"), + start_line: code_start_line.saturating_add(part_idx * 20).max(1), + end_line: code_start_line + .saturating_add(part_idx * 20 + part.len().saturating_sub(1)) + .min(line_count), + context: format!("block:{}:code_part:{}", block_idx, part_idx), + structural_cues: { + let mut part_cues = cues.clone(); + part_cues.push(format!("code_part:{}", part_idx)); + part_cues + }, + category: ChunkCategory::Code, + }) + .collect() + } + + fn chunk_code_by_language(content: &str, language: Option<&str>) -> Vec { + match language { + Some("python") => Self::chunk_python(content), + Some("rust") => Self::chunk_rust(content), + Some("typescript") => Self::chunk_typescript(content), + Some("javascript") => Self::chunk_javascript(content), + Some("go") => Self::chunk_go(content), + Some("html") => Self::chunk_html(content), + Some("css") => Self::chunk_css(content), + Some("php") => Self::chunk_php(content), + Some("java") => Self::chunk_java(content), + _ => Vec::new(), + } + } + + fn code_fence_body(text: &str, start_line: usize) -> (String, usize) { + let mut lines: Vec<&str> = text.lines().collect(); + let has_fence = lines + .first() + .map(|line| { + let trimmed = line.trim(); + trimmed.starts_with("```") || trimmed.starts_with("~~~") + }) + .unwrap_or(false); + if has_fence { + lines.remove(0); + if lines + .last() + .map(|line| { + let trimmed = line.trim(); + trimmed.starts_with("```") || trimmed.starts_with("~~~") + }) + .unwrap_or(false) + { + lines.pop(); } + (lines.join("\n"), start_line.saturating_add(1)) + } else { + (text.to_string(), start_line) } + } - flush_pending( - &mut chunks, - &mut pending, - &mut pending_start, - &mut pending_end, - ); + fn normalize_code_language(language: &str) -> String { + let value = language + .trim() + .to_ascii_lowercase() + .trim_matches(|c: char| !c.is_ascii_alphanumeric()) + .to_string(); + match value.as_str() { + "py" => "python".to_string(), + "rs" => "rust".to_string(), + "ts" | "tsx" => "typescript".to_string(), + "js" | "jsx" => "javascript".to_string(), + "htm" => "html".to_string(), + _ => value, + } + } - if chunks.is_empty() { - return Self::chunk_text_with_config(content, config); + fn is_structured_language(language: Option<&str>) -> bool { + matches!(language, Some("json" | "yaml" | "xml" | "csv")) + } + + fn infer_code_language(content: &str) -> Option { + let lower = content.to_ascii_lowercase(); + let trimmed = lower.trim_start(); + if trimmed.starts_with("def ") + || trimmed.starts_with("class ") + || trimmed.starts_with("from ") + || trimmed.starts_with("import ") + || lower.contains("if __name__ ==") + { + return Some("python".to_string()); + } + if trimmed.starts_with("fn ") + || trimmed.starts_with("use ") + || trimmed.starts_with("struct ") + || trimmed.starts_with("enum ") + || trimmed.starts_with("impl ") + { + return Some("rust".to_string()); } + if trimmed.starts_with("package ") || trimmed.starts_with("func ") { + return Some("go".to_string()); + } + if trimmed.starts_with("#include ") || trimmed.starts_with("public class ") { + return Some("java".to_string()); + } + if lower.contains("=>") + || trimmed.starts_with("const ") + || trimmed.starts_with("let ") + || trimmed.starts_with("function ") + || trimmed.starts_with("interface ") + { + return Some("javascript".to_string()); + } + None + } - chunks + fn looks_like_unfenced_code(content: &str) -> bool { + let lines: Vec<&str> = content.lines().collect(); + let trimmed = content.trim_start(); + let strong_prefix = [ + "def ", "class ", "fn ", "func ", "struct ", "enum ", "impl ", + "package ", "import ", "from ", "#include ", "const ", "let ", + "function ", "interface ", "public class ", + ]; + if strong_prefix.iter().any(|prefix| trimmed.starts_with(prefix)) { + return true; + } + lines.len() >= 2 + && (content.contains("{\n") + || content.contains(";\n") + || content.contains("=>") + || content.contains("::")) } fn logical_text_blocks(content: &str) -> Vec { @@ -1698,20 +1920,26 @@ impl Chunker { let mut last_line = 1usize; let mut in_fence = false; let mut current_kind = BlockKind::Plain; + let mut current_language: Option = None; let flush = |blocks: &mut Vec, current: &mut String, start_line: &mut usize, - last_line: usize| { + last_line: usize, + kind: BlockKind, + language: &mut Option| { let trimmed = current.trim(); if trimmed.is_empty() { current.clear(); + *language = None; return; } blocks.push(TextBlock { text: trimmed.to_string(), start_line: *start_line, end_line: last_line, + kind, + language: language.take(), }); current.clear(); }; @@ -1732,7 +1960,14 @@ impl Chunker { last_line = line_no; if is_fence { in_fence = false; - flush(&mut blocks, &mut current, &mut start_line, last_line); + flush( + &mut blocks, + &mut current, + &mut start_line, + last_line, + current_kind, + &mut current_language, + ); current_kind = BlockKind::Plain; } continue; @@ -1740,10 +1975,22 @@ impl Chunker { if is_fence { if !current.trim().is_empty() { - flush(&mut blocks, &mut current, &mut start_line, last_line); + flush( + &mut blocks, + &mut current, + &mut start_line, + last_line, + current_kind, + &mut current_language, + ); } start_line = line_no; current_kind = BlockKind::CodeFence; + current_language = trimmed + .get(3..) + .and_then(|value| value.trim().split_whitespace().next()) + .map(Self::normalize_code_language) + .filter(|value| !value.is_empty()); in_fence = true; current.push_str(line); current.push('\n'); @@ -1752,7 +1999,14 @@ impl Chunker { } if trimmed.is_empty() { - flush(&mut blocks, &mut current, &mut start_line, last_line); + flush( + &mut blocks, + &mut current, + &mut start_line, + last_line, + current_kind, + &mut current_language, + ); current_kind = BlockKind::Plain; continue; } @@ -1764,7 +2018,14 @@ impl Chunker { || (current_kind == BlockKind::Table && line_kind != BlockKind::Table)); if should_flush { - flush(&mut blocks, &mut current, &mut start_line, last_line); + flush( + &mut blocks, + &mut current, + &mut start_line, + last_line, + current_kind, + &mut current_language, + ); start_line = line_no; current_kind = line_kind; } @@ -1777,7 +2038,14 @@ impl Chunker { } } - flush(&mut blocks, &mut current, &mut start_line, last_line); + flush( + &mut blocks, + &mut current, + &mut start_line, + last_line, + current_kind, + &mut current_language, + ); blocks } diff --git a/src/agent/ingester.rs b/src/agent/ingester.rs index 56a28fe..e543b85 100644 --- a/src/agent/ingester.rs +++ b/src/agent/ingester.rs @@ -2,32 +2,121 @@ use crate::agent::chunker::Chunker; use crate::agent::AgentConfig; use crate::jobs::{Job, JobQueue, MemoryRef}; use ignore::gitignore::{Gitignore, GitignoreBuilder}; +use ignore::Match; use ignore::WalkBuilder; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::time::{sleep, Duration}; use tracing::{debug, info, warn}; +const DEFAULT_NOISE_PATTERNS: &[&str] = &[ + // JS/TS + "package-lock.json", + "yarn.lock", + "pnpm-lock.yaml", + "bun.lockb", + "tsconfig.json", + "node_modules/", + ".npmrc", + ".eslintcache", + ".next/", + ".nuxt/", + "bower_components/", + "__snapshots__/", + // Rust + "Cargo.lock", + "target/", + // Python + "poetry.lock", + "Pipfile.lock", + "__pycache__/", + "venv/", + ".venv/", + "env/", + ".env/", + ".pytest_cache/", + ".ipynb_checkpoints/", + "*.pyc", + "*.pyo", + "*.pyd", + // Go + "go.sum", + // Java/JVM + ".gradle/", + ".m2/", + "build/", + // PHP + "composer.lock", + // Ruby + "Gemfile.lock", + // iOS/macOS + "Pods/", + "DerivedData/", + "*.xcodeproj/", + "*.xcworkspace/", + // System & IDEs + ".DS_Store", + "Thumbs.db", + ".idea/", + ".vscode/", + ".history/", + ".git/", + ".svn/", + ".hg/", +]; +const REPOSITORY_IGNORE_FILENAMES: &[&str] = + &[".gitignore", ".antigravityignore", ".cuemapignore"]; +const INGESTER_STATE_VERSION: u32 = 2; + pub struct Ingester { config: AgentConfig, job_queue: Arc, file_hashes: HashMap, // path -> sha256 - gitignore: Option, + policy_ignore: Option, + repository_ignores: Vec, memory_hashes: HashMap, // memory_id -> content_hash path_to_memories: HashMap>, // path -> set of current memory_ids } #[derive(Serialize, Deserialize, Default)] struct IngesterState { + #[serde(default)] + schema_version: u32, file_hashes: HashMap, memory_hashes: HashMap, path_to_memories: HashMap>, } +#[derive(Debug, Clone, Serialize)] +pub struct DirectoryPreviewEntry { + pub path: String, + pub kind: String, + pub supported_files: usize, + pub bytes: u64, + pub categories: BTreeMap, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DirectoryPreview { + pub watch_dir: String, + pub supported_files: usize, + pub bytes: u64, + pub entries: Vec, + pub scan_errors: usize, +} + +#[derive(Default)] +struct PreviewEntryAccumulator { + kind: String, + supported_files: usize, + bytes: u64, + categories: BTreeMap, +} + impl Ingester { pub fn new(config: AgentConfig, job_queue: Arc) -> Self { // Canonicalize watch_dir to ensure absolute path matching works across the engine @@ -35,139 +124,131 @@ impl Ingester { .unwrap_or_else(|_| PathBuf::from(&config.watch_dir)); debug!("Agent initializing with watch root: {:?}", watch_path); - // Prepare gitignore - let mut gitignore = None; - let mut builder = GitignoreBuilder::new(&watch_path); - - // Search for all .*ignore files recursively in watch_dir - // AND its parents (walking up from watch_dir) - let mut found_any = false; - - // 1. Walk UP from watch_dir to root - let mut current = Some(watch_path.as_path()); - while let Some(p) = current { - if let Ok(entries) = fs::read_dir(p) { - for entry in entries.flatten() { - let file_name = entry.file_name(); - let name_str = file_name.to_string_lossy(); - if name_str.starts_with('.') && name_str.ends_with("ignore") { - let p_gi = entry.path(); - if let Some(err) = builder.add(&p_gi) { - warn!("Error loading ignore file at {:?}: {}", p_gi, err); - } else { - found_any = true; - } - } - } + let policy_ignore = Self::build_policy_ignore(&watch_path, &config.ignored_patterns); + let repository_ignores = Self::build_repository_ignores(&watch_path); + + let mut config = config; + config.watch_dir = watch_path.to_string_lossy().to_string(); + + Self { + config, + job_queue, + file_hashes: HashMap::new(), + policy_ignore, + repository_ignores, + memory_hashes: HashMap::new(), + path_to_memories: HashMap::new(), + } + } + + fn is_ignore_file(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .map(|name| REPOSITORY_IGNORE_FILENAMES.contains(&name)) + .unwrap_or(false) + } + + pub fn is_ignore_config_path(path: &Path) -> bool { + Self::is_ignore_file(path) + } + + fn build_policy_ignore(watch_path: &Path, ignored_patterns: &[String]) -> Option { + let mut builder = GitignoreBuilder::new(watch_path); + + for pattern in DEFAULT_NOISE_PATTERNS { + let _ = builder.add_line(None, pattern); + } + for pattern in ignored_patterns { + let _ = builder.add_line(None, pattern); + } + + match builder.build() { + Ok(gitignore) => Some(gitignore), + Err(error) => { + warn!("Failed to build gitignore: {}", error); + None + } + } + } + + fn build_repository_ignores(watch_path: &Path) -> Vec { + let mut directories = HashSet::new(); + + let mut current = Some(watch_path); + while let Some(directory) = current { + if REPOSITORY_IGNORE_FILENAMES + .iter() + .any(|name| directory.join(name).is_file()) + { + directories.insert(directory.to_path_buf()); } - current = p.parent(); + current = directory.parent(); } - // 2. Walk DOWN from watch_dir (recursive) - for result in WalkBuilder::new(&watch_path) + for result in WalkBuilder::new(watch_path) .hidden(false) - .git_ignore(false) + .git_ignore(true) .build() { if let Ok(entry) = result { - let file_name = entry.file_name(); - let name_str = file_name.to_string_lossy(); - if name_str.starts_with('.') && name_str.ends_with("ignore") { - let p_gi = entry.path(); - if let Some(err) = builder.add(&p_gi) { - warn!("Error loading ignore file at {:?}: {}", p_gi, err); - } else { - found_any = true; + let ignore_path = entry.path(); + if Self::is_ignore_file(ignore_path) { + if let Some(directory) = ignore_path.parent() { + directories.insert(directory.to_path_buf()); } } } } - // Add default "noise" patterns - let default_noise = [ - // JS/TS - "package-lock.json", - "yarn.lock", - "pnpm-lock.yaml", - "bun.lockb", - "tsconfig.json", - "node_modules/", - ".npmrc", - ".eslintcache", - ".next/", - ".nuxt/", - "bower_components/", - "__snapshots__/", - // Rust - "Cargo.lock", - "target/", - // Python - "poetry.lock", - "Pipfile.lock", - "__pycache__/", - "venv/", - ".venv/", - "env/", - ".env/", - ".pytest_cache/", - ".ipynb_checkpoints/", - "*.pyc", - "*.pyo", - "*.pyd", - // Go - "go.sum", - // Java/JVM - ".gradle/", - ".m2/", - "build/", // covers Gradle, target/ is in Rust but also Maven - // PHP - "composer.lock", - // Ruby - "Gemfile.lock", - // iOS/macOS - "Pods/", - "DerivedData/", - "*.xcodeproj/", - "*.xcworkspace/", - // System & IDEs - ".DS_Store", - "Thumbs.db", - ".idea/", - ".vscode/", - ".history/", - ".git/", - ".svn/", - ".hg/", - ]; - for pattern in default_noise { - let _ = builder.add_line(None, pattern); - } + let mut directories: Vec = directories.into_iter().collect(); + directories.sort_by_key(|path| path.components().count()); + + directories + .into_iter() + .filter_map(|directory| { + let mut builder = GitignoreBuilder::new(&directory); + for filename in REPOSITORY_IGNORE_FILENAMES { + let ignore_path = directory.join(filename); + if ignore_path.is_file() { + if let Some(error) = builder.add(&ignore_path) { + warn!("Error loading ignore file at {:?}: {}", ignore_path, error); + } + } + } + match builder.build() { + Ok(ignore) if !ignore.is_empty() => Some(ignore), + Ok(_) => None, + Err(error) => { + warn!("Failed to build repository ignore matcher: {}", error); + None + } + } + }) + .collect() + } - // Add custom patterns from config - for pattern in &config.ignored_patterns { - let _ = builder.add_line(None, pattern); + fn path_is_ignored(&self, path: &Path) -> bool { + if let Some(policy_ignore) = &self.policy_ignore { + if policy_ignore + .matched_path_or_any_parents(path, false) + .is_ignore() + { + return true; + } } - if found_any || !default_noise.is_empty() || !config.ignored_patterns.is_empty() { - match builder.build() { - Ok(gi) => gitignore = Some(gi), - Err(e) => warn!("Failed to build gitignore: {}", e), + for repository_ignore in self.repository_ignores.iter().rev() { + if !path.starts_with(repository_ignore.path()) { + continue; + } + match repository_ignore.matched_path_or_any_parents(path, false) { + Match::Ignore(_) => return true, + Match::Whitelist(_) => return false, + Match::None => {} } - } else { - debug!("No ignore files or patterns found"); } - let mut config = config; - config.watch_dir = watch_path.to_string_lossy().to_string(); - - Self { - config, - job_queue, - file_hashes: HashMap::new(), - gitignore, - memory_hashes: HashMap::new(), - path_to_memories: HashMap::new(), - } + false } pub fn load_state(&mut self, state_path: &std::path::Path) -> Result<(), String> { @@ -181,8 +262,17 @@ impl Ingester { let state: IngesterState = serde_json::from_str(&content) .map_err(|e| format!("Failed to parse agent state: {}", e))?; - self.file_hashes = state.file_hashes; - self.memory_hashes = state.memory_hashes; + if state.schema_version < INGESTER_STATE_VERSION { + debug!( + "Upgrading agent state from schema {} to {}; tracked files will be reingested", + state.schema_version, INGESTER_STATE_VERSION + ); + self.file_hashes = HashMap::new(); + self.memory_hashes = HashMap::new(); + } else { + self.file_hashes = state.file_hashes; + self.memory_hashes = state.memory_hashes; + } self.path_to_memories = state.path_to_memories; debug!( @@ -194,6 +284,7 @@ impl Ingester { pub fn save_state(&self, state_path: &std::path::Path) -> Result<(), String> { let state = IngesterState { + schema_version: INGESTER_STATE_VERSION, file_hashes: self.file_hashes.clone(), memory_hashes: self.memory_hashes.clone(), path_to_memories: self.path_to_memories.clone(), @@ -212,13 +303,163 @@ impl Ingester { Ok(()) } + fn path_is_selected(&self, path: &Path) -> bool { + if self.config.included_paths.is_empty() { + return true; + } + + let Ok(relative) = path.strip_prefix(&self.config.watch_dir) else { + return false; + }; + + self.config.included_paths.iter().any(|included| { + let included_path = Path::new(included); + relative == included_path || relative.starts_with(included_path) + }) + } + + fn path_is_allowed(&self, path: &Path) -> bool { + if !path.is_file() || Chunker::detect_type(path).is_none() { + return false; + } + + if Self::is_ignore_file(path) { + return false; + } + + if let Some(ref state_path) = self.config.state_file { + let matches_state = fs::canonicalize(state_path) + .map(|canonical_state| canonical_state == path) + .unwrap_or_else(|_| state_path == path); + if matches_state { + return false; + } + } + + let Ok(relative) = path.strip_prefix(&self.config.watch_dir) else { + return false; + }; + + if relative.components().any(|component| { + let name = component.as_os_str().to_string_lossy(); + name.starts_with('.') + && name != "." + && name != ".." + && name != ".gitignore" + && name != ".cuemapignore" + && name != ".antigravityignore" + }) { + return false; + } + + if !self.path_is_selected(path) { + return false; + } + + if self.path_is_ignored(path) { + return false; + } + + if let Some(extension) = path.extension().and_then(|value| value.to_str()) { + if self + .config + .ignored_extensions + .iter() + .any(|ignored| ignored.eq_ignore_ascii_case(extension)) + { + return false; + } + } + + true + } + + pub fn preview_scope(&self) -> Result { + let mut entries: BTreeMap = BTreeMap::new(); + let mut supported_files = 0usize; + let mut total_bytes = 0u64; + let mut scan_errors = 0usize; + + let walker = WalkBuilder::new(&self.config.watch_dir) + .hidden(false) + .git_ignore(true) + .build(); + + for result in walker { + let entry = match result { + Ok(entry) => entry, + Err(error) => { + scan_errors += 1; + warn!("Directory preview walk error: {}", error); + continue; + } + }; + let path = match fs::canonicalize(entry.path()) { + Ok(path) if self.path_is_allowed(&path) => path, + _ => continue, + }; + let Ok(relative) = path.strip_prefix(&self.config.watch_dir) else { + continue; + }; + + let mut components = relative.components(); + let Some(first_component) = components.next() else { + continue; + }; + let nested = components.next().is_some(); + let key = if nested { + first_component.as_os_str().to_string_lossy().to_string() + } else { + relative.to_string_lossy().replace('\\', "/") + }; + let kind = if nested { "directory" } else { "file" }; + let bytes = entry.metadata().map(|metadata| metadata.len()).unwrap_or(0); + let category = format!("{:?}", Chunker::get_category_for_file(&path)).to_lowercase(); + + let aggregate = entries.entry(key).or_default(); + aggregate.kind = kind.to_string(); + aggregate.supported_files += 1; + aggregate.bytes += bytes; + *aggregate.categories.entry(category).or_default() += 1; + supported_files += 1; + total_bytes += bytes; + } + + Ok(DirectoryPreview { + watch_dir: self.config.watch_dir.clone(), + supported_files, + bytes: total_bytes, + entries: entries + .into_iter() + .map(|(path, entry)| DirectoryPreviewEntry { + path, + kind: entry.kind, + supported_files: entry.supported_files, + bytes: entry.bytes, + categories: entry.categories, + }) + .collect(), + scan_errors, + }) + } + + pub async fn reload_filters_and_rescan(&mut self) -> Result<(), String> { + let watch_path = PathBuf::from(&self.config.watch_dir); + self.policy_ignore = + Self::build_policy_ignore(&watch_path, &self.config.ignored_patterns); + self.repository_ignores = Self::build_repository_ignores(&watch_path); + self.scan_all().await + } + pub async fn scan_all(&mut self) -> Result<(), String> { debug!("Starting full scan of {}", self.config.watch_dir); let path_str = self.config.watch_dir.clone(); + let mut eligible_paths = HashSet::new(); + let mut walk_failed = false; - // Use ignore crate to respect .gitignore natively (for early pruning) - // All other .*ignore files are handled by our unified Gitignore check in process_file_path + // Use ignore crate to respect .gitignore natively for early pruning. + // CueMap-specific ignore files are evaluated with directory-scoped matchers. let walker = WalkBuilder::new(&path_str) .hidden(false) .git_ignore(true) @@ -227,18 +468,34 @@ impl Ingester { for result in walker { match result { Ok(entry) => { - let path = entry.path(); - if path.is_file() { - if let Err(_e) = self.process_file_path(path.to_path_buf()).await { - // warn!("Failed to process {:?}: {}", path, e); - } - // Throttle - if self.config.throttle_ms > 0 { - sleep(Duration::from_millis(self.config.throttle_ms)).await; - } + let path = match fs::canonicalize(entry.path()) { + Ok(path) if self.path_is_allowed(&path) => path, + _ => continue, + }; + eligible_paths.insert(path.to_string_lossy().to_lowercase()); + if let Err(error) = self.process_file_path(path.clone()).await { + debug!("Skipping file {:?}: {}", path, error); + } + if self.config.throttle_ms > 0 { + sleep(Duration::from_millis(self.config.throttle_ms)).await; } } - Err(err) => warn!("Walk error: {}", err), + Err(err) => { + walk_failed = true; + warn!("Walk error: {}", err); + } + } + } + + if !walk_failed { + let stale_paths: Vec = self + .file_hashes + .keys() + .filter(|path| !eligible_paths.contains(*path)) + .cloned() + .collect(); + for stale_path in stale_paths { + self.delete_tracked_path_key(&stale_path).await; } } @@ -251,61 +508,9 @@ impl Ingester { .map_err(|e| format!("Failed to canonicalize path {:?}: {}", path, e))?; let path_str = path.to_string_lossy().to_string(); - // 0. Ignore state file - if let Some(ref state_path) = self.config.state_file { - if let Ok(abs_path) = std::fs::canonicalize(&path) { - if let Ok(abs_state) = std::fs::canonicalize(state_path) { - if abs_path == abs_state { - debug!("Skipping agent state file: {}", path_str); - return Ok(()); - } - } else { - // If state file doesn't exist yet but paths match string-wise - if path == *state_path { - debug!("Skipping agent state file: {}", path_str); - return Ok(()); - } - } - } - } - - // 0.1 Hidden file check (only for files/dirs BELOW watch_dir) - if let Ok(rel) = path.strip_prefix(&self.config.watch_dir) { - if rel.components().any(|c| { - let s = c.as_os_str().to_string_lossy(); - s.starts_with('.') - && s != "." - && s != ".." - && s != ".gitignore" - && s != ".cuemapignore" - && s != ".antigravityignore" - }) { - debug!("Skipping hidden path: {}", path_str); - return Ok(()); - } - } - - // 0.1 Check Gitignore - if let Some(gi) = &self.gitignore { - // gi.matched_path_or_any_parents handles absolute paths by making them relative to the builder's root. - let match_result = gi.matched_path_or_any_parents(&path, path.is_dir()); - if match_result.is_ignore() { - debug!("Skipping ignored file: {}", path_str); - return Ok(()); - } - } - - // 0.2 Check custom ignored extensions - if let Some(ext) = path.extension().and_then(|s| s.to_str()) { - if self - .config - .ignored_extensions - .iter() - .any(|e| e.eq_ignore_ascii_case(ext)) - { - debug!("Skipping blacklisted extension: {}", path_str); - return Ok(()); - } + if !self.path_is_allowed(&path) { + debug!("Skipping out-of-scope or unsupported file: {}", path_str); + return Ok(()); } // Standardize casing for case-insensitive filesystems (MacOS/Windows) @@ -377,14 +582,35 @@ impl Ingester { self.memory_hashes.insert(memory_id.clone(), chunk_hash); session.expect_write(); + let category = format!("{:?}", chunk.category).to_lowercase(); + let mut structural_cues = chunk.structural_cues.clone(); + structural_cues.push("source_type:repository_file".to_string()); + structural_cues.push("source_channel:filesystem".to_string()); + + let mut metadata = HashMap::new(); + metadata.insert( + "source_type".to_string(), + serde_json::json!("repository_file"), + ); + metadata.insert( + "source_channel".to_string(), + serde_json::json!("filesystem"), + ); + metadata.insert( + "source_path".to_string(), + serde_json::json!(path_str.clone()), + ); + metadata.insert("source_category".to_string(), serde_json::json!(category)); + self.job_queue .enqueue(Job::ExtractAndIngest { project_id: project_id.clone(), source_key: memory_id.clone(), content: chunk.content.clone(), file_path: path_norm.clone(), - structural_cues: chunk.structural_cues.clone(), - metadata: None, + structural_cues, + metadata: Some(metadata), + embedding: None, category: chunk.category, }) .await; @@ -420,25 +646,39 @@ impl Ingester { Ok(()) } - pub async fn delete_file_path(&mut self, path: PathBuf) -> Result<(), String> { - let path_str = path.to_string_lossy().to_string(); - let path_norm = path_str.to_lowercase(); - debug!("Processing deletion: {}", path_str); - - // Remove from tracking - self.file_hashes.remove(&path_norm); - if let Some(mems) = self.path_to_memories.remove(&path_norm) { - for m_id in mems { - self.memory_hashes.remove(&m_id); - // Explicitly delete from engine + async fn delete_tracked_path_key(&mut self, path_norm: &str) { + self.file_hashes.remove(path_norm); + if let Some(memories) = self.path_to_memories.remove(path_norm) { + for memory_id in memories { + self.memory_hashes.remove(&memory_id); self.job_queue .enqueue(Job::DeleteMemory { project_id: self.config.project_id.clone(), - memory_ref: MemoryRef::SourceKey(m_id), + memory_ref: MemoryRef::SourceKey(memory_id), }) .await; } } + } + + pub async fn delete_file_path(&mut self, path: PathBuf) -> Result<(), String> { + // Deletion events arrive after the file is gone, so canonicalize the + // parent directory as a fallback to keep the key consistent with + // process_file_path on symlinked temporary roots (notably macOS /var). + let path = fs::canonicalize(&path).or_else(|_| { + let parent = path + .parent() + .ok_or_else(|| std::io::Error::other("missing parent directory"))?; + let file_name = path + .file_name() + .ok_or_else(|| std::io::Error::other("missing file name"))?; + Ok::(fs::canonicalize(parent)?.join(file_name)) + }).map_err(|error| format!("Failed to canonicalize deleted path {:?}: {}", path, error))?; + let path_str = path.to_string_lossy().to_string(); + let path_norm = path_str.to_lowercase(); + debug!("Processing deletion: {}", path_str); + + self.delete_tracked_path_key(&path_norm).await; Ok(()) } @@ -632,6 +872,7 @@ impl Ingester { file_path: source.clone(), structural_cues: chunk.structural_cues.clone(), metadata: None, + embedding: None, category: chunk.category, }) .await; @@ -717,6 +958,18 @@ impl Ingester { project_id: &str, source: &str, metadata: Option>, + ) -> Result, String> { + self.process_chunks_with_metadata_and_embeddings(chunks, project_id, source, metadata, None) + .await + } + + pub async fn process_chunks_with_metadata_and_embeddings( + &mut self, + chunks: Vec, + project_id: &str, + source: &str, + metadata: Option>, + embeddings: Option>>, ) -> Result, String> { let mut memory_ids = Vec::new(); @@ -726,7 +979,7 @@ impl Ingester { session.expect_write(); } - for chunk in chunks.iter() { + for (chunk_index, chunk) in chunks.iter().enumerate() { let mut chunk_hasher = Sha256::new(); chunk_hasher.update(chunk.content.as_bytes()); let chunk_hash = format!("{:x}", chunk_hasher.finalize()); @@ -743,6 +996,9 @@ impl Ingester { file_path: source.to_string(), structural_cues: chunk.structural_cues.clone(), metadata: metadata.clone(), + embedding: embeddings + .as_ref() + .and_then(|vectors| vectors.get(chunk_index).cloned()), category: chunk.category, }) .await; @@ -792,3 +1048,105 @@ pub struct CrawlProgress { pub links_found: usize, pub links_skipped: usize, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::TuningConfig; + use crate::jobs::JobQueue; + use crate::multi_tenant::MultiTenantEngine; + + fn test_ingester(dir: &Path, state_file: Option) -> Ingester { + let provider = Arc::new(MultiTenantEngine::with_snapshots_dir( + dir.join("snapshots"), + TuningConfig::default(), + )); + let queue = Arc::new(JobQueue::new(provider, None, true)); + Ingester::new( + AgentConfig { + project_id: "ingester-tests".to_string(), + watch_dir: dir.to_string_lossy().to_string(), + throttle_ms: 0, + state_file, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }, + queue, + ) + } + + #[tokio::test] + async fn state_round_trip_and_legacy_upgrade_are_safe() { + let dir = tempfile::tempdir().unwrap(); + let state_path = dir.path().join("state.json"); + let mut ingester = test_ingester(dir.path(), Some(state_path.clone())); + ingester + .file_hashes + .insert("/tmp/note.md".to_string(), "hash".to_string()); + ingester + .memory_hashes + .insert("memory-1".to_string(), "chunk-hash".to_string()); + ingester.path_to_memories.insert( + "/tmp/note.md".to_string(), + ["memory-1".to_string()].into_iter().collect(), + ); + ingester.save_state(&state_path).unwrap(); + + let mut restored = test_ingester(dir.path(), Some(state_path.clone())); + restored.load_state(&state_path).unwrap(); + assert_eq!(restored.get_file_hashes().get("/tmp/note.md"), Some(&"hash".to_string())); + assert!(restored.memory_hashes.contains_key("memory-1")); + + std::fs::write( + &state_path, + serde_json::json!({ + "schema_version": 1, + "file_hashes": {"/tmp/old.md": "old"}, + "memory_hashes": {"old-memory": "old"}, + "path_to_memories": {} + }) + .to_string(), + ) + .unwrap(); + restored.load_state(&state_path).unwrap(); + assert!(restored.get_file_hashes().is_empty()); + assert!(restored.memory_hashes.is_empty()); + + std::fs::write(&state_path, "not-json").unwrap(); + assert!(restored.load_state(&state_path).is_err()); + } + + #[tokio::test] + async fn preview_scope_reports_supported_files_and_scope_filters() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("src")).unwrap(); + std::fs::write(dir.path().join("src/main.rs"), "fn main() {}").unwrap(); + std::fs::write(dir.path().join("README.md"), "# readme").unwrap(); + std::fs::write(dir.path().join("notes.log"), "ignored").unwrap(); + + let mut ingester = test_ingester(dir.path(), None); + ingester.config.included_paths = vec!["src".to_string()]; + ingester.config.ignored_extensions = vec!["log".to_string()]; + let preview = ingester.preview_scope().unwrap(); + assert_eq!(preview.supported_files, 1); + assert_eq!(preview.entries[0].path, "src"); + assert_eq!(preview.entries[0].kind, "directory"); + assert_eq!(preview.entries[0].categories.len(), 1); + } + + #[tokio::test] + async fn file_processing_skips_unchanged_content_and_deletes_tracking() { + let dir = tempfile::tempdir().unwrap(); + let note = dir.path().join("note.md"); + std::fs::write(¬e, "first").unwrap(); + let mut ingester = test_ingester(dir.path(), None); + ingester.process_file_path(note.clone()).await.unwrap(); + let first_hashes = ingester.file_hashes.clone(); + ingester.process_file_path(note.clone()).await.unwrap(); + assert_eq!(ingester.file_hashes, first_hashes); + ingester.delete_file_path(note.clone()).await.unwrap(); + assert!(ingester.file_hashes.is_empty()); + assert!(ingester.path_to_memories.is_empty()); + } +} diff --git a/src/agent/manager.rs b/src/agent/manager.rs index 8e1373b..a13c74c 100644 --- a/src/agent/manager.rs +++ b/src/agent/manager.rs @@ -67,3 +67,67 @@ impl AgentManager { locked.get(project_id).cloned() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::TuningConfig; + use crate::multi_tenant::MultiTenantEngine; + + fn config(project_id: &str, watch_dir: &std::path::Path) -> AgentConfig { + AgentConfig { + project_id: project_id.to_string(), + watch_dir: watch_dir.to_string_lossy().to_string(), + throttle_ms: 1, + state_file: None, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + } + } + + #[tokio::test] + async fn manager_starts_replaces_and_stops_agents() { + let dir = tempfile::tempdir().unwrap(); + let snapshots = dir.path().join("snapshots"); + let provider = Arc::new(MultiTenantEngine::with_snapshots_dir( + &snapshots, + TuningConfig::default(), + )); + let job_queue = Arc::new(JobQueue::new(provider.clone(), None, true)); + let manager = AgentManager::new(job_queue, provider); + + assert!(manager.get_agent("project").await.is_none()); + manager + .start_agent("project", config("project", dir.path())) + .await; + assert!(manager.get_agent("project").await.is_some()); + + manager + .start_agent("project", config("project", dir.path())) + .await; + assert!(manager.get_agent("project").await.is_some()); + + manager.stop_agent("project").await; + assert!(manager.get_agent("project").await.is_none()); + manager.stop_agent("project").await; + } + + #[tokio::test] + async fn manager_does_not_register_agent_when_watcher_cannot_initialize() { + let dir = tempfile::tempdir().unwrap(); + let provider = Arc::new(MultiTenantEngine::with_snapshots_dir( + dir.path().join("snapshots"), + TuningConfig::default(), + )); + let job_queue = Arc::new(JobQueue::new(provider.clone(), None, true)); + let manager = AgentManager::new(job_queue, provider); + manager + .start_agent( + "missing", + config("missing", &dir.path().join("does-not-exist")), + ) + .await; + assert!(manager.get_agent("missing").await.is_none()); + } +} diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 238a1c9..5573328 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -16,6 +16,7 @@ pub struct AgentConfig { pub watch_dir: String, pub throttle_ms: u64, pub state_file: Option, + pub included_paths: Vec, pub ignored_patterns: Vec, pub ignored_extensions: Vec, } diff --git a/src/agent/search.rs b/src/agent/search.rs index c288ab0..0bcfc50 100644 --- a/src/agent/search.rs +++ b/src/agent/search.rs @@ -1,6 +1,24 @@ use reqwest::Client; use scraper::{Html, Selector}; +fn parse_ddg_lite_results(html: &str, limit: usize) -> Vec { + if limit == 0 { + return Vec::new(); + } + + let document = Html::parse_document(html); + // DDG Lite structure: the anchor tag itself has class 'result-link'. + let link_selector = Selector::parse(".result-link").expect("static selector is valid"); + + document + .select(&link_selector) + .filter_map(|element| element.value().attr("href")) + .filter(|href| href.starts_with("http") && !href.contains("duckduckgo.com")) + .take(limit) + .map(str::to_owned) + .collect() +} + /// Search DuckDuckGo Lite and return top N result URLs pub async fn search_ddg_lite(query: &str, limit: usize) -> Result, String> { let client = Client::builder() @@ -38,31 +56,27 @@ pub async fn search_ddg_lite(query: &str, limit: usize) -> Result, S tracing::warn!("DDG Lite returned short response : {}", html); } - let document = Html::parse_document(&html); - - // DDG Lite structure: The anchor tag itself has class 'result-link' - let link_selector = Selector::parse(".result-link").unwrap(); - - let mut results = Vec::new(); + Ok(parse_ddg_lite_results(&html, limit)) +} - for element in document.select(&link_selector) { - if results.len() >= limit { - break; - } +#[cfg(test)] +mod tests { + use super::parse_ddg_lite_results; - if let Some(href) = element.value().attr("href") { - // DDG Lite links need decoding or sometimes are direct - // They look like: /l/?kh=-1&uddg=https%3A%2F%2Fexample.com%2F... - // or sometimes direct links depending on user agent? - // Actually usually plain links in Lite version but let's check. + #[test] + fn parses_external_links_filters_search_links_and_honors_limit() { + let html = r#" + one + internal + redirect + three + "#; - let clean_url = href.to_string(); - // Basic filtering of internal DDG links - if clean_url.starts_with("http") && !clean_url.contains("duckduckgo.com") { - results.push(clean_url); - } - } + assert_eq!( + parse_ddg_lite_results(html, 10), + vec!["https://example.com/one", "https://example.com/three"] + ); + assert_eq!(parse_ddg_lite_results(html, 1), vec!["https://example.com/one"]); + assert!(parse_ddg_lite_results(html, 0).is_empty()); } - - Ok(results) } diff --git a/src/agent/watcher.rs b/src/agent/watcher.rs index 49722ac..0cc3848 100644 --- a/src/agent/watcher.rs +++ b/src/agent/watcher.rs @@ -5,6 +5,68 @@ use std::sync::Arc; use tokio::sync::Mutex; use tracing::{debug, error}; +fn dispatch_event( + res: notify::Result, + ingester: Arc>, + state_file: Option, + handle: tokio::runtime::Handle, +) { + match res { + Ok(event) => { + if event + .paths + .iter() + .any(|path| Ingester::is_ignore_config_path(path)) + { + handle.spawn(async move { + let mut locked = ingester.lock().await; + if let Err(e) = locked.reload_filters_and_rescan().await { + error!("Error reloading ignore configuration: {}", e); + } + if let Some(ref sp) = state_file { + let _ = locked.save_state(sp); + } + }); + return; + } + + if event.kind.is_remove() { + for path in event.paths { + let ingester = ingester.clone(); + let state_file = state_file.clone(); + handle.spawn(async move { + let mut locked = ingester.lock().await; + if let Err(e) = locked.delete_file_path(path.clone()).await { + error!("Error processing deletion {:?}: {}", path, e); + } + if let Some(ref sp) = state_file { + let _ = locked.save_state(sp); + } + }); + } + } else { + for path in event.paths { + if path.exists() || path.extension().is_some() { + debug!("File event {:?}: {:?}", event.kind, path); + let ingester = ingester.clone(); + let state_file = state_file.clone(); + handle.spawn(async move { + let mut locked = ingester.lock().await; + if let Err(e) = locked.process_file_path(path.clone()).await { + debug!("Skipping file {:?}: {}", path, e); + } + if let Some(ref sp) = state_file { + let _ = locked.save_state(sp); + } + }); + } + } + } + } + Err(e) => error!("Watch error: {:?}", e), + } +} + pub struct Watcher { _watcher: RecommendedWatcher, } @@ -17,56 +79,10 @@ impl Watcher { ) -> notify::Result { let path_obj = Path::new(&path); - let tx_ingester = ingester.clone(); - let tx_state_file = state_file.clone(); let handle = tokio::runtime::Handle::current(); let watcher_plugin = move |res: notify::Result| { - match res { - Ok(event) => { - if event.kind.is_remove() { - for path in event.paths { - let ingester = tx_ingester.clone(); - let state_file = tx_state_file.clone(); - handle.spawn(async move { - let mut locked = ingester.lock().await; - if let Err(e) = locked.delete_file_path(path.clone()).await { - error!("Error processing deletion {:?}: {}", path, e); - } - - if let Some(ref sp) = state_file { - let _ = locked.save_state(sp); - } - }); - } - } else { - // Treat everything else as a potential update (Create, Modify, Rename, etc.) - // The Ingester's process_file_path checks if file exists and hashes it, - // so spurious events are cheap/safe. - for path in event.paths { - // Only process if it looks like a file we care about (simple check) - // detailed check is in ingester - if path.exists() || path.extension().is_some() { - debug!("File event {:?}: {:?}", event.kind, path); - let ingester = tx_ingester.clone(); - let state_file = tx_state_file.clone(); - handle.spawn(async move { - let mut locked = ingester.lock().await; - // this handles existence check internally - if let Err(e) = locked.process_file_path(path.clone()).await { - debug!("Skipping file {:?}: {}", path, e); - } - - if let Some(ref sp) = state_file { - let _ = locked.save_state(sp); - } - }); - } - } - } - } - Err(e) => error!("Watch error: {:?}", e), - } + dispatch_event(res, ingester.clone(), state_file.clone(), handle.clone()); }; let mut watcher = notify::recommended_watcher(watcher_plugin)?; @@ -76,3 +92,139 @@ impl Watcher { Ok(Self { _watcher: watcher }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::agent::{AgentConfig, ingester::Ingester}; + use crate::jobs::JobQueue; + use crate::multi_tenant::MultiTenantEngine; + use crate::config::TuningConfig; + use notify::EventKind; + use std::time::Duration; + + async fn wait_for_file_state( + ingester: &Arc>, + path: &std::path::Path, + expected: bool, + ) { + let normalized_path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let key = normalized_path.to_string_lossy().to_lowercase(); + let suffix = format!("/{}", path.file_name().unwrap().to_string_lossy().to_lowercase()); + for _ in 0..80 { + let present = ingester + .lock() + .await + .get_file_hashes() + .keys() + .any(|candidate| candidate == &key || candidate.ends_with(&suffix)); + if present == expected { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + let present = ingester + .lock() + .await + .get_file_hashes() + .keys() + .any(|candidate| candidate == &key || candidate.ends_with(&suffix)); + assert_eq!(present, expected, "timed out waiting for {:?}", path); + } + + #[tokio::test] + async fn watcher_can_attach_to_an_existing_directory() { + let dir = tempfile::tempdir().unwrap(); + let provider = Arc::new(MultiTenantEngine::with_snapshots_dir( + dir.path().join("snapshots"), + TuningConfig::default(), + )); + let queue = Arc::new(JobQueue::new(provider.clone(), None, true)); + let config = AgentConfig { + project_id: "watcher-test".to_string(), + watch_dir: dir.path().to_string_lossy().to_string(), + throttle_ms: 1, + state_file: None, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let ingester = Arc::new(Mutex::new(Ingester::new(config, queue))); + let watcher = Watcher::new(dir.path().to_string_lossy().to_string(), ingester, None); + assert!(watcher.is_ok()); + } + + #[tokio::test] + async fn watcher_processes_updates_removals_and_ignore_config_changes() { + let dir = tempfile::tempdir().unwrap(); + let state_file = dir.path().join("agent-state.json"); + let provider = Arc::new(MultiTenantEngine::with_snapshots_dir( + dir.path().join("snapshots"), + TuningConfig::default(), + )); + let queue = Arc::new(JobQueue::new(provider.clone(), None, true)); + let config = AgentConfig { + project_id: "watcher-events".to_string(), + watch_dir: dir.path().to_string_lossy().to_string(), + throttle_ms: 0, + state_file: Some(state_file.clone()), + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let note = dir.path().join("notes.md"); + std::fs::write(¬e, "first version").unwrap(); + let ingester = Arc::new(Mutex::new(Ingester::new(config, queue))); + ingester + .lock() + .await + .process_file_path(note.clone()) + .await + .unwrap(); + let handle = tokio::runtime::Handle::current(); + dispatch_event( + Ok(Event::new(EventKind::Create(notify::event::CreateKind::Any)).add_path(note.clone())), + ingester.clone(), + Some(state_file.clone()), + handle.clone(), + ); + wait_for_file_state(&ingester, ¬e, true).await; + for _ in 0..20 { + if state_file.exists() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(state_file.exists()); + + std::fs::write(¬e, "third version").unwrap(); + dispatch_event( + Ok(Event::new(EventKind::Modify(notify::event::ModifyKind::Any)).add_path(note.clone())), + ingester.clone(), + Some(state_file.clone()), + handle.clone(), + ); + wait_for_file_state(&ingester, ¬e, true).await; + + let ignore = dir.path().join(".cuemapignore"); + std::fs::write(&ignore, "notes.md\n").unwrap(); + dispatch_event( + Ok(Event::new(EventKind::Modify(notify::event::ModifyKind::Any)).add_path(ignore)), + ingester.clone(), + Some(state_file.clone()), + handle.clone(), + ); + wait_for_file_state(&ingester, ¬e, false).await; + + std::fs::remove_file(¬e).unwrap(); + dispatch_event( + Ok(Event::new(EventKind::Remove(notify::event::RemoveKind::Any)).add_path(note.clone())), + ingester.clone(), + Some(state_file), + handle, + ); + wait_for_file_state(&ingester, ¬e, false).await; + + dispatch_event(Err(notify::Error::generic("synthetic watcher error")), ingester, None, tokio::runtime::Handle::current()); + } +} diff --git a/src/api.rs b/src/api.rs index eed4aa9..a59a319 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,5 +1,6 @@ use crate::auth::AuthConfig; -use crate::jobs::JobQueue; +use crate::jobs::{Job, JobQueue}; +use crate::intent::IntentTarget; use crate::metrics::MetricsCollector; use crate::multi_tenant::{validate_project_id, MultiTenantEngine}; use crate::normalization::normalize_cue; @@ -27,10 +28,14 @@ pub struct AddMemoryRequest { pub cues: Vec, #[serde(default)] pub source_key: Option, + /// Original event timestamp as Unix seconds. When omitted, ingestion time is used. + #[serde(default)] + pub event_time: Option, #[serde(default)] pub metadata: Option>, + /// Optional precomputed embedding for opt-in semantic retrieval. #[serde(default)] - pub cuepacks: Option>, + pub embedding: Option>, #[serde(default)] pub disable_temporal_chunking: bool, #[serde(default)] @@ -64,6 +69,13 @@ pub struct RecallRequest { pub cues: Vec, #[serde(default)] pub query_text: Option, + /// Optional precomputed query embedding for opt-in semantic retrieval. + #[serde(default)] + pub query_embedding: Option>, + /// Selects lexical-only, semantic-only, or combined query signals. + /// Hybrid preserves the existing behavior when omitted. + #[serde(default)] + pub semantic_mode: crate::semantic::SemanticRecallMode, #[serde(default)] pub query_time: Option, #[serde(default = "default_limit")] @@ -87,8 +99,6 @@ pub struct RecallRequest { #[serde(default = "default_depth")] pub depth: usize, #[serde(default)] - pub cuepacks: Option>, - #[serde(default)] pub parent_fusion: ParentFusionMode, #[serde(default = "default_parent_fusion_limit")] pub parent_fusion_limit: usize, @@ -116,6 +126,13 @@ pub struct RecallRequest { pub cuebridge_gap_limit: usize, } +#[derive(Debug, Deserialize, Serialize)] +pub struct IntentClassificationRequest { + pub text: String, + #[serde(default)] + pub target: IntentTarget, +} + #[derive(Debug, Deserialize)] pub struct ProjectExportQuery { #[serde(default)] @@ -216,17 +233,15 @@ impl Default for ParentFusionMode { } } -fn apply_query_intent( +fn apply_query_plan( ctx: &crate::projects::ProjectContext, - cuepack_registry: &crate::cuepacks::CuePackRegistry, - cuepack_selection: Option<&[String]>, query_text: Option<&str>, query_time: Option<&str>, expanded_cues: &mut Vec<(String, f64)>, -) -> Option { +) -> Option { let query_text = query_text?; let total_memories = ctx.main.total_memories().max(1); - let intent = crate::facets::compile_query_intent_with_cuepacks(query_text, query_time, |cue| { + let intent = crate::facets::compile_query_plan_with_reference_time(query_text, query_time, |cue| { let df = ctx.main.get_cue_frequency(cue); if df == 0 { return false; @@ -279,7 +294,7 @@ fn apply_query_intent( } else { df <= 16 || df * 5 <= total_memories } - }, cuepack_registry, cuepack_selection); + }); for (cue, multiplier) in &intent.cue_weight_adjustments { if let Some((_, weight)) = expanded_cues @@ -383,12 +398,12 @@ struct OrderedReconstructionCandidate { fn should_run_ordered_reconstruction( mode: OrderedReconstructionMode, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) -> bool { match mode { OrderedReconstructionMode::Off => false, OrderedReconstructionMode::Force => true, - OrderedReconstructionMode::Auto => query_intent + OrderedReconstructionMode::Auto => query_plan .map(|intent| { intent.labels.iter().any(|label| { label == "ordered_reconstruction" @@ -627,12 +642,12 @@ struct EvidenceCoverageCandidate { fn should_run_evidence_coverage( mode: EvidenceCoverageMode, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) -> bool { match mode { EvidenceCoverageMode::Off => false, EvidenceCoverageMode::Force => true, - EvidenceCoverageMode::Auto => query_intent + EvidenceCoverageMode::Auto => query_plan .map(|intent| { intent .labels @@ -1189,13 +1204,13 @@ struct SlateRerankIntent { fn slate_rerank_requested( ordered_mode: OrderedReconstructionMode, evidence_mode: EvidenceCoverageMode, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) -> bool { if ordered_mode == OrderedReconstructionMode::Off && evidence_mode == EvidenceCoverageMode::Off { return false; } - query_intent + query_plan .map(|intent| { intent.labels.iter().any(|label| { label == "ordered_reconstruction" @@ -1208,9 +1223,9 @@ fn slate_rerank_requested( } fn slate_rerank_intent( - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) -> SlateRerankIntent { - let Some(intent) = query_intent else { + let Some(intent) = query_plan else { return SlateRerankIntent::default(); }; let target_role = if intent.labels.iter().any(|label| label == "source_user") { @@ -1459,11 +1474,11 @@ fn apply_slate_rerank( expanded_cues: &[(String, f64)], ordered_mode: OrderedReconstructionMode, evidence_mode: EvidenceCoverageMode, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, limit: usize, ) -> usize { if all_results.len() <= SLATE_RERANK_PROTECTED_RESULTS - || !slate_rerank_requested(ordered_mode, evidence_mode, query_intent) + || !slate_rerank_requested(ordered_mode, evidence_mode, query_plan) { return 0; } @@ -1486,7 +1501,7 @@ fn apply_slate_rerank( return 0; } - let intent = slate_rerank_intent(query_intent); + let intent = slate_rerank_intent(query_plan); let mut selected = candidates .iter() .take(SLATE_RERANK_PROTECTED_RESULTS) @@ -1614,14 +1629,14 @@ struct ParentFusionHit { fn should_run_parent_fusion( results: &[crate::engine::RecallResult], mode: ParentFusionMode, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) -> bool { match mode { ParentFusionMode::Off => false, ParentFusionMode::Force => true, ParentFusionMode::Auto => { - if !query_supports_parent_fusion(query_intent, query_text) { + if !query_supports_parent_fusion(query_plan, query_text) { return false; } @@ -1639,10 +1654,10 @@ fn should_run_parent_fusion( } fn query_supports_parent_fusion( - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) -> bool { - if query_intent + if query_plan .is_some_and(|intent| intent.labels.iter().any(|label| label == "temporal_order")) { return true; @@ -2026,12 +2041,19 @@ fn source_role_from_metadata(metadata: &HashMap) -> O } fn source_answer_projection_requested( - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) -> bool { - let Some(intent) = query_intent else { + let Some(intent) = query_plan else { return false; }; + if !intent + .labels + .iter() + .any(|label| label == "__semantic_facets_removed__") + { + return false; + } let has_assistant_source = intent.labels.iter().any(|label| label == "source_assistant"); if has_assistant_source { return true; @@ -2073,11 +2095,11 @@ fn query_wants_list_answer(query_text: Option<&str>) -> bool { fn source_answer_projection_cues( ctx: &crate::projects::ProjectContext, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, all_results: &[crate::engine::RecallResult], ) -> Vec<(String, f64)> { - if !source_answer_projection_requested(query_intent, query_text) { + if !source_answer_projection_requested(query_plan, query_text) { return Vec::new(); } if ctx.main.get_cue_frequency("source_role:assistant") == 0 { @@ -2120,13 +2142,20 @@ fn source_answer_projection_cues( fn source_prompt_projection_cues( ctx: &crate::projects::ProjectContext, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, all_results: &[crate::engine::RecallResult], ) -> Vec<(String, f64)> { - let Some(intent) = query_intent else { + let Some(intent) = query_plan else { return Vec::new(); }; + if !intent + .labels + .iter() + .any(|label| label == "__semantic_facets_removed__") + { + return Vec::new(); + } if !intent.labels.iter().any(|label| label == "source_answer") || !intent.labels.iter().any(|label| label == "source_assistant") { @@ -2176,10 +2205,15 @@ fn source_prompt_projection_cues( } fn user_context_projection_requested( - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) -> bool { - if query_intent + if !query_plan + .is_some_and(|intent| intent.labels.iter().any(|label| label == "__semantic_facets_removed__")) + { + return false; + } + if query_plan .map(|intent| { intent.labels.iter().any(|label| { matches!( @@ -2384,9 +2418,9 @@ fn projection_pivot_matches_context( } fn suppress_user_context_projection_for_intent( - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) -> bool { - query_intent + query_plan .map(|intent| { intent .labels @@ -2398,22 +2432,27 @@ fn suppress_user_context_projection_for_intent( fn user_context_projection_cues( ctx: &crate::projects::ProjectContext, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, all_results: &[crate::engine::RecallResult], ) -> Vec<(String, f64)> { - if !user_context_projection_requested(query_intent, query_text) { + if !query_plan + .is_some_and(|intent| intent.labels.iter().any(|label| label == "__semantic_facets_removed__")) + { + return Vec::new(); + } + if !user_context_projection_requested(query_plan, query_text) { return Vec::new(); } if ctx.main.get_cue_frequency("source_role:user") == 0 { return Vec::new(); } - if suppress_user_context_projection_for_intent(query_intent) { + if suppress_user_context_projection_for_intent(query_plan) { return Vec::new(); } - let allow_high_confidence_pivot = query_intent + let allow_high_confidence_pivot = query_plan .map(|intent| { intent .labels @@ -2608,14 +2647,14 @@ struct StandingInstructionProjection { } fn standing_instruction_projection_requested( - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) -> bool { - query_intent + query_plan .map(|intent| { intent .labels .iter() - .any(|label| label == "instruction_applicable") + .any(|label| label == "__semantic_facets_removed__") }) .unwrap_or(false) } @@ -2783,10 +2822,10 @@ fn standing_instruction_projection_anchors(query_text: Option<&str>) -> Vec, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) -> StandingInstructionProjection { - if !standing_instruction_projection_requested(query_intent) { + if !standing_instruction_projection_requested(query_plan) { return StandingInstructionProjection::default(); } if ctx.main.get_cue_frequency("type:standing_instruction") == 0 { @@ -2931,13 +2970,13 @@ struct PreferenceProjection { anchors: Vec, } -fn preference_projection_requested(query_intent: Option<&crate::facets::QueryIntent>) -> bool { - query_intent +fn preference_projection_requested(query_plan: Option<&crate::facets::StructuralQueryPlan>) -> bool { + query_plan .map(|intent| { intent .labels .iter() - .any(|label| label == "preference_applicable") + .any(|label| label == "__semantic_facets_removed__") }) .unwrap_or(false) } @@ -3043,10 +3082,10 @@ fn preference_projection_anchors(query_text: Option<&str>) -> Vec { fn preference_projection_cues( ctx: &crate::projects::ProjectContext, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) -> PreferenceProjection { - if !preference_projection_requested(query_intent) { + if !preference_projection_requested(query_plan) { return PreferenceProjection::default(); } @@ -3210,12 +3249,17 @@ fn merge_preference_projection_results( fn apply_user_context_adjacency_preference( all_results: &mut [crate::engine::RecallResult], - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, query_text: Option<&str>, ) { + if !query_plan + .is_some_and(|intent| intent.labels.iter().any(|label| label == "__semantic_facets_removed__")) + { + return; + } const MAX_ADJACENCY_PIVOTS: usize = 4; - if !user_context_projection_requested(query_intent, query_text) { + if !user_context_projection_requested(query_plan, query_text) { return; } @@ -3335,23 +3379,23 @@ fn apply_user_context_adjacency_preference( } } -fn decision_projection_requested(query_intent: Option<&crate::facets::QueryIntent>) -> bool { - query_intent +fn decision_projection_requested(query_plan: Option<&crate::facets::StructuralQueryPlan>) -> bool { + query_plan .map(|intent| { intent .labels .iter() - .any(|label| label == "decision_selection") + .any(|label| label == "__semantic_facets_removed__") }) .unwrap_or(false) } fn decision_projection_cues( ctx: &crate::projects::ProjectContext, - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, all_results: &[crate::engine::RecallResult], ) -> Vec<(String, f64)> { - if !decision_projection_requested(query_intent) { + if !decision_projection_requested(query_plan) { return Vec::new(); } @@ -3383,7 +3427,7 @@ fn decision_projection_cues( if ctx.main.get_cue_frequency("type:selection") > 0 { cues.push(("type:selection".to_string(), 2.6)); } - if query_intent + if query_plan .map(|intent| intent.labels.iter().any(|label| label == "naming_decision")) .unwrap_or(false) && ctx.main.get_cue_frequency("type:naming") > 0 @@ -3428,9 +3472,9 @@ fn merge_decision_projection_results( fn apply_source_role_preference( all_results: &mut [crate::engine::RecallResult], - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) { - let Some(intent) = query_intent else { + let Some(intent) = query_plan else { return; }; @@ -3467,82 +3511,25 @@ fn apply_source_role_preference( } fn apply_decision_adjacency_preference( - all_results: &mut [crate::engine::RecallResult], - query_intent: Option<&crate::facets::QueryIntent>, + _all_results: &mut [crate::engine::RecallResult], + _query_plan: Option<&crate::facets::StructuralQueryPlan>, ) { - if !decision_projection_requested(query_intent) { - return; - } - - let Some((pivot_idx, pivot_session, pivot_score)) = all_results - .iter() - .enumerate() - .filter_map(|(idx, result)| { - if crate::facets::has_decision_selection_language(&result.content) { - return None; - } - let session = source_session_cue_from_metadata(&result.metadata)?; - Some((idx, session, result.score)) - }) - .max_by(|(_, _, left), (_, _, right)| { - left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal) - }) - else { - return; - }; - - let mut session_positions = all_results - .iter() - .enumerate() - .filter_map(|(idx, result)| { - if source_session_cue_from_metadata(&result.metadata).as_deref() - == Some(pivot_session.as_str()) - { - Some((idx, result.created_at)) - } else { - None - } - }) - .collect::>(); - session_positions.sort_by(|(_, left), (_, right)| { - left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal) - }); - - let Some(pivot_position) = session_positions - .iter() - .position(|(idx, _)| *idx == pivot_idx) - else { - return; - }; - - for (position, (idx, _)) in session_positions.iter().enumerate().skip(pivot_position + 1) { - let distance = position - pivot_position; - if distance > 8 { - break; - } - if !crate::facets::has_decision_selection_language(&all_results[*idx].content) { - continue; - } - let bonus = pivot_score / (distance * distance) as f64; - all_results[*idx].score += bonus; - all_results[*idx].metadata.insert( - "decision_adjacency_boost".to_string(), - serde_json::json!({ - "pivot_session": pivot_session, - "distance": distance, - "bonus": bonus - }), - ); - } } fn apply_source_answer_adjacency_preference( all_results: &mut [crate::engine::RecallResult], - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ) { - let Some(intent) = query_intent else { + let Some(intent) = query_plan else { return; }; + if !intent + .labels + .iter() + .any(|label| label == "__semantic_facets_removed__") + { + return; + } if !intent.labels.iter().any(|label| label == "source_answer") || !intent.labels.iter().any(|label| label == "source_assistant") { @@ -3632,8 +3619,6 @@ pub struct RecallGroundedRequest { pub disable_alias_expansion: bool, #[serde(default = "default_expansion_depth")] pub expansion_depth: usize, - #[serde(default)] - pub cuepacks: Option>, } fn default_true() -> bool { @@ -3743,6 +3728,19 @@ pub struct CreateProjectRequest { pub struct SetWatchDirRequest { pub watch_dir: String, #[serde(default)] + pub included_paths: Option>, + #[serde(default)] + pub ignored_patterns: Option>, + #[serde(default)] + pub ignored_extensions: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct PreviewDirectoryRequest { + pub watch_dir: String, + #[serde(default)] + pub included_paths: Option>, + #[serde(default)] pub ignored_patterns: Option>, #[serde(default)] pub ignored_extensions: Option>, @@ -3772,7 +3770,6 @@ pub struct EngineState { pub cloud_backup: Option>, pub context_signer: Option>, pub agent_manager: Arc, - pub cuepack_registry: Arc, } struct StoredMemoryOutcome { @@ -3795,10 +3792,10 @@ pub fn routes( cloud_backup: Option>, context_signer: Option>, agent_manager: Arc, - cuepack_registry: Arc, ) -> Router { let mut router = Router::new() .route("/", get(root)) + .route("/intent/classify", post(classify_intent)) .route("/memories", post(add_memory)) .route("/memories/batch", post(add_memories_batch)) .route("/recall", post(recall)) @@ -3814,7 +3811,10 @@ pub fn routes( get(project_artifacts).post(reload_project_artifacts), ) .route("/projects/:id/export", get(export_project)) - .route("/projects/:id/watch-dir", post(set_project_watch_dir)) + .route( + "/projects/:id/watch-dir", + get(get_project_watch_dir).post(set_project_watch_dir), + ) .route("/aliases", post(add_alias).get(get_aliases)) .route("/aliases/merge", post(merge_aliases)) .route("/lexicon/inspect/:cue", get(lexicon_inspect)) @@ -3824,6 +3824,7 @@ pub fn routes( .route("/ingest/url", post(ingest_url)) .route("/ingest/content", post(ingest_content)) .route("/ingest/file", post(ingest_file)) + .route("/ingest/directory/preview", post(preview_directory)) .route("/jobs/status", get(jobs_status)) .route("/debug/analyze-text", post(debug_analyze_text)) .route("/metrics", get(prometheus_metrics)) @@ -3842,7 +3843,6 @@ pub fn routes( cloud_backup, context_signer, agent_manager, - cuepack_registry, }); // Add auth middleware if enabled @@ -3860,10 +3860,44 @@ async fn root() -> impl IntoResponse { Json(serde_json::json!({ "name": "CueMap Rust Engine", "version": env!("CARGO_PKG_VERSION"), - "description": "High-performance Temporal-Associative Memory Store" + "description": "High-performance Temporal-Associative Memory Store", + "capabilities": [ + "repository_ingestion_scope_v1", + "semantic_retrieval_v1", + "chunk_embeddings_v1", + "intent_classification_v1", + "intent_job_status_v1" + ] })) } +async fn classify_intent( + State(state): State, + headers: HeaderMap, + Json(req): Json, +) -> (StatusCode, Json) { + let project_id = match extract_project_id(&headers) { + Ok(id) => id, + Err(error) => return error, + }; + let ctx = match state.mt_engine.get_or_create_project(project_id) { + Ok(ctx) => ctx, + Err(error) => { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({"error": error})), + ) + } + }; + match ctx.main.classify_intent(&req.text, req.target) { + Ok(classification) => (StatusCode::OK, Json(serde_json::json!(classification))), + Err(error) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({"error": error})), + ), + } +} + // Handlers fn extract_project_id( headers: &HeaderMap, @@ -3896,6 +3930,26 @@ fn extract_project_id_optional(headers: &HeaderMap) -> Option { .filter(|s| validate_project_id(s)) } +fn source_event_time( + explicit: Option, + metadata: Option<&HashMap>, +) -> Option { + if explicit.is_some() { + return explicit; + } + + let value = metadata?.get("source_timestamp")?; + if let Some(timestamp) = value.as_f64() { + return (timestamp.is_finite() && timestamp >= 0.0).then_some(timestamp); + } + + let parsed = chrono::DateTime::parse_from_rfc3339(value.as_str()?).ok()?; + Some( + parsed.timestamp() as f64 + + f64::from(parsed.timestamp_subsec_nanos()) / 1_000_000_000.0, + ) +} + async fn store_memory_request( state: &EngineState, project_id: &str, @@ -3938,14 +3992,25 @@ async fn store_memory_request( content, cues, source_key, + event_time, metadata, - cuepacks, + embedding, disable_temporal_chunking, async_ingest: _, minimal_response: _, trace_timing: _, } = req; + if event_time.is_some_and(|timestamp| !timestamp.is_finite() || timestamp < 0.0) { + return Err(( + StatusCode::BAD_REQUEST, + serde_json::json!({ + "error": "event_time must be a finite, non-negative Unix timestamp in seconds" + }), + )); + } + let event_time = source_event_time(event_time, metadata.as_ref()); + phase_start = Instant::now(); let mut initial_cues = cues; if initial_cues.is_empty() { @@ -3993,17 +4058,31 @@ async fn store_memory_request( }; phase_start = Instant::now(); - let cuepack_selection = cuepacks.as_deref(); - let memory_id = ctx.main.add_memory_with_cuepacks_and_source_key( - content, - report.accepted, - metadata, - MainStats::default(), - disable_temporal_chunking, - &state.cuepack_registry, - cuepack_selection, - source_key, - ); + let memory_id = if let Some(source_key) = source_key { + ctx.main + .upsert_memory_with_source_key_and_options_and_vector( + source_key, + content, + report.accepted, + metadata, + None, + false, + true, + disable_temporal_chunking, + event_time, + embedding, + ) + } else { + ctx.main.add_memory_with_event_time_and_vector( + content, + report.accepted, + metadata, + MainStats::default(), + disable_temporal_chunking, + event_time, + embedding, + ) + }; if trace_timing { timing.insert( "engine_add_ms".to_string(), @@ -4047,6 +4126,13 @@ async fn add_memory( match store_memory_request(&state, &project_id, req).await { Ok(outcome) => { + state + .job_queue + .enqueue(Job::ClassifyMemory { + project_id: project_id.clone(), + memory_id: outcome.memory_id, + }) + .await; let mut body = if outcome.minimal_response { serde_json::json!({ "id": outcome.memory_id, @@ -4102,6 +4188,13 @@ async fn add_memories_batch( match store_memory_request(&state, &project_id, memory_req).await { Ok(outcome) => { + state + .job_queue + .enqueue(Job::ClassifyMemory { + project_id: project_id.clone(), + memory_id: outcome.memory_id, + }) + .await; ids.push(outcome.memory_id); if let Some(timings) = per_memory_timings.as_mut() { timings.push(serde_json::json!({ @@ -4159,7 +4252,6 @@ async fn recall( let EngineState { ref mt_engine, ref job_queue, - ref cuepack_registry, .. } = &state; @@ -4176,21 +4268,41 @@ async fn recall( Err(_) => return (serde_json::json!({"project_id": project_id, "error": "Capacity reached"}), None), }; - // Collect cues - let mut cues_to_process = req.cues.clone(); + let semantic_only = + req.semantic_mode == crate::semantic::SemanticRecallMode::Semantic; + + // Collect lexical cues unless this is an explicitly semantic-only query. + let mut cues_to_process = if semantic_only { + Vec::new() + } else { + req.cues.clone() + }; // Extract mandatory constraints from explicit cues - let mandatory_cues: Vec = req.cues.iter() - .map(|c| normalize_cue(c, &ctx.normalization).0) - .collect(); + let mandatory_cues: Vec = if semantic_only { + Vec::new() + } else { + req.cues + .iter() + .map(|c| normalize_cue(c, &ctx.normalization).0) + .collect() + }; let mandatory_cues_ref = if mandatory_cues.is_empty() { None } else { Some(&mandatory_cues) }; let (original_tokens, _lexicon_mids) = if let Some(text) = &req.query_text { - let (resolved, lex_mids, tokens) = ctx.resolve_cues_from_text(text, false); - cues_to_process.extend(resolved); - (tokens, lex_mids) + if semantic_only { + (Vec::new(), Vec::new()) + } else { + let (resolved, lex_mids, tokens) = ctx.resolve_cues_from_text(text, false); + cues_to_process.extend(resolved); + (tokens, lex_mids) + } } else { - (req.cues.clone(), Vec::new()) + if semantic_only { + (Vec::new(), Vec::new()) + } else { + (req.cues.clone(), Vec::new()) + } }; // Normalize query cues @@ -4206,14 +4318,49 @@ async fn recall( } else { ctx.expand_query_cues(normalized_cues, &original_tokens) }; - let query_intent = apply_query_intent( - &ctx, - &state.cuepack_registry, - req.cuepacks.as_deref(), - req.query_text.as_deref(), - req.query_time.as_deref(), - &mut expanded_cues, - ); + let query_plan = if semantic_only { + None + } else { + apply_query_plan( + &ctx, + req.query_text.as_deref(), + req.query_time.as_deref(), + &mut expanded_cues, + ) + }; + let query_embedding = match req.semantic_mode { + crate::semantic::SemanticRecallMode::Lexical => None, + crate::semantic::SemanticRecallMode::Semantic + | crate::semantic::SemanticRecallMode::Hybrid => req + .query_embedding + .clone() + .or_else(|| { + req.query_text + .as_deref() + .and_then(|text| ctx.main.encode_semantic_text(text)) + }), + }; + let query_intent = if req.semantic_mode + == crate::semantic::SemanticRecallMode::Hybrid + { + req.query_text + .as_deref() + .and_then(|text| { + query_embedding + .as_deref() + .and_then(|embedding| { + ctx.main.classify_intent_with_embedding( + text, + IntentTarget::Query, + embedding, + ) + .ok() + }) + .or_else(|| ctx.main.classify_intent(text, IntentTarget::Query).ok()) + }) + } else { + None + }; let mut all_results: Vec = Vec::new(); let mut used_pivot_memory_ids = std::collections::HashSet::new(); let limit = req.limit.max(1); @@ -4226,17 +4373,37 @@ async fn recall( let heatmap = ctx.market_heatmap.read().ok(); let heatmap_ref = heatmap.as_deref(); - ctx.main.recall_weighted( - expanded_cues.clone(), - current_limit, - false, - req.min_intersection, - req.expansion_depth, - req.explain, - req.disable_salience_bias, - heatmap_ref, - mandatory_cues_ref - ) + if req.semantic_mode + == crate::semantic::SemanticRecallMode::Hybrid + { + ctx.main + .recall_weighted_with_query_embedding_rerank_only_and_intent( + expanded_cues.clone(), + current_limit, + false, + req.min_intersection, + req.expansion_depth, + req.explain, + req.disable_salience_bias, + heatmap_ref, + mandatory_cues_ref, + query_embedding.as_deref(), + query_intent.as_ref(), + ) + } else { + ctx.main.recall_weighted_with_query_embedding( + expanded_cues.clone(), + current_limit, + false, + req.min_intersection, + req.expansion_depth, + req.explain, + req.disable_salience_bias, + heatmap_ref, + mandatory_cues_ref, + query_embedding.as_deref(), + ) + } }; // Add hop metadata @@ -4281,7 +4448,7 @@ async fn recall( let source_answer_projection_expansions = source_answer_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), &all_results, ); @@ -4305,7 +4472,7 @@ async fn recall( let source_prompt_projection_expansions = source_prompt_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), &all_results, ); @@ -4333,7 +4500,7 @@ async fn recall( let user_context_projection_expansions = user_context_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), &all_results, ); @@ -4356,7 +4523,7 @@ async fn recall( let standing_instruction_projection = standing_instruction_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ); if !standing_instruction_projection.cues.is_empty() { @@ -4383,7 +4550,7 @@ async fn recall( let preference_projection = preference_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ); if !preference_projection.cues.is_empty() { @@ -4409,7 +4576,7 @@ async fn recall( } let decision_projection_expansions = - decision_projection_cues(&ctx, query_intent.as_ref(), &all_results); + decision_projection_cues(&ctx, query_plan.as_ref(), &all_results); if !decision_projection_expansions.is_empty() { let heatmap = ctx.market_heatmap.read().ok(); let heatmap_ref = heatmap.as_deref(); @@ -4427,14 +4594,14 @@ async fn recall( merge_decision_projection_results(&mut all_results, projection_results); } - apply_source_role_preference(&mut all_results, query_intent.as_ref()); - apply_source_answer_adjacency_preference(&mut all_results, query_intent.as_ref()); + apply_source_role_preference(&mut all_results, query_plan.as_ref()); + apply_source_answer_adjacency_preference(&mut all_results, query_plan.as_ref()); apply_user_context_adjacency_preference( &mut all_results, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ); - apply_decision_adjacency_preference(&mut all_results, query_intent.as_ref()); + apply_decision_adjacency_preference(&mut all_results, query_plan.as_ref()); all_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal)); let results = all_results; @@ -4462,8 +4629,7 @@ async fn recall( serde_json::json!({ "query_cues": cues_to_process, "expanded_cues": expanded_cues, - "query_intent": query_intent, - "cuepacks": req.cuepacks.clone().unwrap_or_else(|| vec!["default".to_string()]), + "query_plan": query_plan, "source_answer_projection_cues": source_answer_projection_expansions, "source_prompt_projection_cues": source_prompt_projection_expansions, "user_context_projection_cues": user_context_projection_expansions, @@ -4540,14 +4706,22 @@ async fn recall( // Collect cues phase_start = Instant::now(); - let mut cues_to_process = req.cues.clone(); + let semantic_only = req.semantic_mode == crate::semantic::SemanticRecallMode::Semantic; + let mut cues_to_process = if semantic_only { + Vec::new() + } else { + req.cues.clone() + }; // Extract mandatory constraints from explicit cues - let mandatory_cues: Vec = req - .cues - .iter() - .map(|c| normalize_cue(c, &ctx.normalization).0) - .collect(); + let mandatory_cues: Vec = if semantic_only { + Vec::new() + } else { + req.cues + .iter() + .map(|c| normalize_cue(c, &ctx.normalization).0) + .collect() + }; let mandatory_cues_ref = if mandatory_cues.is_empty() { None } else { @@ -4556,7 +4730,8 @@ async fn recall( let mut lexicon_memory_ids: Vec = Vec::new(); let mut tokens_from_text = Vec::new(); - if let Some(ref text) = req.query_text { + if !semantic_only { + if let Some(ref text) = req.query_text { // 1. Lexicon Recall let (resolved, lex_mids, tokens) = ctx.resolve_cues_from_text(text, false); cues_to_process.extend(resolved); @@ -4566,10 +4741,11 @@ async fn recall( tokens_from_text = tokens; for token in &tokens_from_text { if !cues_to_process.contains(token) { - cues_to_process.push(token.clone()); - } + cues_to_process.push(token.clone()); + } } } + } if trace_timing { timing.insert( "query_resolution_ms".to_string(), @@ -4601,7 +4777,9 @@ async fn recall( // Expand aliases phase_start = Instant::now(); - let original_tokens = if req.query_text.is_some() { + let original_tokens = if semantic_only { + Vec::new() + } else if req.query_text.is_some() { tokens_from_text.clone() } else { req.cues.clone() @@ -4625,17 +4803,19 @@ async fn recall( ); } phase_start = Instant::now(); - let query_intent = apply_query_intent( - &ctx, - cuepack_registry, - req.cuepacks.as_deref(), - req.query_text.as_deref(), - req.query_time.as_deref(), - &mut expanded_cues, - ); + let query_plan = if semantic_only { + None + } else { + apply_query_plan( + &ctx, + req.query_text.as_deref(), + req.query_time.as_deref(), + &mut expanded_cues, + ) + }; if trace_timing { timing.insert( - "query_intent_ms".to_string(), + "query_plan_ms".to_string(), serde_json::json!(phase_start.elapsed().as_secs_f64() * 1000.0), ); timing.insert( @@ -4647,6 +4827,41 @@ async fn recall( let mut used_pivot_memory_ids = std::collections::HashSet::new(); let limit = req.limit.max(1); let depth = req.depth.max(1); + phase_start = Instant::now(); + let query_embedding = match req.semantic_mode { + crate::semantic::SemanticRecallMode::Lexical => None, + crate::semantic::SemanticRecallMode::Semantic + | crate::semantic::SemanticRecallMode::Hybrid => req + .query_embedding + .clone() + .or_else(|| { + req.query_text + .as_deref() + .and_then(|text| ctx.main.encode_semantic_text(text)) + }), + }; + let query_intent = if req.semantic_mode == crate::semantic::SemanticRecallMode::Hybrid { + req.query_text + .as_deref() + .and_then(|text| { + query_embedding + .as_deref() + .and_then(|embedding| { + ctx.main + .classify_intent_with_embedding(text, IntentTarget::Query, embedding) + .ok() + }) + .or_else(|| ctx.main.classify_intent(text, IntentTarget::Query).ok()) + }) + } else { + None + }; + if trace_timing { + timing.insert( + "semantic_query_embedding_ms".to_string(), + serde_json::json!(phase_start.elapsed().as_secs_f64() * 1000.0), + ); + } let mut base_recall_timing: Option = None; let mut base_recall_total_ms = 0.0; let mut base_recall_calls = 0usize; @@ -4659,17 +4874,38 @@ async fn recall( let heatmap_ref = heatmap.as_deref(); if trace_timing { - let (results, call_timing) = ctx.main.recall_weighted_with_timing( - expanded_cues.clone(), - current_limit, - false, - req.min_intersection, - req.expansion_depth, - req.explain, + let (results, call_timing) = if req.semantic_mode + == crate::semantic::SemanticRecallMode::Hybrid + { + ctx.main + .recall_weighted_with_query_embedding_rerank_only_with_intent_with_timing( + expanded_cues.clone(), + current_limit, + false, + req.min_intersection, + req.expansion_depth, + req.explain, req.disable_salience_bias, - heatmap_ref, - mandatory_cues_ref, - ); + heatmap_ref, + mandatory_cues_ref, + query_embedding.as_deref(), + query_intent.as_ref(), + ) + } else { + ctx.main + .recall_weighted_with_query_embedding_with_timing( + expanded_cues.clone(), + current_limit, + false, + req.min_intersection, + req.expansion_depth, + req.explain, + req.disable_salience_bias, + heatmap_ref, + mandatory_cues_ref, + query_embedding.as_deref(), + ) + }; base_recall_total_ms += call_timing.total_ms; base_recall_calls += 1; if base_recall_timing.is_none() { @@ -4677,19 +4913,37 @@ async fn recall( } results } else { - ctx.main.recall_weighted( - expanded_cues.clone(), - current_limit, - false, - req.min_intersection, - req.expansion_depth, - req.explain, + if req.semantic_mode == crate::semantic::SemanticRecallMode::Hybrid { + ctx.main + .recall_weighted_with_query_embedding_rerank_only_and_intent( + expanded_cues.clone(), + current_limit, + false, + req.min_intersection, + req.expansion_depth, + req.explain, req.disable_salience_bias, - heatmap_ref, - mandatory_cues_ref, - ) - } - }; + heatmap_ref, + mandatory_cues_ref, + query_embedding.as_deref(), + query_intent.as_ref(), + ) + } else { + ctx.main.recall_weighted_with_query_embedding( + expanded_cues.clone(), + current_limit, + false, + req.min_intersection, + req.expansion_depth, + req.explain, + req.disable_salience_bias, + heatmap_ref, + mandatory_cues_ref, + query_embedding.as_deref(), + ) + } + } + }; // Add hop metadata for r in &mut results { @@ -4737,7 +4991,7 @@ async fn recall( phase_start = Instant::now(); let source_answer_projection_expansions = source_answer_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), &all_results, ); @@ -4771,7 +5025,7 @@ async fn recall( phase_start = Instant::now(); let source_prompt_projection_expansions = source_prompt_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), &all_results, ); @@ -4809,7 +5063,7 @@ async fn recall( phase_start = Instant::now(); let user_context_projection_expansions = user_context_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), &all_results, ); @@ -4843,7 +5097,7 @@ async fn recall( phase_start = Instant::now(); let standing_instruction_projection = standing_instruction_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ); if !standing_instruction_projection.cues.is_empty() { @@ -4881,7 +5135,7 @@ async fn recall( phase_start = Instant::now(); let preference_projection = preference_projection_cues( &ctx, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ); if !preference_projection.cues.is_empty() { @@ -4918,7 +5172,7 @@ async fn recall( phase_start = Instant::now(); let decision_projection_expansions = - decision_projection_cues(&ctx, query_intent.as_ref(), &all_results); + decision_projection_cues(&ctx, query_plan.as_ref(), &all_results); if !decision_projection_expansions.is_empty() { let heatmap = ctx.market_heatmap.read().ok(); let heatmap_ref = heatmap.as_deref(); @@ -4957,7 +5211,7 @@ async fn recall( .map(|artifacts| { artifacts.gap_expansions( &expanded_cues, - query_intent.as_ref(), + query_plan.as_ref(), &original_tokens, |cue| ctx.main.get_cue_frequency(cue) > 0, req.cuebridge_gap_limit, @@ -5013,7 +5267,7 @@ async fn recall( phase_start = Instant::now(); let mut ordered_reconstruction_applied_count = 0usize; - if should_run_ordered_reconstruction(req.ordered_reconstruction, query_intent.as_ref()) { + if should_run_ordered_reconstruction(req.ordered_reconstruction, query_plan.as_ref()) { let ordered_results = ordered_reconstruction_results( &ctx, &expanded_cues, @@ -5041,7 +5295,7 @@ async fn recall( phase_start = Instant::now(); let mut evidence_coverage_applied_count = 0usize; - if should_run_evidence_coverage(req.evidence_coverage, query_intent.as_ref()) { + if should_run_evidence_coverage(req.evidence_coverage, query_plan.as_ref()) { let evidence_results = evidence_coverage_results( &ctx, &expanded_cues, @@ -5072,7 +5326,7 @@ async fn recall( if should_run_parent_fusion( &all_results, req.parent_fusion, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ) { let fusion_limit = req @@ -5114,21 +5368,21 @@ async fn recall( } phase_start = Instant::now(); - apply_source_role_preference(&mut all_results, query_intent.as_ref()); - apply_source_answer_adjacency_preference(&mut all_results, query_intent.as_ref()); + apply_source_role_preference(&mut all_results, query_plan.as_ref()); + apply_source_answer_adjacency_preference(&mut all_results, query_plan.as_ref()); apply_user_context_adjacency_preference( &mut all_results, - query_intent.as_ref(), + query_plan.as_ref(), req.query_text.as_deref(), ); - apply_decision_adjacency_preference(&mut all_results, query_intent.as_ref()); + apply_decision_adjacency_preference(&mut all_results, query_plan.as_ref()); let slate_rerank_applied_count = apply_slate_rerank( &ctx, &mut all_results, &expanded_cues, req.ordered_reconstruction, req.evidence_coverage, - query_intent.as_ref(), + query_plan.as_ref(), limit, ); all_results.sort_by(|a, b| { @@ -5227,8 +5481,7 @@ async fn recall( "explain": { "query_cues": cues_to_process, "expanded_cues": expanded_cues, - "query_intent": query_intent, - "cuepacks": req.cuepacks.clone().unwrap_or_else(|| vec!["default".to_string()]), + "query_plan": query_plan, "source_answer_projection_cues": source_answer_projection_expansions, "source_prompt_projection_cues": source_prompt_projection_expansions, "user_context_projection_cues": user_context_projection_expansions, @@ -5481,23 +5734,41 @@ async fn jobs_status( headers: HeaderMap, ) -> (StatusCode, Json) { let project_id_opt = extract_project_id_optional(&headers); - let EngineState { job_queue, .. } = state; + let EngineState { mt_engine, job_queue, .. } = state; if let Some(project_id) = project_id_opt { - if let Some(session) = job_queue.get_session(&project_id) { - let progress = session.get_progress(); - (StatusCode::OK, Json(serde_json::json!(progress))) - } else { - // No active session - return idle status - ( - StatusCode::OK, - Json(serde_json::json!({ - "phase": "idle", + let mut status = if let Some(session) = job_queue.get_session(&project_id) { + serde_json::to_value(session.get_progress()).unwrap_or_else(|_| { + serde_json::json!({ + "phase": "processing", "writes_completed": 0, - "writes_total": 0 - })), - ) + "writes_total": 0, + "intent_completed": 0, + "intent_total": 0, + "intent_failed": 0 + }) + }) + } else { + serde_json::json!({ + "phase": "idle", + "writes_completed": 0, + "writes_total": 0, + "intent_completed": 0, + "intent_total": 0, + "intent_failed": 0 + }) + }; + + match mt_engine.get_or_create_project(project_id) { + Ok(ctx) => add_intent_coverage_to_status(&mut status, &ctx), + Err(error) => { + if let Some(object) = status.as_object_mut() { + object.insert("intent_ready".to_string(), serde_json::json!(false)); + object.insert("intent_error".to_string(), serde_json::json!(error)); + } + } } + (StatusCode::OK, Json(status)) } else { // Global progress let progress = job_queue.get_global_progress(); @@ -5505,6 +5776,43 @@ async fn jobs_status( } } +fn add_intent_coverage_to_status(status: &mut serde_json::Value, ctx: &crate::projects::ProjectContext) { + let (memory_total, annotated, missing, stale) = ctx.main.intent_coverage(); + let job_intent_total = status + .get("intent_total") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + let job_intent_completed = status + .get("intent_completed") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + let job_intent_failed = status + .get("intent_failed") + .and_then(serde_json::Value::as_u64) + .unwrap_or(0); + let jobs_ready = job_intent_total == 0 + || (job_intent_completed >= job_intent_total && job_intent_failed == 0); + let coverage_ready = missing == 0 && annotated >= memory_total; + if let Some(object) = status.as_object_mut() { + object.insert("intent_memory_total".to_string(), serde_json::json!(memory_total)); + object.insert("intent_annotated".to_string(), serde_json::json!(annotated)); + object.insert("intent_missing".to_string(), serde_json::json!(missing)); + object.insert("intent_stale".to_string(), serde_json::json!(stale)); + object.insert( + "intent_ready".to_string(), + serde_json::json!(coverage_ready && jobs_ready), + ); + object.insert( + "intent_model_version".to_string(), + serde_json::json!(ctx.main.config.semantic.model_version), + ); + object.insert( + "intent_taxonomy_version".to_string(), + serde_json::json!(crate::intent::INTENT_TAXONOMY_VERSION), + ); + } +} + async fn recall_grounded( State(state): State, headers: HeaderMap, @@ -5530,7 +5838,6 @@ async fn recall_grounded( let EngineState { ref mt_engine, - ref cuepack_registry, .. } = &state; let start = Instant::now(); @@ -5559,10 +5866,8 @@ async fn recall_grounded( // tokens were computed in step 1, reuse them! ctx.expand_query_cues(normalized_cues, &tokens) }; - let _query_intent = apply_query_intent( + let _query_plan = apply_query_plan( &ctx, - cuepack_registry, - req.cuepacks.as_deref(), Some(&req.query_text), None, &mut expanded_cues, @@ -5818,6 +6123,123 @@ async fn export_project( ) } +fn normalize_included_paths(paths: Option>) -> Result, String> { + let mut normalized = Vec::new(); + for raw_path in paths.unwrap_or_default() { + let candidate = raw_path.trim().replace('\\', "/"); + let candidate = candidate.trim_matches('/'); + if candidate.is_empty() || candidate == "." { + return Ok(Vec::new()); + } + + let mut components = Vec::new(); + for component in std::path::Path::new(candidate).components() { + match component { + std::path::Component::Normal(value) => { + components.push(value.to_string_lossy().to_string()) + } + std::path::Component::CurDir => {} + _ => { + return Err(format!( + "Included path '{}' must stay within the watch directory", + raw_path + )) + } + } + } + if !components.is_empty() { + normalized.push(components.join("/")); + } + } + normalized.sort(); + normalized.dedup(); + Ok(normalized) +} + +fn normalize_ignored_extensions(extensions: Option>) -> Vec { + let mut normalized: Vec = extensions + .unwrap_or_default() + .into_iter() + .map(|extension| extension.trim().trim_start_matches('.').to_lowercase()) + .filter(|extension| !extension.is_empty()) + .collect(); + normalized.sort(); + normalized.dedup(); + normalized +} + +async fn preview_directory( + State(state): State, + Json(req): Json, +) -> (StatusCode, Json) { + let watch_dir = match std::fs::canonicalize(&req.watch_dir) { + Ok(path) if path.is_dir() => path.to_string_lossy().to_string(), + _ => { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ + "error": format!("Directory '{}' does not exist", req.watch_dir) + })), + ) + } + }; + let included_paths = match normalize_included_paths(req.included_paths) { + Ok(paths) => paths, + Err(error) => { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({"error": error})), + ) + } + }; + let config = crate::agent::AgentConfig { + project_id: "directory-preview".to_string(), + watch_dir, + throttle_ms: 0, + state_file: None, + included_paths, + ignored_patterns: req.ignored_patterns.unwrap_or_default(), + ignored_extensions: normalize_ignored_extensions(req.ignored_extensions), + }; + let ingester = crate::agent::ingester::Ingester::new(config, state.job_queue); + + match ingester.preview_scope() { + Ok(preview) => ( + StatusCode::OK, + Json(serde_json::to_value(preview).unwrap_or_else(|error| { + serde_json::json!({"error": format!("Failed to serialize preview: {}", error)}) + })), + ), + Err(error) => ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({"error": error})), + ), + } +} + +async fn get_project_watch_dir( + State(state): State, + Path(project_id): Path, +) -> (StatusCode, Json) { + match state.mt_engine.load_project_meta(&project_id) { + Ok(meta) => ( + StatusCode::OK, + Json(serde_json::json!({ + "project_id": project_id, + "initialized": meta.agent_enabled && meta.watch_dir.is_some(), + "watch_dir": meta.watch_dir, + "included_paths": meta.included_paths, + "ignored_patterns": meta.ignored_patterns, + "ignored_extensions": meta.ignored_extensions, + })), + ), + Err(error) => ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({"error": error})), + ), + } +} + async fn set_project_watch_dir( State(state): State, Path(project_id): Path, @@ -5827,6 +6249,7 @@ async fn set_project_watch_dir( mt_engine, read_only, agent_manager, + data_dir, .. } = state; @@ -5839,19 +6262,50 @@ async fn set_project_watch_dir( ); } - match mt_engine.set_project_watch_dir(&project_id, Some(req.watch_dir.clone())) { + let watch_dir = match std::fs::canonicalize(&req.watch_dir) { + Ok(path) if path.is_dir() => path.to_string_lossy().to_string(), + _ => { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ + "error": format!("Directory '{}' does not exist", req.watch_dir) + })), + ) + } + }; + let included_paths = match normalize_included_paths(req.included_paths) { + Ok(paths) => paths, + Err(error) => { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({"error": error})), + ) + } + }; + let ignored_patterns = req.ignored_patterns.unwrap_or_default(); + let ignored_extensions = normalize_ignored_extensions(req.ignored_extensions); + + match mt_engine.set_project_watch_config( + &project_id, + watch_dir.clone(), + included_paths.clone(), + ignored_patterns.clone(), + ignored_extensions.clone(), + ) { Ok(_) => { // Immediately start/update the agent let agent_config = crate::agent::AgentConfig { project_id: project_id.clone(), - watch_dir: req.watch_dir.clone(), + watch_dir: watch_dir.clone(), throttle_ms: 100, // Small throttle to prevent CPU pinning - state_file: Some(std::path::PathBuf::from(format!( - "./snapshots/{}_agent_state.json", - project_id - ))), - ignored_patterns: req.ignored_patterns.unwrap_or_default(), - ignored_extensions: req.ignored_extensions.unwrap_or_default(), + state_file: Some( + std::path::PathBuf::from(data_dir) + .join("snapshots") + .join(format!("{}_agent_state.json", project_id)), + ), + included_paths: included_paths.clone(), + ignored_patterns: ignored_patterns.clone(), + ignored_extensions: ignored_extensions.clone(), }; // Spawn the starting of the agent securely @@ -5866,7 +6320,11 @@ async fn set_project_watch_dir( StatusCode::OK, Json(serde_json::json!({ "status": "updated", - "project_id": project_id + "project_id": project_id, + "watch_dir": watch_dir, + "included_paths": included_paths, + "ignored_patterns": ignored_patterns, + "ignored_extensions": ignored_extensions, })), ) } @@ -6408,6 +6866,7 @@ async fn recall_web( watch_dir: String::new(), throttle_ms: 0, state_file: None, + included_paths: Vec::new(), ignored_patterns: Vec::new(), ignored_extensions: Vec::new(), }; @@ -6552,6 +7011,7 @@ async fn recall_web( watch_dir: String::new(), throttle_ms: 0, state_file: None, + included_paths: Vec::new(), ignored_patterns: Vec::new(), ignored_extensions: Vec::new(), }; @@ -6639,6 +7099,7 @@ async fn ingest_url( watch_dir: String::new(), // Not used for API-driven ingestion throttle_ms: 0, state_file: None, + included_paths: Vec::new(), ignored_patterns: Vec::new(), ignored_extensions: Vec::new(), }; @@ -6708,6 +7169,11 @@ pub struct IngestContentRequest { pub metadata: Option>, #[serde(default)] pub structural_cues: Vec, + /// Optional one-vector-per-produced-chunk embeddings. Supplying one + /// vector for the whole document is intentionally not supported because + /// it would make every chunk share the same semantic representation. + #[serde(default)] + pub embeddings: Option>>, #[serde(default)] pub segmenter: TextSegmenterMode, #[serde(default)] @@ -6732,8 +7198,6 @@ pub struct DebugAnalyzeTextRequest { #[serde(default)] pub available_cues: Vec, #[serde(default)] - pub cuepacks: Option>, - #[serde(default)] pub filename: Option, #[serde(default)] pub segmenter: TextSegmenterMode, @@ -6831,30 +7295,21 @@ async fn debug_analyze_text( }) .collect(); - let cuepack_selection = req.cuepacks.as_deref(); let core_facets = crate::facets::extract_memory_facets_core( &req.text, req.metadata.as_ref(), &req.existing_cues, ); - let memory_facets = crate::facets::extract_memory_facets_with_cuepacks( - &req.text, - req.metadata.as_ref(), - &req.existing_cues, - &state.cuepack_registry, - cuepack_selection, - ); + let memory_facets = core_facets.clone(); let available_cues: HashSet = req .available_cues .iter() .map(|cue| cue.to_lowercase()) .collect(); - let query_intent = crate::facets::compile_query_intent_with_cuepacks( + let query_plan = crate::facets::compile_query_plan_with_reference_time( &req.text, req.query_time.as_deref(), |cue| ctx.main.get_cue_frequency(cue) > 0 || available_cues.contains(&cue.to_lowercase()), - &state.cuepack_registry, - cuepack_selection, ); let segmenter_config = segmenter_config_from_debug_request(&req); @@ -6891,7 +7346,7 @@ async fn debug_analyze_text( "normalized_cues": normalized_cues, "core_facets": core_facets, "memory_facets": memory_facets, - "query_intent": query_intent, + "query_plan": query_plan, "segmenter": req.segmenter, "filename": req.filename.unwrap_or_else(|| "content.txt".to_string()), "chunks": chunk_summaries, @@ -6970,19 +7425,41 @@ async fn ingest_content( ); } + if let Some(embeddings) = req.embeddings.as_ref() { + if embeddings.len() != chunks.len() { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ + "error": format!( + "embeddings length ({}) must match produced chunk count ({})", + embeddings.len(), + chunks.len() + ) + })), + ); + } + } + // Create an ingester for this request let config = AgentConfig { project_id: project_id.clone(), watch_dir: String::new(), throttle_ms: 0, state_file: None, + included_paths: Vec::new(), ignored_patterns: Vec::new(), ignored_extensions: Vec::new(), }; let mut ingester = Ingester::new(config, job_queue); match ingester - .process_chunks_with_metadata(chunks, &project_id, &source, req.metadata) + .process_chunks_with_metadata_and_embeddings( + chunks, + &project_id, + &source, + req.metadata, + req.embeddings, + ) .await { Ok(memory_ids) => ( @@ -7133,12 +7610,11 @@ async fn ingest_file( file_path: source.clone(), structural_cues: chunk.structural_cues.clone(), metadata: None, + embedding: None, category: chunk.category, }) .await; - session.write_complete(); - source_keys.push(source_key); } @@ -7554,1519 +8030,5 @@ async fn backup_delete( } #[cfg(test)] -mod tests { - use super::*; - use crate::config::TuningConfig; - use crate::normalization::NormalizationConfig; - use crate::projects::ProjectContext; - use crate::structures::MainStats; - use crate::taxonomy::Taxonomy; - use std::collections::HashMap; - use std::sync::Arc; - - fn recall_result( - match_integrity: f64, - intersection_count: usize, - ) -> crate::engine::RecallResult { - crate::engine::RecallResult { - memory_id: 1, - content: "content".to_string(), - score: 1.0, - match_integrity, - intersection_count, - recency_score: 0.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata: HashMap::new(), - explain: None, - } - } - - #[test] - fn parent_fusion_defaults_off_and_force_runs() { - let results = vec![recall_result(0.95, 4)]; - - assert!(!should_run_parent_fusion( - &results, - ParentFusionMode::Off, - None, - Some("summarize the key points"), - )); - assert!(should_run_parent_fusion( - &results, - ParentFusionMode::Force, - None, - Some("plain lookup"), - )); - } - - #[test] - fn parent_fusion_auto_requires_synthesis_query() { - assert!(!should_run_parent_fusion( - &[recall_result(0.4, 1)], - ParentFusionMode::Auto, - None, - Some("what is my favorite dessert"), - )); - assert!(should_run_parent_fusion( - &[recall_result(0.4, 1)], - ParentFusionMode::Auto, - None, - Some("summarize my language service progress in order"), - )); - } - - #[test] - fn ordered_reconstruction_is_opt_in_and_intent_gated() { - let mut intent = crate::facets::QueryIntent::default(); - assert!(!should_run_ordered_reconstruction( - OrderedReconstructionMode::Off, - Some(&intent) - )); - assert!(should_run_ordered_reconstruction( - OrderedReconstructionMode::Force, - None - )); - assert!(!should_run_ordered_reconstruction( - OrderedReconstructionMode::Auto, - Some(&intent) - )); - - intent.labels.push("ordered_reconstruction".to_string()); - assert!(should_run_ordered_reconstruction( - OrderedReconstructionMode::Auto, - Some(&intent) - )); - - intent.labels.clear(); - intent - .labels - .push("multi_evidence_collection".to_string()); - assert!(should_run_ordered_reconstruction( - OrderedReconstructionMode::Auto, - Some(&intent) - )); - } - - #[test] - fn evidence_coverage_is_opt_in_and_intent_gated() { - let mut intent = crate::facets::QueryIntent::default(); - assert!(!should_run_evidence_coverage( - EvidenceCoverageMode::Off, - Some(&intent) - )); - assert!(should_run_evidence_coverage( - EvidenceCoverageMode::Force, - None - )); - assert!(!should_run_evidence_coverage( - EvidenceCoverageMode::Auto, - Some(&intent) - )); - - intent.labels.push("multi_evidence_summary".to_string()); - assert!(should_run_evidence_coverage( - EvidenceCoverageMode::Auto, - Some(&intent) - )); - - intent.labels.clear(); - intent - .labels - .push("multi_evidence_collection".to_string()); - assert!(should_run_evidence_coverage( - EvidenceCoverageMode::Auto, - Some(&intent) - )); - - intent.labels.clear(); - intent.labels.push("ordered_reconstruction".to_string()); - assert!(should_run_evidence_coverage( - EvidenceCoverageMode::Auto, - Some(&intent) - )); - } - - #[test] - fn evidence_coverage_selects_diverse_session_evidence() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "evidence_coverage_test".to_string(), - ); - - let add_turn = |session: &str, - order: i64, - plan: Option, - content: &str, - cues: &[&str]| - -> MemoryId { - let mut metadata = HashMap::new(); - metadata.insert("source_session_id".to_string(), serde_json::json!(session)); - metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); - if let Some(plan) = plan { - metadata.insert("source_plan_idx".to_string(), serde_json::json!(plan)); - } - ctx.main.add_memory( - content.to_string(), - cues.iter().map(|cue| cue.to_string()).collect(), - Some(metadata), - MainStats::default(), - false, - ) - }; - - let integration = add_turn( - "thread-a", - 1, - Some(1), - "We designed language service integration.", - &["source_role:assistant", "type:answer", "has:list", "language", "service", "integration", "architecture"], - ); - let deployment = add_turn( - "thread-a", - 2, - Some(2), - "We planned deployment and release steps.", - &["source_role:assistant", "type:answer", "deployment", "release", "service"], - ); - let performance = add_turn( - "thread-a", - 3, - Some(3), - "We improved performance and latency.", - &["source_role:assistant", "type:answer", "performance", "latency", "service"], - ); - let unrelated = add_turn( - "thread-a", - 4, - Some(4), - "We discussed a lunch menu.", - &["source_role:assistant", "type:answer", "lunch", "menu"], - ); - let distractor = add_turn( - "thread-b", - 1, - Some(2), - "A different deployment discussion happened elsewhere.", - &["source_role:assistant", "type:answer", "deployment", "service"], - ); - - let pivot = crate::engine::RecallResult { - memory_id: deployment, - content: "We planned deployment and release steps.".to_string(), - score: 140.0, - match_integrity: 0.6, - intersection_count: 2, - recency_score: 1.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata: HashMap::new(), - explain: None, - }; - let pivot_score = pivot.score; - - let evidence = evidence_coverage_results( - &ctx, - &[ - ("language".to_string(), 1.0), - ("service".to_string(), 0.8), - ("integration".to_string(), 1.0), - ("deployment".to_string(), 1.0), - ("performance".to_string(), 1.0), - ], - &[pivot], - 10, - 100, - 1, - true, - ); - - let ids: Vec = evidence.iter().map(|result| result.memory_id).collect(); - assert!(ids.contains(&integration)); - assert!(ids.contains(&deployment)); - assert!(ids.contains(&performance)); - assert!(!ids.contains(&unrelated)); - assert!(!ids.contains(&distractor)); - assert!(evidence - .iter() - .all(|result| result.metadata.contains_key("evidence_coverage"))); - assert!(evidence.iter().any(|result| result - .metadata - .contains_key("evidence_coverage_source_plan"))); - assert!(evidence - .iter() - .all(|result| result.score < pivot_score)); - } - - #[test] - fn slate_rerank_is_mode_and_intent_gated() { - let mut intent = crate::facets::QueryIntent::default(); - intent.labels.push("multi_evidence_summary".to_string()); - - assert!(!slate_rerank_requested( - OrderedReconstructionMode::Off, - EvidenceCoverageMode::Off, - Some(&intent) - )); - assert!(slate_rerank_requested( - OrderedReconstructionMode::Auto, - EvidenceCoverageMode::Off, - Some(&intent) - )); - - let plain_intent = crate::facets::QueryIntent::default(); - assert!(!slate_rerank_requested( - OrderedReconstructionMode::Auto, - EvidenceCoverageMode::Off, - Some(&plain_intent) - )); - } - - #[test] - fn slate_rerank_promotes_coverage_candidates_below_protected_top() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "slate_rerank_test".to_string(), - ); - - let add_turn = |session: &str, - order: i64, - role: &str, - content: &str, - cues: &[&str]| - -> MemoryId { - let mut metadata = HashMap::new(); - metadata.insert("source_session_id".to_string(), serde_json::json!(session)); - metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); - metadata.insert("source_role".to_string(), serde_json::json!(role)); - ctx.main.add_memory( - content.to_string(), - cues.iter().map(|cue| cue.to_string()).collect(), - Some(metadata), - MainStats::default(), - false, - ) - }; - let make_result = |memory_id: MemoryId, - score: f64, - metadata: HashMap| - -> crate::engine::RecallResult { - crate::engine::RecallResult { - memory_id, - content: format!("memory {memory_id}"), - score, - match_integrity: 0.2, - intersection_count: 1, - recency_score: 0.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata, - explain: None, - } - }; - - let protected_a = add_turn( - "thread-a", - 1, - "assistant", - "Protected top result A.", - &["overview"], - ); - let protected_b = add_turn( - "thread-a", - 2, - "assistant", - "Protected top result B.", - &["overview"], - ); - let protected_c = add_turn( - "thread-a", - 3, - "assistant", - "Protected top result C.", - &["overview"], - ); - let mut results = vec![ - make_result(protected_a, 300.0, HashMap::new()), - make_result(protected_b, 290.0, HashMap::new()), - make_result(protected_c, 280.0, HashMap::new()), - ]; - - for rank in 0..25 { - let id = add_turn( - "thread-b", - rank, - "assistant", - "Generic distractor.", - &["generic", "discussion"], - ); - results.push(make_result(id, 270.0 - rank as f64, HashMap::new())); - } - - let relevant_late = add_turn( - "thread-a", - 24, - "assistant", - "We covered deployment and latency.", - &["deployment", "latency", "service", "type:answer"], - ); - let relevant_later = add_turn( - "thread-a", - 40, - "assistant", - "We also covered integration architecture.", - &["integration", "architecture", "service", "type:answer"], - ); - let mut evidence_metadata = HashMap::new(); - evidence_metadata.insert("evidence_coverage".to_string(), serde_json::json!(true)); - results.push(make_result(relevant_late, 150.0, evidence_metadata.clone())); - results.push(make_result(relevant_later, 149.0, evidence_metadata)); - - let mut intent = crate::facets::QueryIntent::default(); - intent.labels.push("multi_evidence_summary".to_string()); - let moved = apply_slate_rerank( - &ctx, - &mut results, - &[ - ("deployment".to_string(), 1.0), - ("latency".to_string(), 1.0), - ("integration".to_string(), 1.0), - ("architecture".to_string(), 1.0), - ("service".to_string(), 0.8), - ], - OrderedReconstructionMode::Auto, - EvidenceCoverageMode::Off, - Some(&intent), - 100, - ); - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let ids: Vec = results.iter().map(|result| result.memory_id).collect(); - assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); - assert!(ids.iter().position(|id| *id == relevant_late).unwrap() < 20); - assert!(ids.iter().position(|id| *id == relevant_later).unwrap() < 20); - assert!(moved >= 2); - assert!(results - .iter() - .any(|result| result.memory_id == relevant_late - && result.metadata.contains_key("slate_rerank"))); - } - - #[test] - fn slate_rerank_promotes_strong_summary_candidates_without_helper_metadata() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "slate_rerank_summary_signal_test".to_string(), - ); - - let add_turn = |session: &str, - order: i64, - content: &str, - cues: &[&str]| - -> MemoryId { - let mut metadata = HashMap::new(); - metadata.insert("source_session_id".to_string(), serde_json::json!(session)); - metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); - ctx.main.add_memory( - content.to_string(), - cues.iter().map(|cue| cue.to_string()).collect(), - Some(metadata), - MainStats::default(), - false, - ) - }; - let make_result = |memory_id: MemoryId, - score: f64| - -> crate::engine::RecallResult { - crate::engine::RecallResult { - memory_id, - content: format!("memory {memory_id}"), - score, - match_integrity: 0.2, - intersection_count: 1, - recency_score: 0.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata: HashMap::new(), - explain: None, - } - }; - - let protected_a = add_turn("thread-a", 1, "Protected A.", &["overview"]); - let protected_b = add_turn("thread-a", 2, "Protected B.", &["overview"]); - let protected_c = add_turn("thread-a", 3, "Protected C.", &["overview"]); - let mut results = vec![ - make_result(protected_a, 300.0), - make_result(protected_b, 290.0), - make_result(protected_c, 280.0), - ]; - - for rank in 0..25 { - let id = add_turn( - "thread-b", - rank, - "Generic project discussion.", - &["generic", "project"], - ); - results.push(make_result(id, 270.0 - rank as f64)); - } - - let relevant = add_turn( - "thread-c", - 8, - "City autocomplete in the weather app uses a debounced API lookup.", - &["city", "autocomplete", "weather", "app", "lookup"], - ); - results.push(make_result(relevant, 150.0)); - - let mut intent = crate::facets::QueryIntent::default(); - intent.labels.push("multi_evidence_summary".to_string()); - let moved = apply_slate_rerank( - &ctx, - &mut results, - &[ - ("city".to_string(), 1.0), - ("autocomplete".to_string(), 1.0), - ("weather".to_string(), 1.0), - ("app".to_string(), 0.8), - ("implementation".to_string(), 0.8), - ], - OrderedReconstructionMode::Auto, - EvidenceCoverageMode::Off, - Some(&intent), - 100, - ); - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let ids: Vec = results.iter().map(|result| result.memory_id).collect(); - assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); - assert!(ids.iter().position(|id| *id == relevant).unwrap() < 20); - assert!(moved >= 1); - assert!(results - .iter() - .any(|result| result.memory_id == relevant - && result.metadata.contains_key("slate_rerank"))); - } - - #[test] - fn slate_rerank_promotes_standing_instruction_for_instruction_query() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "slate_rerank_instruction_test".to_string(), - ); - - let add_turn = |content: &str, cues: &[&str]| -> MemoryId { - ctx.main.add_memory( - content.to_string(), - cues.iter().map(|cue| cue.to_string()).collect(), - None, - MainStats::default(), - false, - ) - }; - let make_result = |memory_id: MemoryId, - score: f64| - -> crate::engine::RecallResult { - crate::engine::RecallResult { - memory_id, - content: format!("memory {memory_id}"), - score, - match_integrity: 0.1, - intersection_count: 1, - recency_score: 0.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata: HashMap::new(), - explain: None, - } - }; - - let protected_a = add_turn("Protected A", &["layout"]); - let protected_b = add_turn("Protected B", &["layout"]); - let protected_c = add_turn("Protected C", &["layout"]); - let mut results = vec![ - make_result(protected_a, 300.0), - make_result(protected_b, 290.0), - make_result(protected_c, 280.0), - ]; - for rank in 0..45 { - let id = add_turn("Generic layout discussion", &["layout", "project"]); - results.push(make_result(id, 270.0 - rank as f64)); - } - - let instruction = add_turn( - "Always include semantic HTML5 tag usage details when I ask about markup structure.", - &[ - "type:standing_instruction", - "instruction_trigger:markup", - "semantic", - "html5", - "tag", - "structure", - ], - ); - results.push(make_result(instruction, 120.0)); - - let mut intent = crate::facets::QueryIntent::default(); - intent.labels.push("instruction_applicable".to_string()); - let moved = apply_slate_rerank( - &ctx, - &mut results, - &[ - ("blog".to_string(), 1.0), - ("layout".to_string(), 1.0), - ("header".to_string(), 1.0), - ("navigation".to_string(), 1.0), - ("footer".to_string(), 1.0), - ], - OrderedReconstructionMode::Auto, - EvidenceCoverageMode::Off, - Some(&intent), - 100, - ); - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let ids: Vec = results.iter().map(|result| result.memory_id).collect(); - assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); - assert!(ids.iter().position(|id| *id == instruction).unwrap() < 20); - assert!(moved >= 1); - assert!(results - .iter() - .any(|result| result.memory_id == instruction - && result.metadata.contains_key("slate_rerank"))); - } - - #[test] - fn slate_rerank_orders_selected_ordered_candidates_after_selection() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "slate_rerank_ordered_test".to_string(), - ); - - let add_turn = |session: &str, order: i64, content: &str, cues: &[&str]| -> MemoryId { - let mut metadata = HashMap::new(); - metadata.insert("source_session_id".to_string(), serde_json::json!(session)); - metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); - ctx.main.add_memory( - content.to_string(), - cues.iter().map(|cue| cue.to_string()).collect(), - Some(metadata), - MainStats::default(), - false, - ) - }; - let make_result = |memory_id: MemoryId, - score: f64, - ordered: bool| - -> crate::engine::RecallResult { - let mut metadata = HashMap::new(); - if ordered { - metadata.insert("ordered_reconstruction".to_string(), serde_json::json!(true)); - } - crate::engine::RecallResult { - memory_id, - content: format!("memory {memory_id}"), - score, - match_integrity: 0.3, - intersection_count: 1, - recency_score: 0.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata, - explain: None, - } - }; - - let protected_a = add_turn("thread-a", 1, "Protected A", &["bootstrap"]); - let protected_b = add_turn("thread-a", 2, "Protected B", &["bootstrap"]); - let protected_c = add_turn("thread-a", 3, "Protected C", &["bootstrap"]); - let mut results = vec![ - make_result(protected_a, 300.0, false), - make_result(protected_b, 290.0, false), - make_result(protected_c, 280.0, false), - ]; - for rank in 0..25 { - let id = add_turn("thread-b", rank, "Generic project discussion", &["project"]); - results.push(make_result(id, 270.0 - rank as f64, false)); - } - - let first = add_turn("thread-a", 5, "Bootstrap CDN setup", &["bootstrap", "cdn"]); - let second = add_turn("thread-a", 7, "Bootstrap form classes", &["bootstrap", "form"]); - let third = add_turn("thread-a", 11, "Bootstrap modal upgrade", &["bootstrap", "modal"]); - results.push(make_result(third, 151.0, true)); - results.push(make_result(first, 150.0, true)); - results.push(make_result(second, 149.0, true)); - - let mut intent = crate::facets::QueryIntent::default(); - intent.labels.push("ordered_reconstruction".to_string()); - let moved = apply_slate_rerank( - &ctx, - &mut results, - &[ - ("bootstrap".to_string(), 1.0), - ("cdn".to_string(), 1.0), - ("form".to_string(), 1.0), - ("modal".to_string(), 1.0), - ], - OrderedReconstructionMode::Auto, - EvidenceCoverageMode::Off, - Some(&intent), - 100, - ); - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let ids: Vec = results.iter().map(|result| result.memory_id).collect(); - assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); - let first_pos = ids.iter().position(|id| *id == first).unwrap(); - let second_pos = ids.iter().position(|id| *id == second).unwrap(); - let third_pos = ids.iter().position(|id| *id == third).unwrap(); - assert!(first_pos < 20); - assert!(second_pos < 20); - assert!(third_pos < 20); - assert!(first_pos < second_pos); - assert!(second_pos < third_pos); - assert!(moved >= 3); - } - - #[test] - fn ordered_reconstruction_scans_selected_session_in_order() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "ordered_test".to_string(), - ); - - let add_turn = |session: &str, order: i64, content: &str, cues: &[&str]| -> MemoryId { - let mut metadata = HashMap::new(); - metadata.insert("source_session_id".to_string(), serde_json::json!(session)); - metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); - ctx.main.add_memory( - content.to_string(), - cues.iter().map(|cue| cue.to_string()).collect(), - Some(metadata), - MainStats::default(), - false, - ) - }; - - let first = add_turn( - "thread-a", - 1, - "We integrated the language detection service.", - &["language", "service", "integrate"], - ); - let second = add_turn( - "thread-a", - 2, - "Then we optimized translation service latency.", - &["translation", "service", "optimize"], - ); - let distractor = add_turn( - "thread-b", - 1, - "A different service discussion happened elsewhere.", - &["translation", "service", "discussion"], - ); - - let mut pivot_metadata = HashMap::new(); - pivot_metadata.insert("source_session_id".to_string(), serde_json::json!("thread-a")); - pivot_metadata.insert("source_turn_index".to_string(), serde_json::json!(2)); - let pivot = crate::engine::RecallResult { - memory_id: second, - content: "Then we optimized translation service latency.".to_string(), - score: 120.0, - match_integrity: 0.6, - intersection_count: 2, - recency_score: 1.0, - reinforcement_score: 0.0, - salience_score: 0.0, - created_at: 0.0, - metadata: pivot_metadata, - explain: None, - }; - - let ordered = ordered_reconstruction_results( - &ctx, - &[ - ("language".to_string(), 1.0), - ("translation".to_string(), 1.0), - ("service".to_string(), 1.0), - ("optimize".to_string(), 1.0), - ], - &[pivot], - 10, - 100, - 1, - true, - ); - - let ids: Vec = ordered.iter().map(|result| result.memory_id).collect(); - assert!(ids.contains(&first)); - assert!(ids.contains(&second)); - assert!(!ids.contains(&distractor)); - assert!(ordered - .iter() - .all(|result| result.metadata.contains_key("ordered_reconstruction"))); - } - - #[test] - fn segment_link_requires_parent_and_chunk_idx() { - assert_eq!( - segment_link_from_cues(&[ - "parent:abc".to_string(), - "chunk_idx:7".to_string(), - "source_role:user".to_string(), - ]), - Some(("parent:abc".to_string(), 7)) - ); - assert_eq!( - segment_link_from_cues(&["parent:abc".to_string()]), - None - ); - } - - #[test] - fn stitched_chunk_join_removes_overlapped_sentences() { - let joined = join_stitched_chunk_contents(&[ - "First sentence. Shared sentence.".to_string(), - "Shared sentence. Final sentence.".to_string(), - ]); - - assert_eq!(joined, "First sentence. Shared sentence. Final sentence."); - } - - #[test] - fn source_answer_projection_requires_assistant_answer_language() { - let source_answer_intent = crate::facets::QueryIntent { - labels: vec!["source_answer".to_string()], - ..Default::default() - }; - assert!(!source_answer_projection_requested( - Some(&source_answer_intent), - Some("What did I buy last week?") - )); - assert!(source_answer_projection_requested( - Some(&source_answer_intent), - Some("What was in the assistant answer?") - )); - - let assistant_intent = crate::facets::QueryIntent { - labels: vec!["source_assistant".to_string()], - ..Default::default() - }; - assert!(source_answer_projection_requested( - Some(&assistant_intent), - Some("Can you remind me?") - )); - } - - #[test] - fn user_context_projection_targets_advice_without_source_intents() { - assert!(user_context_projection_requested( - None, - Some("I've been having trouble with battery life. Any tips?") - )); - - let recommendation_intent = crate::facets::QueryIntent { - labels: vec!["recommendation".to_string()], - ..Default::default() - }; - assert!(user_context_projection_requested( - Some(&recommendation_intent), - Some("Can you recommend something for me?") - )); - assert!(!user_context_projection_requested( - Some(&recommendation_intent), - Some("Can you suggest a hotel for my upcoming trip to Miami?") - )); - - for label in ["source_answer", "source_assistant", "source_user", "decision_selection"] { - let intent = crate::facets::QueryIntent { - labels: vec![label.to_string()], - ..Default::default() - }; - assert!( - !user_context_projection_requested(Some(&intent), Some("Any tips?")), - "source-specific query should not request user context projection for {label}" - ); - } - } - - #[test] - fn user_context_projection_anchors_require_specific_context() { - let phone_accessory_anchors = - projection_anchor_cues(Some("Can you suggest some useful accessories for my phone?")); - assert_eq!( - phone_accessory_anchors, - vec!["accessory".to_string(), "phone".to_string()] - ); - - let media_recommendation_anchors = - projection_anchor_cues(Some("Can you recommend a show or movie for me to watch tonight?")); - assert!(media_recommendation_anchors.is_empty()); - - let troubleshooting_anchors = projection_anchor_cues(Some( - "I've been having trouble with the battery life on my phone lately. Any tips?", - )); - assert!(troubleshooting_anchors.contains(&"battery".to_string())); - assert!(troubleshooting_anchors.contains(&"life".to_string())); - assert!(troubleshooting_anchors.contains(&"phone".to_string())); - - let navigation_anchors = projection_anchor_cues(Some( - "I'm a bit anxious about getting around Tokyo. Do you have any helpful tips?", - )); - assert_eq!( - navigation_anchors, - vec!["anxious".to_string(), "tokyo".to_string()] - ); - - let relevant = "assistant: A power bank can help with phone battery life while traveling."; - let incidental = "assistant: You could schedule a phone call during the morning."; - - assert!(projection_anchor_match_count(relevant, &troubleshooting_anchors) >= 2); - assert!(projection_anchor_match_count(incidental, &troubleshooting_anchors) < 2); - assert!(!projection_pivot_matches_context( - "assistant: A camera bag can complement your Sony setup.", - 4, - &phone_accessory_anchors, - true - )); - assert!(!projection_pivot_matches_context( - "assistant: A camera bag can complement your Sony setup.", - 3, - &phone_accessory_anchors, - true - )); - assert!(projection_pivot_matches_context( - "assistant: A phone case is a useful accessory for your phone setup.", - 2, - &phone_accessory_anchors, - true - )); - assert!(!projection_pivot_matches_context( - "assistant: A camera bag can complement your Sony setup.", - 4, - &phone_accessory_anchors, - false - )); - - let vague_interest_intent = crate::facets::QueryIntent { - labels: vec!["vague_interest_recommendation".to_string()], - ..Default::default() - }; - assert!(suppress_user_context_projection_for_intent(Some( - &vague_interest_intent - ))); - } - - #[test] - fn standing_instruction_projection_is_cuepack_intent_gated() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "standing_instruction_test".to_string(), - ); - - let instruction_id = ctx.main.add_memory( - "Always provide fallback strategies when I ask about error handling in API services." - .to_string(), - vec!["api".to_string(), "error_handling".to_string()], - None, - MainStats::default(), - false, - ); - - assert!(standing_instruction_projection_cues( - &ctx, - None, - Some("What are some ways I can manage problems that come up when my API calls fail?") - ) - .cues - .is_empty()); - - let intent = crate::facets::QueryIntent { - labels: vec!["instruction_applicable".to_string()], - ..Default::default() - }; - let projection = standing_instruction_projection_cues( - &ctx, - Some(&intent), - Some("What are some ways I can manage problems that come up when my API calls fail?"), - ); - - assert!(projection - .cues - .iter() - .any(|(cue, _)| cue == "type:standing_instruction")); - assert!(projection - .cues - .iter() - .any(|(cue, _)| cue == "instruction_trigger:api")); - - let projection_results = ctx.main.recall_weighted( - projection.cues.clone(), - 10, - false, - None, - 1, - false, - true, - None, - None, - ); - let mut all_results = Vec::new(); - merge_standing_instruction_projection_results( - &ctx, - &mut all_results, - projection_results, - &projection.anchors, - ); - - let projected = all_results - .iter() - .find(|result| result.memory_id == instruction_id) - .expect("standing instruction should be projected"); - assert!(projected - .metadata - .contains_key("standing_instruction_projection")); - } - - #[test] - fn standing_instruction_projection_uses_morphological_anchor_variants() { - let anchors = - standing_instruction_projection_anchors(Some("How do I implement a login feature?")); - assert!(anchors.contains(&"implement".to_string())); - assert!(anchors.contains(&"implementation".to_string())); - - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "standing_instruction_morphology_test".to_string(), - ); - - ctx.main.add_memory( - "Always format code snippets with syntax highlighting when I ask about implementation details." - .to_string(), - Vec::new(), - None, - MainStats::default(), - false, - ); - - let intent = crate::facets::QueryIntent { - labels: vec!["instruction_applicable".to_string()], - ..Default::default() - }; - let projection = standing_instruction_projection_cues( - &ctx, - Some(&intent), - Some("How do I implement a login feature?"), - ); - - assert!(projection - .cues - .iter() - .any(|(cue, _)| cue == "instruction_trigger:implementation")); - } - - #[test] - fn standing_instruction_projection_maps_chance_to_probability_anchor() { - let anchors = standing_instruction_projection_anchors(Some( - "How do I calculate the chance of drawing a red card from a standard deck?", - )); - assert!(anchors.contains(&"chance".to_string())); - assert!(anchors.contains(&"probability".to_string())); - - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "standing_instruction_probability_test".to_string(), - ); - - ctx.main.add_memory( - "Always provide step-by-step explanations with concrete examples when I ask about probability concepts." - .to_string(), - Vec::new(), - None, - MainStats::default(), - false, - ); - - let intent = crate::facets::QueryIntent { - labels: vec!["instruction_applicable".to_string()], - ..Default::default() - }; - let projection = standing_instruction_projection_cues( - &ctx, - Some(&intent), - Some("How do I calculate the chance of drawing a red card from a standard deck?"), - ); - - assert!(projection - .cues - .iter() - .any(|(cue, _)| cue == "instruction_trigger:probability")); - } - - #[test] - fn preference_projection_is_cuepack_intent_gated() { - let ctx = ProjectContext::new( - NormalizationConfig::default(), - Taxonomy::default(), - Arc::new(TuningConfig::default()), - crate::config::ServerConfig::default(), - "preference_projection_test".to_string(), - ); - - let memory_id = ctx.main.add_memory( - "I prefer geometric vector methods over purely trigonometric formulas for clarity, so can you explain how to use vector algebra to calculate geodesic length between two points on a sphere?".to_string(), - vec![ - "sphere".to_string(), - "two_point".to_string(), - "vector".to_string(), - "geodesic".to_string(), - ], - None, - MainStats::default(), - false, - ); - - let query = "Can you show me how to find the shortest path between two points on a sphere?"; - assert!(preference_projection_cues(&ctx, None, Some(query)) - .cues - .is_empty()); - - let intent = crate::facets::QueryIntent { - labels: vec!["preference_applicable".to_string()], - ..Default::default() - }; - let projection = preference_projection_cues(&ctx, Some(&intent), Some(query)); - - assert!(projection - .cues - .iter() - .any(|(cue, _)| cue == "type:preference")); - assert!(projection - .cues - .iter() - .any(|(cue, _)| cue == "sphere" || cue == "two_point")); - - let projection_results = ctx.main.recall_weighted( - projection.cues.clone(), - 10, - false, - None, - 1, - false, - true, - None, - None, - ); - let mut all_results = Vec::new(); - merge_preference_projection_results( - &ctx, - &mut all_results, - projection_results, - &projection.anchors, - ); - - let projected = all_results - .iter() - .find(|result| result.memory_id == memory_id) - .expect("matching preference should be projected"); - assert!(projected.metadata.contains_key("preference_projection")); - } - - #[test] - fn user_context_projection_merge_marks_and_updates_results() { - let mut existing = recall_result(0.2, 1); - existing.memory_id = 10; - existing.score = 10.0; - - let mut projected = recall_result(0.8, 3); - projected.memory_id = 10; - projected.score = 50.0; - projected - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - - let mut all_results = vec![existing]; - merge_user_context_projection_results(&mut all_results, vec![projected]); - - assert_eq!(all_results.len(), 1); - assert_eq!(all_results[0].score, 50.0); - assert!(all_results[0] - .metadata - .contains_key("user_context_projection")); - } - - #[test] - fn source_prompt_projection_filters_short_scaffold_prompts() { - let mut scaffold = recall_result(0.1, 1); - scaffold.memory_id = 20; - scaffold.content = "user: Write another scene".to_string(); - scaffold.score = 5000.0; - scaffold - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - - let mut source = recall_result(0.8, 9); - source.memory_id = 21; - source.content = - "user: Write a comedy movie scene. Andy wears an untidy stained white shirt." - .to_string(); - source.score = 600.0; - source - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - - let mut results = Vec::new(); - merge_source_prompt_projection_results( - &mut results, - vec![scaffold, source], - Some("what was Andy wearing in the script you wrote for the comedy movie scene?"), - ); - - assert_eq!(results.len(), 1); - assert_eq!(results[0].memory_id, 21); - assert!(results[0] - .metadata - .contains_key("source_prompt_projection")); - assert!(results[0].score > 600.0); - } - - #[test] - fn user_context_adjacency_prefers_nearest_prior_user_turn() { - let mut expected = recall_result(0.2, 1); - expected.memory_id = 30; - expected.score = 100.0; - expected.created_at = 1.0; - expected - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - expected.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-3"), - ); - expected - .metadata - .insert("user_context_projection".to_string(), serde_json::json!(true)); - - let mut pivot = recall_result(0.6, 2); - pivot.memory_id = 31; - pivot.score = 500.0; - pivot.created_at = 2.0; - pivot - .metadata - .insert("source_role".to_string(), serde_json::json!("assistant")); - pivot.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-3"), - ); - - let mut later_user = recall_result(0.2, 1); - later_user.memory_id = 32; - later_user.score = 900.0; - later_user.created_at = 3.0; - later_user - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - later_user.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-3"), - ); - later_user - .metadata - .insert("user_context_projection".to_string(), serde_json::json!(true)); - - let mut results = vec![expected, pivot, later_user]; - apply_user_context_adjacency_preference(&mut results, None, Some("Any tips?")); - - assert!(results[0].score > results[2].score); - assert!(results[0] - .metadata - .contains_key("user_context_adjacency_boost")); - assert!(!results[2] - .metadata - .contains_key("user_context_adjacency_boost")); - } - - #[test] - fn user_context_adjacency_considers_bounded_multiple_pivots() { - fn with_source( - mut result: crate::engine::RecallResult, - role: &str, - session: &str, - projected: bool, - ) -> crate::engine::RecallResult { - result - .metadata - .insert("source_role".to_string(), serde_json::json!(role)); - result.metadata.insert( - "source_session_id".to_string(), - serde_json::json!(session), - ); - if projected { - result - .metadata - .insert("user_context_projection".to_string(), serde_json::json!(true)); - } - result - } - - let mut first_user = recall_result(0.2, 1); - first_user.memory_id = 40; - first_user.score = 90.0; - first_user.created_at = 1.0; - - let mut first_pivot = recall_result(0.6, 2); - first_pivot.memory_id = 41; - first_pivot.score = 900.0; - first_pivot.created_at = 2.0; - - let mut second_user = recall_result(0.2, 1); - second_user.memory_id = 42; - second_user.score = 80.0; - second_user.created_at = 3.0; - - let mut second_pivot = recall_result(0.6, 2); - second_pivot.memory_id = 43; - second_pivot.score = 800.0; - second_pivot.created_at = 4.0; - - let mut expected = recall_result(0.2, 1); - expected.memory_id = 44; - expected.score = 70.0; - expected.created_at = 5.0; - - let mut expected_pivot = recall_result(0.6, 2); - expected_pivot.memory_id = 45; - expected_pivot.score = 500.0; - expected_pivot.created_at = 6.0; - - let mut results = vec![ - with_source(first_user, "user", "conversation-7", true), - with_source(first_pivot, "assistant", "conversation-7", false), - with_source(second_user, "user", "conversation-7", true), - with_source(second_pivot, "assistant", "conversation-7", false), - with_source(expected, "user", "conversation-7", true), - with_source(expected_pivot, "assistant", "conversation-7", false), - ]; - - apply_user_context_adjacency_preference( - &mut results, - None, - Some("Any helpful tips?"), - ); - - assert!(results[4] - .metadata - .contains_key("user_context_adjacency_boost")); - assert!(results[4].score > 70.0); - } - - #[test] - fn source_session_cue_is_derived_from_structured_metadata() { - let mut metadata = HashMap::new(); - metadata.insert( - "source_session_id".to_string(), - serde_json::json!("Answer ShareGPT hA7AkP3 0"), - ); - - assert_eq!( - source_session_cue_from_metadata(&metadata).as_deref(), - Some("source_session:answer_sharegpt_ha7akp3_0") - ); - } - - #[test] - fn list_answer_detection_covers_ordinals_without_topic_words() { - assert!(query_wants_list_answer(Some( - "What was the 7th item you listed?" - ))); - assert!(query_wants_list_answer(Some( - "Remind me what was in the list you provided." - ))); - assert!(!query_wants_list_answer(Some( - "What did I purchase yesterday?" - ))); - } - - #[test] - fn source_role_preference_demotes_structured_role_mismatches() { - let mut user_result = recall_result(1.0, 3); - user_result.score = 100.0; - user_result - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - - let mut assistant_result = recall_result(1.0, 3); - assistant_result.memory_id = 50; - assistant_result.score = 80.0; - assistant_result - .metadata - .insert("source_role".to_string(), serde_json::json!("assistant")); - - let intent = crate::facets::QueryIntent { - labels: vec!["source_assistant".to_string()], - ..Default::default() - }; - let mut results = vec![user_result, assistant_result]; - apply_source_role_preference(&mut results, Some(&intent)); - - assert!(results[0].score < results[1].score); - assert_eq!(results[1].score, 80.0); - } - - #[test] - fn source_answer_adjacency_prefers_immediate_assistant_reply() { - let mut pivot = recall_result(1.0, 5); - pivot.score = 1000.0; - pivot.created_at = 1.0; - pivot.metadata - .insert("source_role".to_string(), serde_json::json!("user")); - pivot.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-1"), - ); - - let mut immediate_answer = recall_result(1.0, 2); - immediate_answer.memory_id = 60; - immediate_answer.score = 300.0; - immediate_answer.created_at = 2.0; - immediate_answer - .metadata - .insert("source_role".to_string(), serde_json::json!("assistant")); - immediate_answer.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-1"), - ); - - let mut later_answer = recall_result(1.0, 8); - later_answer.memory_id = 61; - later_answer.score = 1000.0; - later_answer.created_at = 6.0; - later_answer - .metadata - .insert("source_role".to_string(), serde_json::json!("assistant")); - later_answer.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-1"), - ); - - let intent = crate::facets::QueryIntent { - labels: vec!["source_answer".to_string(), "source_assistant".to_string()], - ..Default::default() - }; - let mut results = vec![pivot, immediate_answer, later_answer]; - apply_source_answer_adjacency_preference(&mut results, Some(&intent)); - - assert!(results[1].score > results[2].score); - assert!(results[1] - .metadata - .contains_key("source_answer_adjacency_boost")); - } - - #[test] - fn decision_adjacency_prefers_selection_after_proposal() { - let mut proposal = recall_result(1.0, 6); - proposal.score = 3000.0; - proposal.created_at = 1.0; - proposal.content = - "assistant: Here are some potential names: Radik, Nucleus, Fissionator.".to_string(); - proposal - .metadata - .insert("source_role".to_string(), serde_json::json!("assistant")); - proposal.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-2"), - ); - - let mut selected = recall_result(1.0, 1); - selected.memory_id = 70; - selected.score = 300.0; - selected.created_at = 2.0; - selected.content = "user: Fissionator is a really cool one.".to_string(); - selected - .metadata - .insert("source_role".to_string(), serde_json::json!("user")); - selected.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-2"), - ); - - let mut later = recall_result(1.0, 4); - later.memory_id = 71; - later.score = 900.0; - later.created_at = 5.0; - later.content = "assistant: Fissionator could have radioactive attacks.".to_string(); - later - .metadata - .insert("source_role".to_string(), serde_json::json!("assistant")); - later.metadata.insert( - "source_session_id".to_string(), - serde_json::json!("conversation-2"), - ); - - let intent = crate::facets::QueryIntent { - labels: vec![ - "decision_selection".to_string(), - "naming_decision".to_string(), - ], - ..Default::default() - }; - let mut results = vec![proposal, selected, later]; - apply_decision_adjacency_preference(&mut results, Some(&intent)); - - assert!(results[1].score > results[0].score); - assert!(results[1].score > results[2].score); - assert!(results[1] - .metadata - .contains_key("decision_adjacency_boost")); - } -} +#[path = "../tests/unit/api.rs"] +mod tests; diff --git a/src/auth.rs b/src/auth.rs index 0e2393c..22865aa 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -26,8 +26,9 @@ impl AuthConfig { // Load keys from config for key in &config.api_keys { + let key = key.trim(); if !key.is_empty() { - api_keys.insert(key.clone()); + api_keys.insert(key.to_string()); } } @@ -98,3 +99,92 @@ pub async fn auth_middleware( None => Err((StatusCode::UNAUTHORIZED, "Missing X-API-Key header")), } } + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + body::Body, + middleware, + routing::get, + http::{Request, StatusCode}, + Router, + }; + use tower::ServiceExt; + + fn config(require_auth: bool, keys: &[&str]) -> SecurityConfig { + SecurityConfig { + require_auth, + api_keys: keys.iter().map(|key| (*key).to_string()).collect(), + ..SecurityConfig::default() + } + } + + #[test] + fn disabled_auth_accepts_requests_and_filters_empty_keys() { + let auth = AuthConfig::from_config(&config(false, &["", " "])); + assert!(!auth.is_enabled()); + assert!(auth.validate_key("anything")); + assert!(auth.api_keys.is_empty()); + } + + #[test] + fn configured_keys_enable_auth_and_validate_exact_values() { + let auth = AuthConfig::from_config(&config(false, &["secret", "other"])); + assert!(auth.is_enabled()); + assert!(auth.validate_key("secret")); + assert!(!auth.validate_key("SECRET")); + assert!(!auth.validate_key("missing")); + } + + async fn test_router(auth: AuthConfig) -> Router { + Router::new() + .route("/", get(|| async { "ok" })) + .layer(middleware::from_fn_with_state(auth, auth_middleware)) + } + + #[tokio::test] + async fn middleware_allows_disabled_auth() { + let response = test_router(AuthConfig::from_config(&config(false, &[]))) + .await + .oneshot(Request::new(Body::empty())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn middleware_rejects_missing_and_invalid_keys_and_allows_valid_key() { + let router = test_router(AuthConfig::from_config(&config(true, &["secret"]))).await; + + let missing = router + .clone() + .oneshot(Request::new(Body::empty())) + .await + .unwrap(); + assert_eq!(missing.status(), StatusCode::UNAUTHORIZED); + + let invalid = router + .clone() + .oneshot( + Request::builder() + .header("X-API-Key", "wrong") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid.status(), StatusCode::UNAUTHORIZED); + + let valid = router + .oneshot( + Request::builder() + .header("X-API-Key", "secret") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(valid.status(), StatusCode::OK); + } +} diff --git a/src/config.rs b/src/config.rs index 8fe5f83..25abb97 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,3 +1,4 @@ +use crate::semantic::SemanticConfig; use serde::{Deserialize, Serialize}; use std::env; use std::fs; @@ -23,13 +24,13 @@ pub struct ServerConfig { #[serde(default)] pub jobs: JobsConfig, #[serde(default)] - pub cuepacks: CuePacksConfig, - #[serde(default)] pub agent: AgentConfig, #[serde(default)] pub search: SearchConfig, #[serde(default)] pub tuning: TuningConfig, + #[serde(default)] + pub semantic: SemanticConfig, } impl Default for ServerConfig { @@ -39,10 +40,10 @@ impl Default for ServerConfig { security: SecurityConfig::default(), persistence: PersistenceConfig::default(), jobs: JobsConfig::default(), - cuepacks: CuePacksConfig::default(), agent: AgentConfig::default(), search: SearchConfig::default(), tuning: TuningConfig::default(), + semantic: SemanticConfig::default(), } } } @@ -109,6 +110,106 @@ impl ServerConfig { config.security.master_key = Some(key); } + if let Ok(profile) = env::var("CUEMAP_SEMANTIC_PROFILE") { + config.semantic.profile = match profile.trim().to_ascii_lowercase().as_str() { + "edge" => crate::semantic::SemanticProfile::Edge, + "balanced" => crate::semantic::SemanticProfile::Balanced, + "quality" => crate::semantic::SemanticProfile::Quality, + "off" => crate::semantic::SemanticProfile::Off, + _ => config.semantic.profile, + }; + } + if let Ok(enabled) = env::var("CUEMAP_SEMANTIC_ENCODER_ENABLED") { + if let Ok(enabled) = enabled.parse::() { + config.semantic.encoder_enabled = enabled; + } + } + if let Ok(dimensions) = env::var("CUEMAP_SEMANTIC_DIMENSIONS") { + if let Ok(dimensions) = dimensions.parse::() { + config.semantic.dimensions = dimensions; + } + } + if let Ok(storage) = env::var("CUEMAP_SEMANTIC_STORAGE") { + config.semantic.storage = match storage.trim().to_ascii_lowercase().as_str() { + "f32" => crate::semantic::SemanticStorage::F32, + "f16" => crate::semantic::SemanticStorage::F16, + "int8" => crate::semantic::SemanticStorage::Int8, + "auto" => crate::semantic::SemanticStorage::Auto, + _ => config.semantic.storage, + }; + } + if let Ok(index) = env::var("CUEMAP_SEMANTIC_INDEX") { + config.semantic.index = match index.trim().to_ascii_lowercase().as_str() { + "exact" => crate::semantic::SemanticIndexMode::Exact, + "ann" => crate::semantic::SemanticIndexMode::Ann, + "auto" => crate::semantic::SemanticIndexMode::Auto, + _ => config.semantic.index, + }; + } + if let Ok(model_id) = env::var("CUEMAP_SEMANTIC_MODEL_ID") { + config.semantic.model_id = model_id; + } + if let Ok(model_version) = env::var("CUEMAP_SEMANTIC_MODEL_VERSION") { + config.semantic.model_version = model_version; + } + if let Ok(model_path) = env::var("CUEMAP_SEMANTIC_MODEL_PATH") { + config.semantic.model_path = model_path; + } + if let Ok(tokenizer_path) = env::var("CUEMAP_SEMANTIC_TOKENIZER_PATH") { + config.semantic.tokenizer_path = tokenizer_path; + } + if let Ok(max_tokens) = env::var("CUEMAP_SEMANTIC_MAX_TOKENS") { + if let Ok(max_tokens) = max_tokens.parse::() { + config.semantic.max_tokens = max_tokens; + } + } + if let Ok(threads) = env::var("CUEMAP_SEMANTIC_ENCODER_THREADS") { + if let Ok(threads) = threads.parse::() { + config.semantic.encoder_threads = threads; + } + } + if let Ok(enabled) = env::var("CUEMAP_SEMANTIC_COREML_ENABLED") { + if let Ok(enabled) = enabled.parse::() { + config.semantic.coreml_enabled = enabled; + } + } + if let Ok(weight) = env::var("CUEMAP_SEMANTIC_RERANK_WEIGHT") { + if let Ok(weight) = weight.parse::() { + config.semantic.semantic_rerank_weight = weight; + } + } + if let Ok(limit) = env::var("CUEMAP_SEMANTIC_RERANK_CANDIDATE_LIMIT") { + if let Ok(limit) = limit.parse::() { + config.semantic.semantic_rerank_candidate_limit = limit; + } + } + if let Ok(capacity) = env::var("CUEMAP_SEMANTIC_QUERY_CACHE_CAPACITY") { + if let Ok(capacity) = capacity.parse::() { + config.semantic.query_embedding_cache_capacity = capacity; + } + } + if let Ok(enabled) = env::var("CUEMAP_SEMANTIC_INTENT_RERANK_ENABLED") { + if let Ok(enabled) = enabled.parse::() { + config.semantic.intent_rerank_enabled = enabled; + } + } + if let Ok(weight) = env::var("CUEMAP_SEMANTIC_INTENT_RERANK_WEIGHT") { + if let Ok(weight) = weight.parse::() { + config.semantic.intent_rerank_weight = weight; + } + } + if let Ok(penalty) = env::var("CUEMAP_SEMANTIC_INTENT_NO_RECALL_PENALTY") { + if let Ok(penalty) = penalty.parse::() { + config.semantic.intent_no_recall_penalty = penalty; + } + } + if let Ok(max_delta) = env::var("CUEMAP_SEMANTIC_INTENT_RERANK_MAX_DELTA") { + if let Ok(max_delta) = max_delta.parse::() { + config.semantic.intent_rerank_max_delta = max_delta; + } + } + + config.semantic = config.semantic.resolved(); Ok(config) } @@ -217,23 +318,6 @@ impl Default for JobsConfig { } } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct CuePacksConfig { - pub enabled: bool, - pub default_packs_enabled: bool, - pub dirs: Vec, -} - -impl Default for CuePacksConfig { - fn default() -> Self { - Self { - enabled: true, - default_packs_enabled: true, - dirs: Vec::new(), - } - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct AgentConfig { pub enabled: bool, @@ -299,3 +383,113 @@ impl Default for TuningConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::semantic::{SemanticIndexMode, SemanticProfile, SemanticStorage}; + use std::ffi::OsString; + + #[test] + fn profile_defaults_apply_expected_runtime_modes() { + let read_only = ServerConfig::default_for_profile("read_only"); + assert!(read_only.server.read_only); + assert!(!read_only.persistence.enabled); + assert!(!read_only.jobs.background_processing); + + let live = ServerConfig::default_for_profile("live"); + assert!(live.persistence.enabled); + assert!(live.jobs.background_processing); + + let benchmark = ServerConfig::default_for_profile("benchmark"); + assert!(!benchmark.persistence.enabled); + assert!(!benchmark.jobs.background_processing); + assert_eq!(benchmark.server.log_level, "warn"); + + let default = ServerConfig::default_for_profile("unknown"); + assert_eq!(default.server.port, 8080); + } + + #[test] + fn load_reads_toml_and_applies_environment_overrides() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("server.toml"); + let mut file_config = ServerConfig::default(); + file_config.server.port = 9000; + file_config.persistence.enabled = false; + std::fs::write(&path, toml::to_string(&file_config).unwrap()).unwrap(); + + let vars = [ + ("CUEMAP_PORT", "9123"), + ("CUEMAP_DATA_DIR", "/tmp/cuemap-test-data"), + ("CUEMAP_SNAPSHOT_INTERVAL_SECONDS", "7"), + ("CUEMAP_SECRET_KEY", "secret"), + ("CUEMAP_SIGNING_PRIVATE_KEY", "signing"), + ("CUEMAP_MASTER_KEY", "master"), + ("CUEMAP_SEMANTIC_PROFILE", "edge"), + ("CUEMAP_SEMANTIC_ENCODER_ENABLED", "true"), + ("CUEMAP_SEMANTIC_DIMENSIONS", "384"), + ("CUEMAP_SEMANTIC_STORAGE", "int8"), + ("CUEMAP_SEMANTIC_INDEX", "exact"), + ("CUEMAP_SEMANTIC_MODEL_ID", "test-model"), + ("CUEMAP_SEMANTIC_MODEL_VERSION", "v-test"), + ("CUEMAP_SEMANTIC_MAX_TOKENS", "64"), + ("CUEMAP_SEMANTIC_ENCODER_THREADS", "2"), + ("CUEMAP_SEMANTIC_COREML_ENABLED", "true"), + ("CUEMAP_SEMANTIC_RERANK_WEIGHT", "0.75"), + ("CUEMAP_SEMANTIC_RERANK_CANDIDATE_LIMIT", "11"), + ("CUEMAP_SEMANTIC_QUERY_CACHE_CAPACITY", "13"), + ("CUEMAP_SEMANTIC_INTENT_RERANK_ENABLED", "true"), + ("CUEMAP_SEMANTIC_INTENT_RERANK_WEIGHT", "0.25"), + ("CUEMAP_SEMANTIC_INTENT_NO_RECALL_PENALTY", "0.5"), + ("CUEMAP_SEMANTIC_INTENT_RERANK_MAX_DELTA", "0.2"), + ]; + for (key, value) in vars { + std::env::set_var(key, value); + } + + let loaded = ServerConfig::load(Some(path), Some("live".to_string())).unwrap(); + + for (key, _) in vars { + std::env::remove_var(key); + } + + assert_eq!(loaded.server.port, 9123); + assert_eq!(loaded.server.data_dir, "/tmp/cuemap-test-data"); + assert_eq!(loaded.persistence.snapshot_interval_seconds, 7); + assert_eq!(loaded.security.secret_key.as_deref(), Some("secret")); + assert_eq!(loaded.security.signing_private_key.as_deref(), Some("signing")); + assert_eq!(loaded.security.master_key.as_deref(), Some("master")); + assert_eq!(loaded.semantic.profile, SemanticProfile::Edge); + assert!(loaded.semantic.encoder_enabled); + assert_eq!(loaded.semantic.dimensions, 384); + assert_eq!(loaded.semantic.storage, SemanticStorage::Int8); + assert_eq!(loaded.semantic.index, SemanticIndexMode::Exact); + assert_eq!(loaded.semantic.model_id, "test-model"); + assert_eq!(loaded.semantic.model_version, "v-test"); + assert_eq!(loaded.semantic.max_tokens, 64); + assert_eq!(loaded.semantic.encoder_threads, 2); + assert!(loaded.semantic.coreml_enabled); + assert_eq!(loaded.semantic.semantic_rerank_weight, 0.75); + assert_eq!(loaded.semantic.semantic_rerank_candidate_limit, 11); + assert_eq!(loaded.semantic.query_embedding_cache_capacity, 13); + assert!(loaded.semantic.intent_rerank_enabled); + assert_eq!(loaded.semantic.intent_rerank_weight, 0.25); + assert_eq!(loaded.semantic.intent_no_recall_penalty, 0.5); + assert_eq!(loaded.semantic.intent_rerank_max_delta, 0.2); + } + + #[test] + fn load_ignores_invalid_environment_values_and_missing_files() { + let key = "CUEMAP_PORT"; + let previous: Option = std::env::var_os(key); + std::env::set_var(key, "not-a-port"); + let loaded = ServerConfig::load(Some(PathBuf::from("/path/that/does/not/exist")), None) + .unwrap(); + match previous { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + assert_eq!(loaded.server.port, 8080); + } +} diff --git a/src/crypto.rs b/src/crypto.rs index 63cb73e..8c74004 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -149,3 +149,66 @@ impl ContextSigner { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn passphrase_keys_are_32_bytes_and_deterministic() { + let first = EncryptionKey::from_passphrase("secret", b"salt"); + let second = EncryptionKey::from_passphrase("secret", b"salt"); + let different = EncryptionKey::from_passphrase("other", b"salt"); + assert_eq!(first.as_bytes().len(), 32); + assert_eq!(first.as_bytes(), second.as_bytes()); + assert_ne!(first.as_bytes(), different.as_bytes()); + } + + #[test] + fn compression_round_trip_and_magic_detection_work() { + let payload = b"a repeated payload that benefits from compression"; + let compressed = compress(payload).unwrap(); + assert!(is_compressed(&compressed)); + assert!(!is_compressed(payload)); + assert!(!is_compressed(&[0x28, 0xB5, 0x2F])); + assert_eq!(decompress(&compressed).unwrap(), payload); + assert!(decompress(b"not zstd").is_err()); + } + + #[test] + fn encryption_round_trip_rejects_short_and_wrong_ciphertext() { + let key = EncryptionKey::new(vec![7; 32]); + let ciphertext = encrypt(b"private text", &key).unwrap(); + assert_ne!(ciphertext, b"private text"); + assert_eq!(decrypt(&ciphertext, &key).unwrap(), b"private text"); + assert!(decrypt(&ciphertext[..11], &key).is_err()); + + let wrong_key = EncryptionKey::new(vec![8; 32]); + assert!(decrypt(&ciphertext, &wrong_key).is_err()); + } + + #[test] + fn context_signers_produce_expected_algorithms() { + let hmac = ContextSigner::from_hmac_secret(b"secret".to_vec()).sign("payload"); + assert_eq!(hmac.algorithm, "hmac-sha256"); + assert_eq!(hmac.signature.len(), 64); + assert!(hmac.public_key.is_none()); + + let seed = format!("ed25519:{}", hex::encode([1u8; 32])); + let ed25519 = ContextSigner::from_ed25519_seed_hex(&seed).unwrap().sign("payload"); + assert_eq!(ed25519.algorithm, "ed25519"); + assert_eq!(ed25519.signature.len(), 128); + assert!(ed25519.public_key.as_deref().unwrap().starts_with("ed25519:")); + } + + #[test] + fn ed25519_seed_validation_reports_useful_errors() { + assert!(ContextSigner::from_ed25519_seed_hex("not-hex").is_err()); + let short = hex::encode([1u8; 16]); + let error = match ContextSigner::from_ed25519_seed_hex(&short) { + Ok(_) => panic!("short seeds must be rejected"), + Err(error) => error, + }; + assert!(error.contains("32-byte hex seed")); + } +} diff --git a/src/cuebridge.rs b/src/cuebridge.rs index 0e090e3..33e66a7 100644 --- a/src/cuebridge.rs +++ b/src/cuebridge.rs @@ -250,7 +250,7 @@ impl CueBridgeArtifacts { pub fn gap_expansions( &self, query_cues: &[(String, f64)], - query_intent: Option<&crate::facets::QueryIntent>, + query_plan: Option<&crate::facets::StructuralQueryPlan>, ordered_tokens: &[String], available: F, max_expansions: usize, @@ -264,7 +264,7 @@ impl CueBridgeArtifacts { let query_cue_set = normalized_set(query_cues.iter().map(|(cue, _)| cue)); let token_set = normalized_set(ordered_tokens.iter()); - let intent_set = query_intent + let intent_set = query_plan .map(|intent| normalized_set(intent.labels.iter())) .unwrap_or_default(); let mut out = Vec::new(); @@ -592,24 +592,5 @@ fn sanitize_weight(value: f64, default: f64) -> f64 { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn gap_pack_rejects_bare_entry_without_gates() { - let entry = RuntimeGapEntry { - artifact: "test".to_string(), - artifact_hash: "hash".to_string(), - id: "gap".to_string(), - signature: RuntimeQuerySignature::default(), - expansions: vec![RawExpansion { - cue: "target".to_string(), - weight: 1.0, - }], - negative_gates: Vec::new(), - confidence: 1.0, - max_fanout: 1, - }; - assert!(!entry.matches(&HashSet::new(), &HashSet::new(), &HashSet::new())); - } -} +#[path = "../tests/unit/cuebridge.rs"] +mod tests; diff --git a/src/cuepacks.rs b/src/cuepacks.rs deleted file mode 100644 index 82ec6e8..0000000 --- a/src/cuepacks.rs +++ /dev/null @@ -1,469 +0,0 @@ -use regex::Regex; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::sync::OnceLock; - -const BUNDLED_MEMORY_GENERAL: &str = include_str!("../cuepacks/memory-general.toml"); -const OFF_SENTINELS: &[&str] = &["off", "none", "disabled", "core-only"]; -const DEFAULT_SENTINELS: &[&str] = &["default", "defaults", "bundled"]; - -#[derive(Debug, Clone, Default, Serialize)] -pub struct CuePackRegistry { - packs: Vec, - load_errors: Vec, -} - -#[derive(Debug, Clone, Serialize)] -pub struct CuePackInfo { - pub name: String, - pub version: String, - pub description: Option, - pub enabled_by_default: bool, - pub source: String, - pub memory_rules: usize, - pub query_rules: usize, -} - -#[derive(Debug, Clone, Default, Serialize, PartialEq)] -pub struct CuePackFacetOutput { - pub facets: Vec, - pub matched_rules: Vec, -} - -#[derive(Debug, Clone, Default, Serialize, PartialEq)] -pub struct CuePackQueryOutput { - pub labels: Vec, - pub weighted_cues: Vec<(String, f64)>, - pub cue_weight_adjustments: Vec<(String, f64)>, - pub suppress_generic: bool, - pub matched_rules: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -struct RawCuePack { - name: String, - version: Option, - description: Option, - #[serde(default = "default_true")] - enabled_by_default: bool, - #[serde(default)] - memory_rules: Vec, - #[serde(default)] - query_rules: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -struct RawRule { - id: String, - #[serde(default)] - contains_any: Vec, - #[serde(default)] - contains_all: Vec, - #[serde(default)] - regex_any: Vec, - #[serde(default)] - emits: Vec, - #[serde(default)] - labels: Vec, - #[serde(default)] - weighted_cues: Vec, - #[serde(default)] - cue_weight_adjustments: Vec, - #[serde(default)] - suppress_generic: bool, -} - -#[derive(Debug, Clone, Deserialize)] -struct RawWeightedCue { - cue: String, - weight: f64, -} - -#[derive(Debug, Clone, Serialize)] -struct CompiledCuePack { - name: String, - version: String, - description: Option, - enabled_by_default: bool, - source: String, - memory_rules: Vec, - query_rules: Vec, -} - -#[derive(Debug, Clone, Serialize)] -struct CompiledRule { - id: String, - contains_any: Vec, - contains_all: Vec, - #[serde(skip)] - regex_any: Vec, - emits: Vec, - labels: Vec, - weighted_cues: Vec<(String, f64)>, - cue_weight_adjustments: Vec<(String, f64)>, - suppress_generic: bool, -} - -fn default_true() -> bool { - true -} - -pub fn default_registry() -> &'static CuePackRegistry { - static REGISTRY: OnceLock = OnceLock::new(); - REGISTRY.get_or_init(CuePackRegistry::bundled) -} - -impl CuePackRegistry { - pub fn bundled() -> Self { - let mut registry = Self::default(); - match Self::compile_pack(BUNDLED_MEMORY_GENERAL, "bundled:memory-general") { - Ok(pack) => registry.packs.push(pack), - Err(err) => registry.load_errors.push(err), - } - registry.sort(); - registry - } - - pub fn load(default_packs_enabled: bool, dirs: &[PathBuf]) -> Self { - let mut registry = if default_packs_enabled { - Self::bundled() - } else { - Self::default() - }; - - for dir in dirs { - registry.load_dir(dir); - } - registry.sort(); - registry - } - - pub fn load_from_default_locations(default_packs_enabled: bool) -> Self { - let base_dir = crate::config::get_base_dir(); - Self::load(default_packs_enabled, &[base_dir.join("cuepacks")]) - } - - pub fn load_errors(&self) -> &[String] { - &self.load_errors - } - - pub fn infos(&self) -> Vec { - self.packs - .iter() - .map(|pack| CuePackInfo { - name: pack.name.clone(), - version: pack.version.clone(), - description: pack.description.clone(), - enabled_by_default: pack.enabled_by_default, - source: pack.source.clone(), - memory_rules: pack.memory_rules.len(), - query_rules: pack.query_rules.len(), - }) - .collect() - } - - pub fn validate_file(path: &Path) -> Result { - let content = fs::read_to_string(path) - .map_err(|err| format!("failed to read {}: {}", path.display(), err))?; - let pack = Self::compile_pack(&content, &path.display().to_string())?; - Ok(CuePackInfo { - name: pack.name, - version: pack.version, - description: pack.description, - enabled_by_default: pack.enabled_by_default, - source: pack.source, - memory_rules: pack.memory_rules.len(), - query_rules: pack.query_rules.len(), - }) - } - - pub fn extract_memory_facets( - &self, - content: &str, - selection: Option<&[String]>, - ) -> CuePackFacetOutput { - let normalized = padded_normalized(content); - let mut out = CuePackFacetOutput::default(); - let mut seen = HashSet::new(); - - for pack in self.active_packs(selection) { - for rule in &pack.memory_rules { - if rule.matches(content, &normalized) { - out.matched_rules.push(format!("{}:{}", pack.name, rule.id)); - for facet in &rule.emits { - if seen.insert(facet.clone()) { - out.facets.push(facet.clone()); - } - } - } - } - } - - out - } - - pub fn compile_query_intent( - &self, - query: &str, - selection: Option<&[String]>, - available: F, - ) -> CuePackQueryOutput - where - F: Fn(&str) -> bool, - { - let normalized = padded_normalized(query); - let mut out = CuePackQueryOutput::default(); - let mut labels = HashSet::new(); - let mut cues = HashSet::new(); - let mut adjustments: HashMap = HashMap::new(); - - for pack in self.active_packs(selection) { - for rule in &pack.query_rules { - if !rule.matches(query, &normalized) { - continue; - } - - let mut emitted = false; - for label in &rule.labels { - if labels.insert(label.clone()) { - out.labels.push(label.clone()); - emitted = true; - } - } - for (cue, weight) in &rule.weighted_cues { - if available(cue) && cues.insert(cue.clone()) { - out.weighted_cues.push((cue.clone(), *weight)); - emitted = true; - } - } - for (cue, weight) in &rule.cue_weight_adjustments { - if !available(cue) { - continue; - } - adjustments - .entry(cue.clone()) - .and_modify(|existing| { - if *existing > *weight { - *existing = *weight; - } - }) - .or_insert(*weight); - emitted = true; - } - if rule.suppress_generic { - out.suppress_generic = true; - emitted = true; - } - if emitted { - out.matched_rules.push(format!("{}:{}", pack.name, rule.id)); - } - } - } - - out.cue_weight_adjustments = adjustments.into_iter().collect(); - out.cue_weight_adjustments.sort_by(|a, b| a.0.cmp(&b.0)); - out - } - - fn load_dir(&mut self, dir: &Path) { - if !dir.exists() { - return; - } - - let entries = match fs::read_dir(dir) { - Ok(entries) => entries, - Err(err) => { - self.load_errors - .push(format!("failed to read {}: {}", dir.display(), err)); - return; - } - }; - - let mut paths = entries - .flatten() - .map(|entry| entry.path()) - .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("toml")) - .collect::>(); - paths.sort(); - - for path in paths { - match fs::read_to_string(&path) - .map_err(|err| format!("failed to read {}: {}", path.display(), err)) - .and_then(|content| Self::compile_pack(&content, &path.display().to_string())) - { - Ok(pack) => self.replace_or_push(pack), - Err(err) => self.load_errors.push(err), - } - } - } - - fn replace_or_push(&mut self, pack: CompiledCuePack) { - if let Some(existing) = self - .packs - .iter_mut() - .find(|existing| existing.name == pack.name) - { - *existing = pack; - } else { - self.packs.push(pack); - } - } - - fn active_packs(&self, selection: Option<&[String]>) -> Vec<&CompiledCuePack> { - let Some(selection) = selection else { - return self - .packs - .iter() - .filter(|pack| pack.enabled_by_default) - .collect(); - }; - - if selection - .iter() - .any(|name| OFF_SENTINELS.contains(&name.to_lowercase().as_str())) - { - return Vec::new(); - } - - let mut include_defaults = false; - let requested = selection - .iter() - .filter_map(|name| { - let lower = name.to_lowercase(); - if DEFAULT_SENTINELS.contains(&lower.as_str()) { - include_defaults = true; - None - } else { - Some(lower) - } - }) - .collect::>(); - - self.packs - .iter() - .filter(|pack| { - (include_defaults && pack.enabled_by_default) - || requested.contains(&pack.name.to_lowercase()) - }) - .collect() - } - - fn compile_pack(content: &str, source: &str) -> Result { - let raw: RawCuePack = - toml::from_str(content).map_err(|err| format!("invalid cuepack {source}: {err}"))?; - if raw.name.trim().is_empty() { - return Err(format!("invalid cuepack {source}: missing name")); - } - - Ok(CompiledCuePack { - name: raw.name.trim().to_string(), - version: raw.version.unwrap_or_else(|| "0.1.0".to_string()), - description: raw.description, - enabled_by_default: raw.enabled_by_default, - source: source.to_string(), - memory_rules: compile_rules(raw.memory_rules, source)?, - query_rules: compile_rules(raw.query_rules, source)?, - }) - } - - fn sort(&mut self) { - self.packs.sort_by(|a, b| a.name.cmp(&b.name)); - self.load_errors.sort(); - } -} - -impl CompiledRule { - fn matches(&self, original: &str, normalized: &str) -> bool { - if !self.contains_all.is_empty() - && !self - .contains_all - .iter() - .all(|term| normalized.contains(term)) - { - return false; - } - - if !self.contains_any.is_empty() - && !self - .contains_any - .iter() - .any(|term| normalized.contains(term)) - { - return false; - } - - if !self.regex_any.is_empty() && !self.regex_any.iter().any(|re| re.is_match(original)) { - return false; - } - - true - } -} - -fn compile_rules(rules: Vec, source: &str) -> Result, String> { - let mut compiled = Vec::with_capacity(rules.len()); - let mut ids = HashSet::new(); - - for raw in rules { - if raw.id.trim().is_empty() { - return Err(format!("invalid cuepack {source}: rule with empty id")); - } - if !ids.insert(raw.id.clone()) { - return Err(format!("invalid cuepack {source}: duplicate rule {}", raw.id)); - } - - let mut regex_any = Vec::with_capacity(raw.regex_any.len()); - for pattern in raw.regex_any { - regex_any.push( - Regex::new(&pattern) - .map_err(|err| format!("invalid cuepack {source} rule {}: {err}", raw.id))?, - ); - } - - compiled.push(CompiledRule { - id: raw.id, - contains_any: raw - .contains_any - .into_iter() - .map(|term| padded_rule_term(&term)) - .collect(), - contains_all: raw - .contains_all - .into_iter() - .map(|term| padded_rule_term(&term)) - .collect(), - regex_any, - emits: raw.emits, - labels: raw.labels, - weighted_cues: raw - .weighted_cues - .into_iter() - .map(|item| (item.cue, item.weight)) - .collect(), - cue_weight_adjustments: raw - .cue_weight_adjustments - .into_iter() - .map(|item| (item.cue, item.weight)) - .collect(), - suppress_generic: raw.suppress_generic, - }); - } - - compiled.sort_by(|a, b| a.id.cmp(&b.id)); - Ok(compiled) -} - -fn padded_normalized(input: &str) -> String { - format!(" {} ", crate::nl::normalize_text(input)) -} - -fn padded_rule_term(input: &str) -> String { - let normalized = crate::nl::normalize_text(input); - if normalized.starts_with(' ') || normalized.ends_with(' ') { - normalized - } else { - format!(" {normalized} ") - } -} diff --git a/src/engine.rs b/src/engine.rs index a814282..d0f70e6 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,17 +1,24 @@ use crate::config::TuningConfig; use crate::crypto::EncryptionKey; +use crate::intent::{ + intent_compatibility, IntentClassification, IntentClassifier, IntentTarget, + INTENT_TAXONOMY_VERSION, +}; +use crate::semantic::{LinearReranker, SemanticEncoder, SemanticIndex, StoredSemanticVector}; use crate::structures::{ LexiconStats, MainStats, Memory, MemoryId, MemoryScoringFeatures, MemoryStats, OrderedSet, INVALID_MEMORY_ID, }; use ahash::RandomState; use dashmap::DashMap; +use lru::LruCache; use serde::{Deserialize, Serialize}; use std::any::TypeId; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, RwLock}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::num::NonZeroUsize; const MEMORY_SCORING_FEATURES_VERSION: u8 = 1; const FAMILY_PERSON: u64 = 1 << 0; @@ -69,6 +76,8 @@ pub struct RecallTimingBreakdown { pub scanned_posting_count: usize, pub adaptive_scan_limit: usize, pub max_posting_len: usize, + pub semantic_rerank_candidate_limit: usize, + pub semantic_rerank_candidate_count: usize, } #[derive(Debug, Clone, Default)] @@ -118,6 +127,9 @@ pub struct ScoredMemoryCandidate { pub match_count: f64, pub rerank_bonus: f64, pub generic_penalty: f64, + pub semantic_similarity: f32, + pub intent_compatibility: f64, + pub intent_rerank_bonus: f64, } struct QueryScoringCue<'a> { @@ -493,6 +505,19 @@ where /// Global occurrences of each cue (for IDF weighting) pub cue_global_counts: Arc>, + /// Optional vector index. The index is rebuilt from persisted vectors and + /// remains empty unless semantic retrieval is explicitly enabled. + semantic_index: Arc>, + /// Optional local text encoder. It is loaded only when the binary includes + /// the encoder feature and the project explicitly enables it in config. + semantic_encoder: Arc>>>, + /// Intent classifier built from the configured local encoder and the + /// versioned CueKey taxonomy. + intent_classifier: Arc>>>, + /// Bounded cache for repeated query text embeddings. The lock is held + /// only while accessing the cache, never while running the encoder. + query_embedding_cache: Arc>>>, + // Storage context pub config: crate::config::ServerConfig, pub project_id: String, @@ -522,6 +547,14 @@ where master_key: None, tuning: Arc::new(TuningConfig::default()), cue_global_counts: Arc::new(DashMap::with_hasher(RandomState::new())), + semantic_index: Arc::new(RwLock::new(SemanticIndex::new( + crate::semantic::SemanticConfig::default(), + ))), + semantic_encoder: Arc::new(RwLock::new(None)), + intent_classifier: Arc::new(RwLock::new(None)), + query_embedding_cache: Self::new_query_embedding_cache( + &crate::semantic::SemanticConfig::default(), + ), config: crate::config::ServerConfig::default(), project_id: "default".to_string(), } @@ -547,6 +580,211 @@ where self.tuning = Arc::new(tuning); } + pub fn set_semantic_config(&mut self, config: crate::semantic::SemanticConfig) { + let config = config.resolved(); + self.config.semantic = config.clone(); + self.query_embedding_cache = Self::new_query_embedding_cache(&config); + let mut index = SemanticIndex::new(config); + index.rebuild(self.memories.iter().filter_map(|entry| { + entry + .semantic_vector + .as_ref() + .map(|vector| (entry.id, vector.clone())) + })); + self.semantic_index = Arc::new(RwLock::new(index)); + } + + pub fn set_semantic_encoder(&mut self, encoder: Option>) { + let classifier = encoder.as_ref().and_then(|encoder| { + match IntentClassifier::new( + encoder.clone(), + self.config.semantic.model_version.clone(), + ) { + Ok(classifier) => Some(Arc::new(classifier)), + Err(error) => { + tracing::warn!(error = %error, "Intent classifier unavailable"); + None + } + } + }); + self.semantic_encoder = Arc::new(RwLock::new(encoder)); + self.intent_classifier = Arc::new(RwLock::new(classifier)); + if let Ok(mut cache) = self.query_embedding_cache.lock() { + cache.clear(); + } + } + + pub fn configure_semantic_encoder(&mut self) -> Result<(), String> { + let encoder = crate::semantic::load_configured_encoder(&self.config.semantic)?; + self.set_semantic_encoder(encoder); + Ok(()) + } + + fn new_query_embedding_cache( + config: &crate::semantic::SemanticConfig, + ) -> Arc>>> { + let capacity = NonZeroUsize::new(config.resolved().query_embedding_cache_capacity.max(1)) + .expect("query embedding cache capacity is always non-zero"); + Arc::new(Mutex::new(LruCache::new(capacity))) + } + + /// Encode text with the bundled local encoder when enabled. Callers can + /// still provide an embedding directly for a single memory/query, and + /// can disable automatic encoding through `SemanticConfig`. + pub fn encode_semantic_text(&self, text: &str) -> Option> { + let config = self.config.semantic.resolved(); + if !config.enabled || !config.encoder_enabled { + return None; + } + let cache_enabled = config.query_embedding_cache_capacity > 0; + if cache_enabled { + if let Ok(mut cache) = self.query_embedding_cache.lock() { + if let Some(vector) = cache.get(text) { + return Some(vector.clone()); + } + } + } + let encoder = self.semantic_encoder.read().ok()?.clone()?; + let vector = match encoder.encode(text) { + Ok(vector) => vector, + Err(error) => { + tracing::debug!(error = %error, "Semantic text encoding skipped"); + return None; + } + }; + if vector.len() != encoder.dimensions() { + tracing::debug!( + expected = encoder.dimensions(), + received = vector.len(), + "Semantic text encoder returned incompatible dimensions" + ); + return None; + } + if cache_enabled { + if let Ok(mut cache) = self.query_embedding_cache.lock() { + cache.put(text.to_owned(), vector.clone()); + } + } + Some(vector) + } + + pub fn classify_intent( + &self, + text: &str, + target: IntentTarget, + ) -> Result { + let classifier = self + .intent_classifier + .read() + .map_err(|_| "intent classifier lock poisoned".to_string())? + .clone() + .ok_or_else(|| "intent classifier unavailable".to_string())?; + classifier.classify(text, target) + } + + pub fn classify_intent_with_embedding( + &self, + text: &str, + target: IntentTarget, + embedding: &[f32], + ) -> Result { + let classifier = self + .intent_classifier + .read() + .map_err(|_| "intent classifier lock poisoned".to_string())? + .clone() + .ok_or_else(|| "intent classifier unavailable".to_string())?; + classifier.classify_with_embedding(text, embedding, target) + } + + pub fn attach_intent_classification( + &self, + memory_id: MemoryId, + classification: IntentClassification, + ) -> bool { + if let Some(mut memory) = self.memories.get_mut(&memory_id) { + memory.intent_classification = Some(classification); + true + } else { + false + } + } + + /// Return `(total, annotated, missing_current_version, stale_version)` for + /// memory intent metadata. A stale annotation is treated as missing for + /// readiness because changing the model or taxonomy invalidates its score. + pub fn intent_coverage(&self) -> (usize, usize, usize, usize) { + let expected_model_version = self.config.semantic.model_version.as_str(); + let mut total = 0; + let mut annotated = 0; + let mut current = 0; + let mut stale = 0; + + for entry in self.memories.iter() { + total += 1; + match entry.intent_classification.as_ref() { + Some(classification) => { + annotated += 1; + if classification.taxonomy_version == INTENT_TAXONOMY_VERSION + && classification.model_version == expected_model_version + { + current += 1; + } else { + stale += 1; + } + } + None => {} + } + } + + (total, annotated, total.saturating_sub(current), stale) + } + + pub fn semantic_index_stats(&self) -> (bool, usize, Option) { + self.semantic_index + .read() + .map(|index| { + ( + index.config().enabled, + index.len(), + index.dimensions(), + ) + }) + .unwrap_or((false, 0, None)) + } + + fn prepare_semantic_vector(&self, vector: Option>) -> Option { + let vector = vector?; + let config = self.config.semantic.resolved(); + if !config.enabled { + return None; + } + if config.dimensions != 0 && vector.len() != config.dimensions { + tracing::debug!( + expected = config.dimensions, + received = vector.len(), + "Skipping semantic vector with incompatible dimensions" + ); + return None; + } + let memory_count = self.memory_count.load(Ordering::Relaxed).saturating_add(1); + if !config.within_memory_budget_for_dimensions(vector.len(), memory_count) { + tracing::debug!( + memory_count, + max_memory_mb = config.max_memory_mb, + "Skipping semantic vector because the configured memory budget is full" + ); + return None; + } + match StoredSemanticVector::from_f32(&vector, config.storage) { + Ok(vector) => Some(vector), + Err(error) => { + tracing::debug!(error = %error, "Skipping invalid semantic vector"); + None + } + } + } + pub fn get_master_key(&self) -> Option> { self.master_key.clone() } @@ -557,9 +795,10 @@ where cue_index: DashMap, next_memory_id: MemoryId, loaded_global_counts: Option>, - config: crate::config::ServerConfig, + mut config: crate::config::ServerConfig, project_id: String, ) -> Self { + config.semantic = config.semantic.resolved(); let cue_global_counts = loaded_global_counts .map(Arc::new) .unwrap_or_else(|| Arc::new(DashMap::with_hasher(RandomState::new()))); @@ -578,9 +817,23 @@ where next_memory_id: Arc::new(AtomicU32::new(next_memory_id.max(1))), master_key: None, tuning: Arc::new(TuningConfig::default()), + semantic_index: Arc::new(RwLock::new(SemanticIndex::new( + config.semantic.clone(), + ))), + semantic_encoder: Arc::new(RwLock::new(None)), + intent_classifier: Arc::new(RwLock::new(None)), + query_embedding_cache: Self::new_query_embedding_cache(&config.semantic), config: config.clone(), project_id: project_id.clone(), }; + if let Ok(mut index) = engine.semantic_index.write() { + index.rebuild(engine.memories.iter().filter_map(|entry| { + entry + .semantic_vector + .as_ref() + .map(|vector| (entry.id, vector.clone())) + })); + } engine.rebuild_source_order_index(); // Migration logic: Sync RAM/Disk state with config @@ -943,8 +1196,6 @@ where content: &str, metadata: Option<&HashMap>, cues: Vec, - cuepacks: &crate::cuepacks::CuePackRegistry, - cuepack_selection: Option<&[String]>, ) -> Vec { if TypeId::of::() != TypeId::of::() { return cues; @@ -952,13 +1203,7 @@ where let mut enriched = cues; let mut seen: HashSet = enriched.iter().map(|cue| cue.to_lowercase()).collect(); - for facet in crate::facets::extract_memory_facets_with_cuepacks( - content, - metadata, - &enriched, - cuepacks, - cuepack_selection, - ) { + for facet in crate::facets::extract_memory_facets(content, metadata, &enriched) { if seen.insert(facet.to_lowercase()) { enriched.push(facet); } @@ -974,57 +1219,117 @@ where stats: T, disable_temporal_chunking: bool, ) -> MemoryId { - self.add_memory_with_cuepacks( + self.add_memory_with_source_key_and_event_time( + content, + cues, + metadata, + stats, + disable_temporal_chunking, + None, + None, + ) + } + + pub fn add_memory_with_source_key( + &self, + content: String, + cues: Vec, + metadata: Option>, + stats: T, + disable_temporal_chunking: bool, + source_key: Option, + ) -> MemoryId { + self.add_memory_with_source_key_and_event_time( + content, + cues, + metadata, + stats, + disable_temporal_chunking, + source_key, + None, + ) + } + + pub fn add_memory_with_event_time( + &self, + content: String, + cues: Vec, + metadata: Option>, + stats: T, + disable_temporal_chunking: bool, + event_time: Option, + ) -> MemoryId { + self.add_memory_with_source_key_and_event_time( content, cues, metadata, stats, disable_temporal_chunking, - crate::cuepacks::default_registry(), None, + event_time, ) } - pub fn add_memory_with_cuepacks( + pub fn add_memory_with_event_time_and_vector( &self, content: String, cues: Vec, metadata: Option>, stats: T, disable_temporal_chunking: bool, - cuepacks: &crate::cuepacks::CuePackRegistry, - cuepack_selection: Option<&[String]>, + event_time: Option, + semantic_vector: Option>, ) -> MemoryId { - self.add_memory_with_cuepacks_and_source_key( + self.add_memory_with_source_key_and_event_time_and_vector( content, cues, metadata, stats, disable_temporal_chunking, - cuepacks, - cuepack_selection, None, + event_time, + semantic_vector, ) } - pub fn add_memory_with_cuepacks_and_source_key( + fn add_memory_with_source_key_and_event_time( &self, content: String, cues: Vec, metadata: Option>, stats: T, disable_temporal_chunking: bool, - cuepacks: &crate::cuepacks::CuePackRegistry, - cuepack_selection: Option<&[String]>, source_key: Option, + event_time: Option, ) -> MemoryId { - let cues = self.with_synchronous_facets( - &content, - metadata.as_ref(), + self.add_memory_with_source_key_and_event_time_and_vector( + content, cues, - cuepacks, - cuepack_selection, - ); + metadata, + stats, + disable_temporal_chunking, + source_key, + event_time, + None, + ) + } + + #[allow(clippy::too_many_arguments)] + fn add_memory_with_source_key_and_event_time_and_vector( + &self, + content: String, + cues: Vec, + metadata: Option>, + stats: T, + disable_temporal_chunking: bool, + source_key: Option, + event_time: Option, + semantic_vector: Option>, + ) -> MemoryId { + let semantic_vector = semantic_vector + .or_else(|| self.encode_semantic_text(&content)); + let semantic_vector = self.prepare_semantic_vector(semantic_vector); + let cues = self.with_synchronous_facets(&content, metadata.as_ref(), cues); // Create payload (Compressed or Encrypted) let payload = match Memory::::create_payload(&content, self.master_key.as_deref()) { @@ -1041,6 +1346,10 @@ where let mut memory = Memory::new(payload, metadata); memory.id = memory_id; memory.source_key = source_key.clone(); + memory.semantic_vector = semantic_vector.clone(); + if let Some(event_time) = event_time { + memory.created_at = event_time; + } // Store cues in memory memory.cues = cues.clone(); @@ -1080,7 +1389,10 @@ where 0.0 }; - if time_diff < 300.0 && overlap_ratio > 0.5 && !disable_temporal_chunking { + if (0.0..300.0).contains(&time_diff) + && overlap_ratio > 0.5 + && !disable_temporal_chunking + { let episode_cue = format!("episode:{}", last_id); memory.cues.push(episode_cue.clone()); } @@ -1099,6 +1411,17 @@ where self.source_key_to_id.insert(source_key, memory_id); } self.index_memory_cues(memory_id, &indexed_cues); + if let Some(vector) = semantic_vector.as_ref() { + if let Ok(mut index) = self.semantic_index.write() { + if let Err(error) = index.insert(memory_id, vector) { + tracing::debug!( + memory_id, + error = %error, + "Skipping invalid semantic vector" + ); + } + } + } if let Some((session, order)) = source_order_link { self.add_source_order_entry(session, order, memory_id); } @@ -1144,6 +1467,9 @@ where pub fn delete_memory(&self, memory_id: MemoryId) -> bool { if let Some((_, memory)) = self.memories.remove(&memory_id) { self.memory_count.fetch_sub(1, Ordering::Relaxed); + if let Ok(mut index) = self.semantic_index.write() { + index.remove(memory_id); + } self.remove_source_order_entry(memory_id); if let Some(source_key) = memory.source_key { self.source_key_to_id.remove(&source_key); @@ -1217,15 +1543,69 @@ where reinforce: bool, overwrite_cues: bool, ) -> MemoryId { - let cues = self.with_synchronous_facets( - &content, - metadata.as_ref(), + self.upsert_memory_with_source_key_and_options( + source_key, + content, + cues, + metadata, + stats, + reinforce, + overwrite_cues, + true, + None, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn upsert_memory_with_source_key_and_options( + &self, + source_key: String, + content: String, + cues: Vec, + metadata: Option>, + stats: Option, + reinforce: bool, + overwrite_cues: bool, + disable_temporal_chunking: bool, + event_time: Option, + ) -> MemoryId { + self.upsert_memory_with_source_key_and_options_and_vector( + source_key, + content, cues, - crate::cuepacks::default_registry(), + metadata, + stats, + reinforce, + overwrite_cues, + disable_temporal_chunking, + event_time, None, - ); + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn upsert_memory_with_source_key_and_options_and_vector( + &self, + source_key: String, + content: String, + cues: Vec, + metadata: Option>, + stats: Option, + reinforce: bool, + overwrite_cues: bool, + disable_temporal_chunking: bool, + event_time: Option, + semantic_vector: Option>, + ) -> MemoryId { + let semantic_vector = semantic_vector + .or_else(|| self.encode_semantic_text(&content)); + let semantic_vector = self.prepare_semantic_vector(semantic_vector); + let cues = self.with_synchronous_facets(&content, metadata.as_ref(), cues); if let Some(existing_id) = self.source_key_to_id.get(&source_key).map(|entry| *entry) { + if let Ok(mut index) = self.semantic_index.write() { + index.remove(existing_id); + } self.remove_source_order_entry(existing_id); { if let Some(mut memory) = self.memories.get_mut(&existing_id) { @@ -1260,6 +1640,10 @@ where if let Some(s) = stats.clone() { memory.stats = s; } + if let Some(event_time) = event_time { + memory.created_at = event_time; + } + memory.semantic_vector = semantic_vector.clone(); memory.source_key = Some(source_key.clone()); // We need to drop lock before attach/overwrite ops to avoid deadlocks // (though attach_cues re-acquires check, better safe) @@ -1295,6 +1679,17 @@ where self.add_source_order_entry(session, order, existing_id); } } + if let Some(vector) = semantic_vector.as_ref() { + if let Ok(mut index) = self.semantic_index.write() { + if let Err(error) = index.insert(existing_id, vector) { + tracing::debug!( + memory_id = existing_id, + error = %error, + "Skipping invalid semantic vector" + ); + } + } + } return existing_id; } @@ -1313,18 +1708,74 @@ where let mut memory = Memory::new(payload, metadata); memory.id = memory_id; memory.source_key = Some(source_key.clone()); + memory.semantic_vector = semantic_vector.clone(); + if let Some(event_time) = event_time { + memory.created_at = event_time; + } memory.cues = cues.clone(); + + if self.config.server.store_content_on_disk { + let path = self + .get_disk_content_dir() + .join(format!("{}.bin", memory.id)); + if let Err(error) = std::fs::write(&path, &memory.content) { + tracing::error!("Failed to write memory content to disk: {}", error); + } else { + memory.content = Vec::new(); + memory.disk_backed = true; + } + } + + let project_id = memory + .metadata + .get("project_id") + .and_then(|value| value.as_str()) + .unwrap_or("default") + .to_string(); + if let Some(last_event) = self.last_events.get(&project_id) { + let (last_id, last_time, last_cues) = last_event.clone(); + let time_diff = memory.created_at - last_time; + let overlap = memory.cues.iter().filter(|cue| last_cues.contains(cue)).count(); + let overlap_ratio = if memory.cues.is_empty() { + 0.0 + } else { + (overlap as f64) / (memory.cues.len() as f64) + }; + + if (0.0..300.0).contains(&time_diff) + && overlap_ratio > 0.5 + && !disable_temporal_chunking + { + memory.cues.push(format!("episode:{}", last_id)); + } + } memory.scoring_features = compute_memory_scoring_features(&memory.cues); if let Some(s) = stats { memory.stats = s; } + let indexed_cues = memory.cues.clone(); let source_order_link = Self::source_order_link_for_memory(&memory); + self.last_events.insert( + project_id, + (memory_id, memory.created_at, memory.cues.clone()), + ); if self.memories.insert(memory_id, memory).is_none() { self.memory_count.fetch_add(1, Ordering::Relaxed); } self.source_key_to_id.insert(source_key, memory_id); - self.index_memory_cues(memory_id, &cues); + self.index_memory_cues(memory_id, &indexed_cues); + if let Some(vector) = semantic_vector.as_ref() { + if let Ok(mut index) = self.semantic_index.write() { + if let Err(error) = index.insert(memory_id, vector) { + tracing::debug!( + memory_id, + error = %error, + "Skipping invalid semantic vector" + ); + } + } + } if let Some((session, order)) = source_order_link { self.add_source_order_entry(session, order, memory_id); } @@ -1654,6 +2105,7 @@ where disable_salience_bias, heatmap, mandatory_cues, + None, false, ) .0 @@ -1681,11 +2133,12 @@ where disable_salience_bias, heatmap, mandatory_cues, + None, true, ) } - fn recall_weighted_profiled( + pub fn recall_weighted_with_query_embedding( &self, query_cues: Vec<(String, f64)>, limit: usize, @@ -1696,57 +2149,309 @@ where disable_salience_bias: bool, heatmap: Option<&HashMap>, mandatory_cues: Option<&Vec>, - collect_detailed_timing: bool, - ) -> (Vec, RecallTimingBreakdown) { - let total_start = Instant::now(); - let mut timing = RecallTimingBreakdown::default(); - if query_cues.is_empty() { - timing.total_ms = total_start.elapsed().as_secs_f64() * 1000.0; - return (Vec::new(), timing); - } - - // Normalize primary cues - let phase_start = Instant::now(); - let active_cues: Vec<(String, f64)> = query_cues - .iter() - .map(|(c, w)| (c.to_lowercase().trim().to_string(), *w)) - .filter(|(c, _)| !c.is_empty() && self.cue_index.contains_key(c)) - .collect(); - timing.normalize_filter_ms = phase_start.elapsed().as_secs_f64() * 1000.0; - timing.initial_active_cue_count = active_cues.len(); - - if active_cues.is_empty() { - timing.total_ms = total_start.elapsed().as_secs_f64() * 1000.0; - return (Vec::new(), timing); - } - - timing.active_cue_count = active_cues.len(); - - // Consolidated search using Selective Set Intersection - let phase_start = Instant::now(); - let (mut results, search_timing) = self.consolidated_search( - &active_cues, + query_embedding: Option<&[f32]>, + ) -> Vec { + self.recall_weighted_profiled( + query_cues, limit, + auto_reinforce, + min_intersection, + expansion_depth, explain, disable_salience_bias, heatmap, mandatory_cues, - collect_detailed_timing, - ); - timing.consolidated_search_ms = phase_start.elapsed().as_secs_f64() * 1000.0; - timing.candidate_generation_ms = search_timing.candidate_generation_ms; - timing.candidate_scoring_ms = search_timing.candidate_scoring_ms; - timing.scoring_filter_ms = search_timing.scoring_filter_ms; - timing.scoring_position_ms = search_timing.scoring_position_ms; - timing.scoring_salience_ms = search_timing.scoring_salience_ms; - timing.scoring_structured_ms = search_timing.scoring_structured_ms; - timing.scoring_finalize_ms = search_timing.scoring_finalize_ms; - timing.candidate_count = search_timing.candidate_count; - timing.scored_candidate_count = search_timing.scored_candidate_count; - timing.cue_count_with_postings = search_timing.cue_count_with_postings; - timing.scanned_posting_count = search_timing.scanned_posting_count; - timing.adaptive_scan_limit = search_timing.adaptive_scan_limit; - timing.max_posting_len = search_timing.max_posting_len; + query_embedding, + false, + ) + .0 + } + + /// Runs semantic scoring only over the lexical candidates produced by the + /// same request. This is the bounded hybrid path: it does not query the + /// semantic index or add semantic-only candidates. + pub fn recall_weighted_with_query_embedding_rerank_only( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + ) -> Vec { + self.recall_weighted_with_query_embedding_rerank_only_and_intent( + query_cues, + limit, + auto_reinforce, + min_intersection, + expansion_depth, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + query_embedding, + None, + ) + } + + pub fn recall_weighted_with_query_embedding_rerank_only_and_intent( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + query_intent: Option<&IntentClassification>, + ) -> Vec { + self.recall_weighted_profiled_with_options( + query_cues, + limit, + auto_reinforce, + min_intersection, + expansion_depth, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + query_embedding, + query_intent, + false, + true, + ) + .0 + } + + pub fn recall_weighted_with_query_embedding_with_timing( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + ) -> (Vec, RecallTimingBreakdown) { + self.recall_weighted_profiled( + query_cues, + limit, + auto_reinforce, + min_intersection, + expansion_depth, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + query_embedding, + true, + ) + } + + /// Timing variant of the bounded hybrid path. Semantic scoring is + /// restricted to the lexical result set and cannot introduce new IDs. + pub fn recall_weighted_with_query_embedding_rerank_only_with_timing( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + ) -> (Vec, RecallTimingBreakdown) { + self.recall_weighted_with_query_embedding_rerank_only_with_intent_with_timing( + query_cues, + limit, + auto_reinforce, + min_intersection, + expansion_depth, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + query_embedding, + None, + ) + } + + pub fn recall_weighted_with_query_embedding_rerank_only_with_intent_with_timing( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + query_intent: Option<&IntentClassification>, + ) -> (Vec, RecallTimingBreakdown) { + self.recall_weighted_profiled_with_options( + query_cues, + limit, + auto_reinforce, + min_intersection, + expansion_depth, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + query_embedding, + query_intent, + true, + true, + ) + } + + fn recall_weighted_profiled( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + collect_detailed_timing: bool, + ) -> (Vec, RecallTimingBreakdown) { + self.recall_weighted_profiled_with_options( + query_cues, + limit, + auto_reinforce, + min_intersection, + expansion_depth, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + query_embedding, + None, + collect_detailed_timing, + false, + ) + } + + fn recall_weighted_profiled_with_options( + &self, + query_cues: Vec<(String, f64)>, + limit: usize, + auto_reinforce: bool, + min_intersection: Option, + expansion_depth: usize, + explain: bool, + disable_salience_bias: bool, + heatmap: Option<&HashMap>, + mandatory_cues: Option<&Vec>, + query_embedding: Option<&[f32]>, + query_intent: Option<&IntentClassification>, + collect_detailed_timing: bool, + semantic_rerank_only: bool, + ) -> (Vec, RecallTimingBreakdown) { + let total_start = Instant::now(); + let mut timing = RecallTimingBreakdown::default(); + if query_cues.is_empty() && query_embedding.is_none() { + timing.total_ms = total_start.elapsed().as_secs_f64() * 1000.0; + return (Vec::new(), timing); + } + + // Normalize primary cues + let phase_start = Instant::now(); + let active_cues: Vec<(String, f64)> = query_cues + .iter() + .map(|(c, w)| (c.to_lowercase().trim().to_string(), *w)) + .filter(|(c, _)| !c.is_empty() && self.cue_index.contains_key(c)) + .collect(); + timing.normalize_filter_ms = phase_start.elapsed().as_secs_f64() * 1000.0; + timing.initial_active_cue_count = active_cues.len(); + + if active_cues.is_empty() && query_embedding.is_none() { + timing.total_ms = total_start.elapsed().as_secs_f64() * 1000.0; + return (Vec::new(), timing); + } + + timing.active_cue_count = active_cues.len(); + + // Consolidated search using Selective Set Intersection + let phase_start = Instant::now(); + let (mut results, search_timing) = self.consolidated_search( + &active_cues, + limit, + explain, + disable_salience_bias, + heatmap, + mandatory_cues, + collect_detailed_timing, + ); + timing.consolidated_search_ms = phase_start.elapsed().as_secs_f64() * 1000.0; + timing.candidate_generation_ms = search_timing.candidate_generation_ms; + timing.candidate_scoring_ms = search_timing.candidate_scoring_ms; + timing.scoring_filter_ms = search_timing.scoring_filter_ms; + timing.scoring_position_ms = search_timing.scoring_position_ms; + timing.scoring_salience_ms = search_timing.scoring_salience_ms; + timing.scoring_structured_ms = search_timing.scoring_structured_ms; + timing.scoring_finalize_ms = search_timing.scoring_finalize_ms; + timing.candidate_count = search_timing.candidate_count; + timing.scored_candidate_count = search_timing.scored_candidate_count; + timing.cue_count_with_postings = search_timing.cue_count_with_postings; + timing.scanned_posting_count = search_timing.scanned_posting_count; + timing.adaptive_scan_limit = search_timing.adaptive_scan_limit; + timing.max_posting_len = search_timing.max_posting_len; + + if semantic_rerank_only { + let config = self.config.semantic.resolved(); + let rerank_limit = config.semantic_rerank_candidate_limit.max(limit); + results.sort_unstable_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + results.truncate(rerank_limit); + timing.semantic_rerank_candidate_limit = rerank_limit; + timing.semantic_rerank_candidate_count = results.len(); + } + + // Apply intent before semantic fusion so the semantic pass operates on + // the intent-aware lexical slate. The intent bonus is kept separately + // and reintroduced by rerank_existing_semantic_candidates after + // semantic normalization, so normalization cannot wash it out. + if let Some(query_intent) = query_intent { + self.apply_intent_reranker(&mut results, query_intent); + } + + if let Some(query_embedding) = query_embedding { + let semantic_start = Instant::now(); + let (new_candidates, semantic_candidates) = if semantic_rerank_only { + ( + 0, + self.rerank_existing_semantic_candidates(&mut results, query_embedding), + ) + } else { + self.merge_semantic_candidates(&mut results, query_embedding, limit) + }; + timing.candidate_generation_ms += semantic_start.elapsed().as_secs_f64() * 1000.0; + timing.candidate_count += new_candidates; + timing.scored_candidate_count += semantic_candidates; + self.apply_semantic_reranker(&mut results, query_embedding); + } // Filter by minimum intersection if specified (on primary cues only?) // For now, simple retention. @@ -1792,8 +2497,11 @@ where "recency_score": candidate.recency_score, "reinforcement_score": candidate.reinforcement_score, "salience_score": candidate.salience_score, + "semantic_similarity": candidate.semantic_similarity, "rerank_bonus": candidate.rerank_bonus, "generic_penalty": candidate.generic_penalty, + "intent_compatibility": candidate.intent_compatibility, + "intent_rerank_bonus": candidate.intent_rerank_bonus, })) } else { None @@ -1922,6 +2630,270 @@ where (final_results, timing) } + fn rerank_existing_semantic_candidates( + &self, + results: &mut [ScoredMemoryCandidate], + query_embedding: &[f32], + ) -> usize { + let config = self.config.semantic.resolved(); + let normalized_query = match StoredSemanticVector::normalized_query(query_embedding) { + Ok(query) => query, + Err(error) => { + tracing::debug!(error = %error, "Semantic reranking skipped"); + return 0; + } + }; + + let mut scored = Vec::new(); + for (index, candidate) in results.iter_mut().enumerate() { + let Some(memory) = self.memories.get(&candidate.memory_id) else { + continue; + }; + let Some(vector) = memory.semantic_vector.as_ref() else { + continue; + }; + let Ok(similarity) = vector.cosine_similarity_normalized(&normalized_query) else { + continue; + }; + + candidate.semantic_similarity = similarity; + scored.push((index, similarity)); + } + + if scored.len() < 2 { + return scored.len(); + } + + // Intent is applied before this method, so recover the original + // lexical score from the separately tracked intent delta. Semantic + // normalization must compare lexical evidence on its own, then add + // the strong intent prior back to the fused result. + let lexical_min = results + .iter() + .map(|candidate| candidate.score - candidate.intent_rerank_bonus) + .fold(f64::INFINITY, f64::min); + let lexical_max = results + .iter() + .map(|candidate| candidate.score - candidate.intent_rerank_bonus) + .fold(f64::NEG_INFINITY, f64::max); + let lexical_range = lexical_max - lexical_min; + let semantic_min = scored + .iter() + .map(|(_, similarity)| *similarity) + .fold(f32::INFINITY, f32::min); + let semantic_max = scored + .iter() + .map(|(_, similarity)| *similarity) + .fold(f32::NEG_INFINITY, f32::max); + let semantic_range = semantic_max - semantic_min; + let semantic_weight = config.semantic_rerank_weight.clamp(0.0, 1.0); + + if !lexical_range.is_finite() || lexical_range <= f64::EPSILON + || !semantic_range.is_finite() + || semantic_range <= f32::EPSILON + || semantic_weight <= f64::EPSILON + { + return scored.len(); + } + + let lexical_weight = 1.0 - semantic_weight; + for (index, similarity) in scored.iter().copied() { + let candidate = &mut results[index]; + let lexical_score = candidate.score - candidate.intent_rerank_bonus; + let lexical_quality = ((lexical_score - lexical_min) / lexical_range).clamp(0.0, 1.0); + let semantic_quality = + ((similarity - semantic_min) / semantic_range).clamp(0.0, 1.0) as f64; + let fused_quality = + lexical_quality * lexical_weight + semantic_quality * semantic_weight; + let fused_score = lexical_min + fused_quality * lexical_range; + let final_score = fused_score + candidate.intent_rerank_bonus; + let delta = final_score - candidate.score; + candidate.score = final_score; + candidate.rerank_bonus += delta; + } + + scored.len() + } + + fn merge_semantic_candidates( + &self, + results: &mut Vec, + query_embedding: &[f32], + limit: usize, + ) -> (usize, usize) { + let config = self.config.semantic.resolved(); + let candidate_limit = config.candidate_limit.max(limit); + let normalized_query = match StoredSemanticVector::normalized_query(query_embedding) { + Ok(query) => query, + Err(error) => { + tracing::debug!(error = %error, "Semantic query skipped"); + return (0, 0); + } + }; + let candidate_ids = match self.semantic_index.read() { + Ok(index) => match index.query_candidate_ids(query_embedding, candidate_limit) { + Ok(candidate_ids) => candidate_ids, + Err(error) => { + tracing::debug!(error = %error, "Semantic query skipped"); + return (0, 0); + } + }, + Err(_) => return (0, 0), + }; + let mut semantic_candidates = candidate_ids + .into_iter() + .filter_map(|memory_id| { + let memory = self.memories.get(&memory_id)?; + let vector = memory.semantic_vector.as_ref()?; + let similarity = vector.cosine_similarity_normalized(&normalized_query).ok()?; + Some((memory_id, similarity)) + }) + .collect::>(); + semantic_candidates.sort_unstable_by(|a, b| { + b.1.partial_cmp(&a.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(&b.0)) + }); + semantic_candidates.truncate(candidate_limit); + + let mut existing = results + .iter() + .map(|candidate| candidate.memory_id) + .collect::>(); + let mut new_count = 0; + + for (memory_id, similarity) in semantic_candidates.iter().copied() { + if let Some(candidate) = results + .iter_mut() + .find(|candidate| candidate.memory_id == memory_id) + { + candidate.semantic_similarity = similarity; + candidate.score += + similarity as f64 * config.semantic_score_multiplier; + continue; + } + + if !existing.insert(memory_id) { + continue; + } + let Some(memory) = self.memories.get(&memory_id) else { + continue; + }; + let reinforcement_score = if memory.stats.get_reinforcement_count() > 0 { + (memory.stats.get_reinforcement_count() as f64).log10() + } else { + 0.0 + }; + results.push(ScoredMemoryCandidate { + memory_id, + score: similarity as f64 * config.semantic_score_multiplier, + match_integrity: similarity.max(0.0) as f64, + intersection_count: 0, + recency_score: 0.0, + reinforcement_score, + salience_score: 0.0, + created_at: memory.created_at, + intersection_weighted: 0.0, + match_count: 0.0, + rerank_bonus: 0.0, + generic_penalty: 1.0, + semantic_similarity: similarity, + intent_compatibility: 0.0, + intent_rerank_bonus: 0.0, + }); + new_count += 1; + } + + (new_count, semantic_candidates.len()) + } + + fn apply_semantic_reranker( + &self, + results: &mut [ScoredMemoryCandidate], + _query_embedding: &[f32], + ) { + let config = &self.config.semantic; + if !config.reranker_enabled || config.reranker_weights.is_empty() { + return; + } + let model = LinearReranker::from_config(config); + for candidate in results { + let features = [ + (candidate.score / self.tuning.intersection_score_multiplier) + .clamp(-10.0, 10.0) as f32, + candidate.semantic_similarity, + candidate.match_integrity.clamp(0.0, 1.0) as f32, + (candidate.intersection_count as f32 / 8.0).min(1.0), + candidate.recency_score.clamp(0.0, 1.0) as f32, + (candidate.salience_score / 10.0).clamp(0.0, 1.0) as f32, + ]; + let delta = model.score(&features) as f64 * config.reranker_scale; + candidate.rerank_bonus += delta; + candidate.score += delta; + } + } + + fn apply_intent_reranker( + &self, + results: &mut [ScoredMemoryCandidate], + query_intent: &IntentClassification, + ) { + let config = self.config.semantic.resolved(); + if !config.intent_rerank_enabled + || !query_intent.is_recall_intent() + || results.len() < 2 + { + return; + } + let lexical_min = results + .iter() + .map(|candidate| candidate.score - candidate.intent_rerank_bonus) + .fold(f64::INFINITY, f64::min); + let lexical_max = results + .iter() + .map(|candidate| candidate.score - candidate.intent_rerank_bonus) + .fold(f64::NEG_INFINITY, f64::max); + let lexical_range = lexical_max - lexical_min; + if !lexical_range.is_finite() || lexical_range <= f64::EPSILON { + return; + } + + let query_weight = f64::from(query_intent.confidence_weight); + for candidate in results { + let Some(memory) = self.memories.get(&candidate.memory_id) else { + continue; + }; + let Some(memory_intent) = memory.intent_classification.as_ref() else { + continue; + }; + if memory_intent.taxonomy_version != crate::intent::INTENT_TAXONOMY_VERSION + || memory_intent.model_version != config.model_version + { + continue; + } + let compatibility = intent_compatibility(query_intent, memory_intent); + let memory_weight = f64::from(memory_intent.confidence_weight); + let positive = compatibility + * query_weight + * memory_weight + * config.intent_rerank_weight; + let no_recall_penalty = if memory_intent.memory_eligible { + 0.0 + } else { + // Suppression should be strong only when both sides are + // decisive. A low-margin query must not apply a categorical + // penalty to every action/chitchat memory. + query_weight * memory_weight * config.intent_no_recall_penalty + }; + let delta = ((positive - no_recall_penalty) * lexical_range) + .clamp(-config.intent_rerank_max_delta, config.intent_rerank_max_delta); + candidate.intent_compatibility = compatibility; + candidate.intent_rerank_bonus = delta; + candidate.score += delta; + candidate.rerank_bonus += delta; + } + } + fn consolidated_search( &self, query_cues: &[(String, f64)], @@ -2399,6 +3371,9 @@ where match_count, rerank_bonus, generic_penalty, + semantic_similarity: 0.0, + intent_compatibility: 0.0, + intent_rerank_bonus: 0.0, }); if let Some(start) = detail_start.as_mut() { timing.finalize_ms += start.elapsed().as_secs_f64() * 1000.0; @@ -2872,3 +3847,626 @@ impl CueMapEngine { trending } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::semantic::SemanticConfig; + use std::collections::BTreeMap; + + fn classification( + primary_intent: &str, + confidence_weight: f32, + memory_eligible: bool, + ) -> IntentClassification { + let mut scores = BTreeMap::new(); + for label in crate::intent::INTENT_LABELS { + scores.insert(label.to_string(), if label == primary_intent { 1.0 } else { 0.0 }); + } + IntentClassification { + primary_intent: primary_intent.to_string(), + scores, + top_intents: vec![primary_intent.to_string()], + top_score: 1.0, + margin: 1.0, + confidence_weight, + recall_eligible: memory_eligible, + recall_action: if memory_eligible { + "recall".to_string() + } else { + "no_recall".to_string() + }, + memory_eligible, + model_version: SemanticConfig::default().model_version, + taxonomy_version: INTENT_TAXONOMY_VERSION.to_string(), + } + } + + fn candidate(memory_id: MemoryId, score: f64) -> ScoredMemoryCandidate { + ScoredMemoryCandidate { + memory_id, + score, + match_integrity: 1.0, + intersection_count: 1, + recency_score: 0.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + intersection_weighted: score, + match_count: 1.0, + rerank_bonus: 0.0, + generic_penalty: 1.0, + semantic_similarity: 0.0, + intent_compatibility: 0.0, + intent_rerank_bonus: 0.0, + } + } + + #[test] + fn intent_rerank_penalty_is_confidence_scaled_and_bounded() { + let mut engine = CueMapEngine::::new(); + let mut config = SemanticConfig::default(); + config.intent_rerank_weight = 1.0; + config.intent_no_recall_penalty = 1.0; + config.intent_rerank_max_delta = 32.0; + engine.set_semantic_config(config); + + let memory_id = engine.add_memory( + "Run the deployment command".to_string(), + vec!["deployment".to_string()], + None, + MainStats::default(), + true, + ); + engine.attach_intent_classification( + memory_id, + classification("action_or_command", 1.0, false), + ); + + let query = classification("event_or_plan", 1.0, true); + let mut results = vec![candidate(memory_id, 1000.0), candidate(INVALID_MEMORY_ID, 0.0)]; + engine.apply_intent_reranker(&mut results, &query); + assert_eq!(results[0].intent_rerank_bonus, -32.0); + + let uncertain_query = classification("event_or_plan", 0.1, true); + let mut results = vec![candidate(memory_id, 100.0), candidate(INVALID_MEMORY_ID, 0.0)]; + engine.apply_intent_reranker(&mut results, &uncertain_query); + assert!(results[0].intent_rerank_bonus > -32.0); + } + + #[test] + fn intent_bonus_survives_semantic_normalization() { + let mut engine = CueMapEngine::::new(); + let mut config = SemanticConfig::default(); + config.enabled = true; + config.dimensions = 3; + config.storage = crate::semantic::SemanticStorage::F32; + config.index = crate::semantic::SemanticIndexMode::Exact; + config.semantic_rerank_weight = 0.60; + config.intent_rerank_weight = 1.0; + config.intent_rerank_max_delta = 64.0; + engine.set_semantic_config(config); + + let matching_id = engine.add_memory_with_event_time_and_vector( + "A preference memory".to_string(), + vec!["preference".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![1.0, 0.0, 0.0]), + ); + let unrelated_id = engine.add_memory_with_event_time_and_vector( + "An unrelated event memory".to_string(), + vec!["event".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![0.0, 1.0, 0.0]), + ); + engine.attach_intent_classification( + matching_id, + classification("preference", 1.0, true), + ); + engine.attach_intent_classification( + unrelated_id, + classification("event_or_plan", 1.0, true), + ); + + let query = classification("preference", 1.0, true); + let mut results = vec![ + candidate(matching_id, 1000.0), + candidate(unrelated_id, 0.0), + ]; + engine.apply_intent_reranker(&mut results, &query); + assert_eq!(results[0].intent_rerank_bonus, 64.0); + + let semantic_count = + engine.rerank_existing_semantic_candidates(&mut results, &[1.0, 0.0, 0.0]); + + assert_eq!(semantic_count, 2); + assert_eq!(results[0].score, 1064.0); + assert_eq!(results[0].intent_rerank_bonus, 64.0); + assert_eq!(results[0].semantic_similarity, 1.0); + } + + #[test] + fn structured_helpers_cover_facet_families_and_fallbacks() { + let generated = [ + "source_role:user", "source_channel:chat", "source_type:note", + "source_session:s1", "source_time:morning", "source_date:2024", + "source_week:1", "source_month:1", "has:age", "completion_count:2", + "completed_action:ship", "instruction:do", "instruction_trigger:now", + "instruction_action:run", "preference:tea", "preference_value:green", + "preference_topic:drink", "preference_contrast:coffee", "temporal:today", + "co_residence:home", "entity:person", "person_title:dr", "person_role_phrase:lead", + "person_ref:kaan", "quantity_object:items", "quantity_unit:kg", + "quantity_unit_object:bag", "quantity_count:2", "inventory_object:book", + "inventory_count:3", "purchase:item", "companion:friend", "age:42", + "education:college", "travel:trip", "media:book", "reading:novel", + "transport_mode:train", "transport_event:arrival", "activity_domain:work", + "topic:rust", "attribute:fast", "family_relation:sibling", "family_scope:home", + "family_count:2", "sibling_kind:brother", "type:preference", + ]; + for cue in generated { + assert!(is_generated_memory_facet(cue), "{cue} should be generated"); + assert!(cue_structured_family_mask(cue) != 0); + } + for cue in ["type:preference", "media:book", "travel:trip", "topic:rust", "purchase:item"] { + assert!(is_semantic_generated_facet(cue)); + } + assert!(!is_generated_memory_facet("plain lexical cue")); + assert!(!is_semantic_generated_facet("plain lexical cue")); + assert_eq!(cue_structured_family_mask("plain lexical cue"), 0); + assert_eq!(rerank_multiplier_for_prefix("source_time"), 12.0); + assert_eq!(rerank_multiplier_for_prefix("unknown"), 4.0); + + let cues = generated.iter().map(|cue| cue.to_string()).collect::>(); + let features = compute_memory_scoring_features(&cues); + assert_eq!(features.version, MEMORY_SCORING_FEATURES_VERSION); + assert!(features.has_summary_type == false); + assert_eq!(features.scored_cue_len, 1); + assert!(features.structured_family_mask & FAMILY_PERSON != 0); + assert!(features.structured_family_mask & FAMILY_SOURCE_TIME != 0); + + let profile_cues = [ + ("plain".to_string(), 1.0), + ("person_role_phrase:lead".to_string(), 1.0), + ("quantity_object:items".to_string(), 1.0), + ("inventory_object:book".to_string(), 1.0), + ("travel:trip".to_string(), 1.0), + ("age:42".to_string(), 1.0), + ("education:college".to_string(), 1.0), + ("family_count:2".to_string(), 3.0), + ("source_role:user".to_string(), 3.0), + ("source_time:morning".to_string(), 1.0), + ("type:update".to_string(), 1.0), + ("type:preference".to_string(), 3.0), + ]; + let profile = build_query_scoring_profile(&profile_cues); + assert!(profile.lexical_query_seen); + assert!(profile.strong_lexical_query_seen); + assert!(profile.strong_structured_query_seen); + assert!(profile.structured_query_seen); + assert!(profile.person_structured_query_seen); + assert!(profile.quantity_structured_query_seen); + assert!(profile.inventory_structured_query_seen); + assert!(profile.travel_structured_query_seen); + assert!(profile.age_structured_query_seen); + assert!(profile.education_structured_query_seen); + assert!(profile.family_structured_query_seen); + assert!(profile.family_count_structured_query_seen); + assert!(profile.source_role_structured_query_seen); + assert!(profile.source_time_structured_query_seen); + assert!(profile.update_structured_query_seen); + + let mut stale = Memory::::new(Vec::new(), None); + stale.cues = vec!["plain".to_string()]; + let fresh = memory_scoring_features(&stale); + assert_eq!(fresh.scored_cue_len, 1); + stale.scoring_features.version = MEMORY_SCORING_FEATURES_VERSION; + stale.scoring_features.scored_cue_len = 9; + assert_eq!(memory_scoring_features(&stale).scored_cue_len, 9); + } + + #[test] + fn metadata_and_source_order_parsers_cover_types_and_invalid_values() { + let mut metadata = HashMap::new(); + metadata.insert("number".to_string(), serde_json::json!(7)); + metadata.insert("flag".to_string(), serde_json::json!(true)); + metadata.insert("blank".to_string(), serde_json::json!(" ")); + assert_eq!(CueMapEngine::::metadata_string_value(&metadata, &["blank", "number"]), Some("7".to_string())); + assert_eq!(CueMapEngine::::metadata_string_value(&metadata, &["flag"]), Some("true".to_string())); + assert_eq!(CueMapEngine::::metadata_string_value(&metadata, &["missing"]), None); + + assert_eq!(CueMapEngine::::normalize_source_order_value(" Hello, World! "), Some("hello_world".to_string())); + assert_eq!(CueMapEngine::::normalize_source_order_value("---"), None); + metadata.insert("session_id".to_string(), serde_json::json!(" Session A ")); + metadata.insert("source_turn_index".to_string(), serde_json::json!("12")); + assert_eq!(CueMapEngine::::source_order_session_from(&metadata, &[]), Some("session_a".to_string())); + assert_eq!(CueMapEngine::::source_order_value_from(&metadata, &[]), Some(12)); + + let mut numeric = HashMap::new(); + numeric.insert("source_order".to_string(), serde_json::json!(9)); + assert_eq!(CueMapEngine::::source_order_value_from(&numeric, &[]), Some(9)); + let cues = vec!["source_session:Fallback Session".to_string(), "turn_index:4".to_string()]; + assert_eq!(CueMapEngine::::source_order_session_from(&HashMap::new(), &cues), Some("fallback_session".to_string())); + assert_eq!(CueMapEngine::::source_order_value_from(&HashMap::new(), &cues), Some(4)); + assert_eq!(CueMapEngine::::source_order_value_from(&HashMap::new(), &["turn_index:nope".to_string()]), None); + + let engine = CueMapEngine::::new(); + assert!(engine.ordered_entries_for_session("!!!", 10).is_empty()); + assert!(engine.source_order_window("!!!", 0, 2).is_empty()); + } + + #[test] + fn constructors_semantic_setup_and_encoding_error_paths_are_safe() { + struct UnitEncoder; + impl SemanticEncoder for UnitEncoder { + fn dimensions(&self) -> usize { 3 } + fn encode(&self, text: &str) -> Result, String> { + if text == "bad" { return Err("bad input".to_string()); } + Ok(vec![1.0, 0.0, 0.0]) + } + } + let key = crate::crypto::EncryptionKey::new(vec![7; 32]); + let keyed = CueMapEngine::::with_key(Some(key.clone())); + assert_eq!(keyed.get_master_key().unwrap().as_bytes(), &[7; 32]); + let unkeyed = CueMapEngine::::with_key(None); + assert!(unkeyed.get_master_key().is_none()); + + let mut engine = CueMapEngine::::new(); + assert!(engine.encode_semantic_text("disabled").is_none()); + let mut config = SemanticConfig::default(); + config.enabled = true; + config.encoder_enabled = true; + config.dimensions = 3; + engine.set_semantic_config(config); + assert!(engine.encode_semantic_text("no encoder").is_none()); + assert!(matches!(engine.classify_intent("anything", IntentTarget::Memory), Err(error) if error == "intent classifier unavailable")); + engine.set_semantic_encoder(Some(Arc::new(UnitEncoder))); + assert!(engine.encode_semantic_text("works").is_some()); + assert!(engine.encode_semantic_text("bad").is_none()); + let _ = engine.classify_intent("anything", IntentTarget::Memory); + let _ = engine.classify_intent_with_embedding("anything", IntentTarget::Memory, &[1.0, 0.0, 0.0]); + #[cfg(feature = "semantic-encoder")] + assert!(engine.configure_semantic_encoder().is_ok()); + #[cfg(not(feature = "semantic-encoder"))] + assert!(engine.configure_semantic_encoder().is_err()); + + let mut bad = SemanticConfig::default(); + bad.enabled = true; + bad.dimensions = 3; + bad.storage = crate::semantic::SemanticStorage::F32; + engine.set_semantic_config(bad); + let id = engine.add_memory_with_event_time_and_vector("bad vector".to_string(), vec!["v".to_string()], None, MainStats::default(), true, Some(42.0), Some(vec![1.0, 2.0])); + assert_ne!(id, INVALID_MEMORY_ID); + assert!(engine.get_memory(id).unwrap().semantic_vector.is_none()); + assert_eq!(engine.semantic_index_stats(), (true, 0, None)); + } + + #[test] + fn recall_edge_paths_and_maintenance_helpers_are_covered() { + let engine = CueMapEngine::::new(); + assert!(engine.recall(Vec::new(), 10, false, None).is_empty()); + assert!(engine.recall_intersection(Vec::new(), 10).is_empty()); + assert!(engine.recall_intersection(vec![("missing".to_string(), 1.0)], 10).is_empty()); + assert!(engine.recall_fast(Vec::new(), 10).is_empty()); + assert!(engine.recall_weighted(Vec::new(), 10, false, None, 1, false, false, None, None).is_empty()); + assert!(engine.recall_weighted_with_query_embedding(Vec::new(), 10, false, None, 1, false, false, None, None, Some(&[1.0, 0.0])).is_empty()); + + let id = engine.add_memory("alpha beta".to_string(), vec!["alpha".to_string(), "type:update".to_string()], None, MainStats::default(), true); + assert_eq!(engine.get_cue_frequency(" ALPHA "), 1); + assert!(engine.recall_intersection(vec![("alpha".to_string(), 2.0), ("missing".to_string(), 1.0)], 2).len() == 1); + assert_eq!(engine.recall_fast(vec!["alpha".to_string(), "".to_string()], 1).len(), 1); + let mandatory = vec!["not-present".to_string()]; + assert!(engine.recall_weighted(vec![("alpha".to_string(), 1.0)], 10, false, None, 1, true, false, None, Some(&mandatory)).is_empty()); + let mut heatmap = HashMap::new(); + heatmap.insert("alpha".to_string(), 2.0); + let (results, timing) = engine.recall_weighted_with_timing(vec![("alpha".to_string(), 1.0)], 2, true, Some(1), 1, true, false, Some(&heatmap), None); + assert_eq!(results.len(), 1); + assert!(timing.total_ms >= 0.0); + assert!(engine.attach_cues(INVALID_MEMORY_ID, vec!["x".to_string()]) == false); + engine.remove_cues_from_index(id, &[" alpha ".to_string(), "missing".to_string(), "".to_string()]); + assert_eq!(engine.get_cue_frequency("alpha"), 0); + } + + #[test] + fn mainstats_and_lexicon_specialized_paths_are_exercised() { + let engine = CueMapEngine::::new(); + let id = engine.add_memory("hot memory".to_string(), vec!["hot".to_string()], None, MainStats { intrinsic_salience: 0.2, dynamic_salience: 0.0, last_boosted_at: 0, reinforcement_count: 0 }, true); + engine.reinforce_dynamic(id, 4.0); + let mut heatmap = HashMap::new(); + heatmap.insert("hot".to_string(), 2.5); + let scored = engine.score_with_decay_and_market(vec![id, INVALID_MEMORY_ID], &heatmap); + assert_eq!(scored.len(), 1); + assert!(scored[0].score > 2.5); + assert!(scored[0].explain.is_some()); + engine.decay_salience(0.5); + assert!(!engine.get_trending_cues(10).is_empty()); + assert_eq!(engine.prune_low_salience(100.0), 1); + + let dict = CueMapEngine::::new(); + let dict_id = dict.add_memory("word".to_string(), vec!["word".to_string()], None, LexiconStats::default(), true); + dict.reinforce_tiered(dict_id, 3); + let trending = dict.get_trending_items(10); + assert_eq!(trending, vec![(dict_id, 3.0)]); + assert!(dict.get_trending_items(0).is_empty()); + } + + #[test] + fn from_state_rebuilds_indexes_and_disk_migration_roundtrips() { + let source = CueMapEngine::::new(); + let id = source.add_memory("persisted".to_string(), vec!["persist".to_string()], None, MainStats::default(), true); + let mut config = crate::config::ServerConfig::default(); + config.server.data_dir = std::env::temp_dir().join(format!("cuemap-engine-{}", std::process::id())).to_string_lossy().to_string(); + config.server.store_content_on_disk = true; + let disk = CueMapEngine::from_state((**source.get_memories()).clone(), (**source.get_source_key_to_id()).clone(), (**source.get_cue_index()).clone(), source.next_memory_id(), None, config.clone(), "disk-test".to_string()); + let memory = disk.get_memory(id).unwrap(); + assert!(memory.disk_backed); + assert_eq!(disk.read_memory_content(&memory).unwrap(), "persisted"); + let mut ram_config = config; + ram_config.server.store_content_on_disk = false; + let restored = CueMapEngine::from_state((**disk.get_memories()).clone(), (**disk.get_source_key_to_id()).clone(), (**disk.get_cue_index()).clone(), disk.next_memory_id(), None, ram_config, "disk-test".to_string()); + let restored_memory = restored.get_memory(id).unwrap(); + assert!(!restored_memory.disk_backed); + assert_eq!(restored.read_memory_content(&restored_memory).unwrap(), "persisted"); + assert_eq!(restored.get_cue_frequency("persist"), 1); + let _ = std::fs::remove_dir_all(restored.get_disk_content_dir().parent().unwrap().parent().unwrap()); + } + + #[test] + fn temporal_chunking_disk_storage_and_source_aliases_work() { + let mut engine = CueMapEngine::::new(); + let mut config = engine.config.clone(); + config.server.data_dir = std::env::temp_dir().join(format!("cuemap-engine-direct-{}", std::process::id())).to_string_lossy().to_string(); + config.server.store_content_on_disk = true; + engine.config = config; + let first = engine.add_memory_with_event_time( + "first event".to_string(), + vec!["topic:rust".to_string()], + None, + MainStats::default(), + false, + Some(1000.0), + ); + let second = engine.add_memory_with_event_time( + "second event".to_string(), + vec!["topic:rust".to_string()], + None, + MainStats::default(), + false, + Some(1100.0), + ); + let first_memory = engine.get_memory(first).unwrap(); + let second_memory = engine.get_memory(second).unwrap(); + assert!(first_memory.disk_backed && second_memory.disk_backed); + assert!(second_memory.cues.iter().any(|cue| cue == &format!("episode:{first}"))); + assert_eq!(engine.read_memory_content(&second_memory).unwrap(), "second event"); + assert_eq!(engine.get_cue_frequency("topic:rust"), 2); + assert_eq!(engine.source_order_for_memory(first), None); + let _ = std::fs::remove_dir_all(engine.get_disk_content_dir().parent().unwrap().parent().unwrap()); + } + + #[test] + fn semantic_budget_and_reranker_invalid_paths_are_safe() { + let mut engine = CueMapEngine::::new(); + let mut config = SemanticConfig::default(); + config.enabled = true; + config.dimensions = 3; + config.max_memory_mb = 1; + config.storage = crate::semantic::SemanticStorage::F32; + config.reranker_enabled = true; + config.reranker_weights = vec![0.1; 6]; + config.reranker_scale = 0.5; + engine.set_semantic_config(config); + let id = engine.add_memory_with_event_time_and_vector("vector".to_string(), vec!["vector".to_string()], None, MainStats::default(), true, None, Some(vec![1.0, 0.0, 0.0])); + let mut candidates = vec![candidate(id, 1.0), candidate(INVALID_MEMORY_ID, 0.0)]; + assert_eq!(engine.rerank_existing_semantic_candidates(&mut candidates, &[]), 0); + assert_eq!(engine.merge_semantic_candidates(&mut candidates, &[], 2), (0, 0)); + engine.apply_semantic_reranker(&mut candidates, &[]); + let invalid_query_results = engine.recall_weighted_with_query_embedding(vec![("vector".to_string(), 1.0)], 2, false, None, 1, false, false, None, None, Some(&[])); + assert_eq!(invalid_query_results.len(), 1); + assert!(engine.get_memory(id).unwrap().semantic_vector.is_some()); + + let mut budget_engine = CueMapEngine::::new(); + let mut budget_config = SemanticConfig::default(); + budget_config.enabled = true; + budget_config.dimensions = 1_000_000; + budget_config.max_memory_mb = 1; + budget_engine.set_semantic_config(budget_config); + let huge = budget_engine.add_memory_with_event_time_and_vector("budget".to_string(), vec!["budget".to_string()], None, MainStats::default(), true, None, Some(vec![0.0; 1_000_000])); + assert!(budget_engine.get_memory(huge).unwrap().semantic_vector.is_none()); + } + + #[test] + fn intent_coverage_tracks_current_and_stale_annotations() { + let engine = CueMapEngine::::new(); + let current_id = engine.add_memory("current".to_string(), vec!["current".to_string()], None, MainStats::default(), true); + let stale_id = engine.add_memory("stale".to_string(), vec!["stale".to_string()], None, MainStats::default(), true); + engine.attach_intent_classification(current_id, classification("preference", 1.0, true)); + let mut stale = classification("event_or_plan", 1.0, true); + stale.model_version = "old-model".to_string(); + engine.attach_intent_classification(stale_id, stale); + assert_eq!(engine.intent_coverage(), (2, 2, 1, 1)); + assert!(engine.attach_intent_classification(INVALID_MEMORY_ID, classification("preference", 1.0, true)) == false); + } + + #[test] + fn parent_expansion_symbols_and_consolidation_are_exercised() { + let engine = CueMapEngine::::new(); + let mut parent = HashMap::new(); + parent.insert("parent:doc".to_string(), serde_json::json!(true)); + let _first = engine.add_memory("chunk one".to_string(), vec!["parent:doc".to_string(), "chunk_idx:0".to_string(), "needle".to_string()], None, MainStats::default(), true); + let _middle = engine.add_memory("chunk two".to_string(), vec!["parent:doc".to_string(), "chunk_idx:1".to_string(), "needle".to_string()], None, MainStats::default(), true); + let _third = engine.add_memory("chunk three".to_string(), vec!["parent:doc".to_string(), "chunk_idx:2".to_string(), "needle".to_string()], None, MainStats::default(), true); + let expanded = engine.recall_weighted(vec![("needle".to_string(), 1.0)], 1, false, None, 2, false, false, None, None); + assert!(expanded.iter().any(|result| result.content.contains("chunk two") && result.content.contains("chunk three"))); + + engine.add_memory("fn".to_string(), vec!["defines_function:run".to_string(), "calls_method:save".to_string(), "defines_function:".to_string()], Some(parent), MainStats::default(), true); + let symbols = engine.get_all_symbols(); + assert!(symbols.contains("run") && symbols.contains("save")); + + let mut m1 = MainStats::default(); + m1.intrinsic_salience = 2.0; + let mut m2 = MainStats::default(); + m2.intrinsic_salience = 4.0; + engine.add_memory("merge one".to_string(), vec!["shared".to_string(), "one".to_string()], None, m1, true); + engine.add_memory("merge two".to_string(), vec!["shared".to_string(), "two".to_string()], None, m2, true); + let merged = engine.consolidate_memories(0.3); + assert!(!merged.is_empty()); + let summary = engine.get_memory(merged[0].0).unwrap(); + assert_eq!(summary.metadata.get("consolidated").and_then(|v| v.as_bool()), Some(true)); + assert!(summary.cues.iter().any(|cue| cue == "type:summary")); + } + + #[test] + fn id_exhaustion_and_stats_snapshot_are_safe() { + let engine = CueMapEngine::::new(); + engine.next_memory_id.store(MemoryId::MAX, Ordering::Relaxed); + assert_eq!(engine.add_memory("too late".to_string(), vec!["x".to_string()], None, MainStats::default(), true), INVALID_MEMORY_ID); + let stats = engine.get_stats(); + assert_eq!(stats.get("total_memories").and_then(|v| v.as_u64()), Some(0)); + assert_eq!(stats.get("total_cues").and_then(|v| v.as_u64()), Some(0)); + } + + #[test] + fn public_constructor_and_recall_wrappers_are_smoke_tested() { + let mut engine = CueMapEngine::::with_tuning(crate::config::TuningConfig::default()); + engine.set_tuning_config(crate::config::TuningConfig::default()); + engine.set_master_key(Some(Arc::new(crate::crypto::EncryptionKey::new(vec![3; 32])))); + let id = engine.add_memory_with_source_key( + "wrapper memory".to_string(), + vec!["wrapper".to_string()], + None, + MainStats::default(), + true, + Some("wrapper-key".to_string()), + ); + assert_ne!(id, INVALID_MEMORY_ID); + assert_eq!(engine.memory_id_for_source_key("wrapper-key"), Some(id)); + assert!(engine.get_memories().contains_key(&id)); + assert!(engine.get_source_key_to_id().contains_key("wrapper-key")); + assert!(engine.get_cue_index().contains_key("wrapper")); + assert!(engine.next_memory_id() > id); + let reranked = engine.recall_weighted_with_query_embedding_rerank_only(vec![("wrapper".to_string(), 1.0)], 1, false, None, 1, false, false, None, None, Some(&[1.0])); + assert_eq!(reranked.len(), 1); + let (_results, timing) = engine.recall_weighted_with_query_embedding_rerank_only_with_timing(vec![("wrapper".to_string(), 1.0)], 1, false, None, 1, false, false, None, None, Some(&[1.0])); + assert!(timing.total_ms >= 0.0); + } + + #[test] + fn generic_engine_paths_are_instantiated_for_lexicon_and_main_stats() { + let mut metadata = HashMap::new(); + metadata.insert("session_id".to_string(), serde_json::json!("lex-session")); + metadata.insert("source_order".to_string(), serde_json::json!(1)); + let mut lexicon = CueMapEngine::::new(); + let lex_id = lexicon.add_memory("lexicon word".to_string(), vec!["word".to_string()], Some(metadata), LexiconStats::default(), true); + let lex_mem = lexicon.get_memory(lex_id).unwrap(); + assert_eq!(lexicon.read_memory_content(&lex_mem).unwrap(), "lexicon word"); + assert_eq!(lexicon.recall_fast(vec!["word".to_string()], 1).len(), 1); + assert_eq!(lexicon.recall_intersection(vec![("word".to_string(), 1.0)], 1).len(), 1); + assert_eq!(lexicon.source_order_window("lex-session", 1, 2).len(), 1); + let _ = lexicon.set_semantic_config(SemanticConfig::default()); + let restored = CueMapEngine::from_state((**lexicon.get_memories()).clone(), (**lexicon.get_source_key_to_id()).clone(), (**lexicon.get_cue_index()).clone(), lexicon.next_memory_id(), None, crate::config::ServerConfig::default(), "lexicon".to_string()); + assert_eq!(restored.get_memory(lex_id).unwrap().id, lex_id); + + let main = CueMapEngine::::new(); + let main_id = main.add_memory("main word".to_string(), vec!["word".to_string()], None, MainStats::default(), true); + assert_eq!(main.recall_fast(vec!["word".to_string()], 1).len(), 1); + assert_eq!(main.recall_intersection(vec![("word".to_string(), 1.0)], 1).len(), 1); + let main_mem = main.get_memory(main_id).unwrap(); + assert_eq!(main.read_memory_content(&main_mem).unwrap(), "main word"); + } + + #[test] + fn structured_reranking_penalty_matrix_exercises_all_families() { + let cases: [(&str, f64, &str); 15] = [ + ("type:update", 1.0, "family_relation:sibling"), + ("family_count:2", 1.0, "family_relation:sibling"), + ("source_role:user", 3.0, "family_relation:sibling"), + ("source_time:morning", 1.0, "family_relation:sibling"), + ("source_time:morning", 1.0, "source_time:evening"), + ("type:preference", 3.0, "family_relation:sibling"), + ("person_role_phrase:lead", 1.0, "quantity_object:item"), + ("quantity_object:item", 1.0, "inventory_object:book"), + ("inventory_object:book", 1.0, "travel:trip"), + ("travel:trip", 1.0, "age:42"), + ("age:42", 1.0, "education:college"), + ("education:college", 1.0, "family_relation:sibling"), + ("family_relation:sibling", 1.0, "family_scope:home"), + ("family_scope:home", 1.0, "family_relation:sibling"), + ("source_time:morning", 1.0, "topic:rust"), + ]; + for (query_cue, weight, candidate_structured) in cases { + let engine = CueMapEngine::::new(); + engine.add_memory( + "structured seed".to_string(), + vec!["seed".to_string(), query_cue.to_string()], + None, + MainStats::default(), + true, + ); + engine.add_memory( + "lexical candidate".to_string(), + vec!["lexical".to_string(), candidate_structured.to_string()], + None, + MainStats::default(), + true, + ); + let results = engine.recall_weighted( + vec![("lexical".to_string(), 1.0), (query_cue.to_string(), weight)], + 5, + false, + None, + 1, + false, + true, + None, + None, + ); + assert!(!results.is_empty(), "no result for {query_cue}"); + } + } + + #[test] + fn salience_decay_and_consolidation_truncation_paths_are_verified() { + let engine = CueMapEngine::::new(); + let cold = engine.add_memory("cold".to_string(), vec!["cold".to_string()], None, MainStats::default(), true); + let warm = engine.add_memory("warm".to_string(), vec!["warm".to_string()], None, MainStats::default(), true); + if let Some(mut memory) = engine.get_memories().get_mut(&cold) { + memory.stats.dynamic_salience = 0.005; + memory.stats.last_boosted_at = 1; + } + if let Some(mut memory) = engine.get_memories().get_mut(&warm) { + memory.stats.dynamic_salience = 3.0; + memory.stats.last_boosted_at = 1; + } + engine.decay_salience(0.5); + assert_eq!(engine.get_memory(cold).unwrap().stats.dynamic_salience, 0.0); + assert!(engine.get_memory(warm).unwrap().stats.dynamic_salience < 3.0); + assert!(engine.get_trending_cues(10).is_empty()); + + let long = "x".repeat(700); + engine.add_memory(long.clone(), vec!["merge-long".to_string()], None, MainStats::default(), true); + engine.add_memory(long, vec!["merge-long".to_string()], None, MainStats::default(), true); + let merged = engine.consolidate_memories(0.5); + assert!(!merged.is_empty()); + let summary = engine.get_memory(merged[0].0).unwrap(); + assert!(engine.read_memory_content(&summary).unwrap().contains("[truncated]")); + } + + #[test] + fn recall_intersection_and_fast_paths_cover_duplicates_and_matches() { + let engine = CueMapEngine::::new(); + let id = engine.add_memory("both cues".to_string(), vec!["first".to_string(), "second".to_string()], None, MainStats::default(), true); + let other = engine.add_memory("first only".to_string(), vec!["first".to_string()], None, MainStats::default(), true); + let intersection = engine.recall_intersection(vec![("first".to_string(), 2.0), ("second".to_string(), 3.0)], 10); + assert_eq!(intersection[0].memory_id, id); + assert_eq!(intersection[0].intersection_count, 2); + let fast = engine.recall_fast(vec!["first".to_string(), "second".to_string(), "first".to_string()], 10); + assert_eq!(fast.len(), 2); + assert!(fast.iter().any(|result| result.memory_id == other)); + } +} diff --git a/src/facets.rs b/src/facets.rs index acfcdf4..b6517e3 100644 --- a/src/facets.rs +++ b/src/facets.rs @@ -1,3 +1,15 @@ +//! Deterministic structural extraction and bounded query-shape planning. +//! +//! This module deliberately does not attempt to classify language into an +//! ontology. Content extraction emits cues for observable structure: numbers, +//! quantities, identifiers, dates, times, durations, lists, document/code +//! markers, source metadata, surface entities, emoji, discourse markers, and +//! explicit temporal relations. Query planning adds only bounded English +//! query-shape heuristics (grammatical perspective, answer shape, +//! collection/summary/order shape, and reference-time resolution). Semantic +//! retrieval belongs in a vector/learned layer rather than in domain-specific +//! regex families here. + use chrono::{Datelike, Duration, NaiveDate}; use regex::Regex; use serde::{Deserialize, Serialize}; @@ -7,6 +19,10 @@ use std::sync::OnceLock; const MAX_FACETS: usize = 64; const MAX_ENTITIES: usize = 16; +// Keep this explicit so source-role weighting can be benchmark-ablated without +// confusing it with a semantic classifier. The current value preserves the +// established v0.7.2 retrieval behavior. +const QUERY_PERSPECTIVE_SOURCE_ROLE_WEIGHT: f64 = 2.0; fn money_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); @@ -18,668 +34,317 @@ fn money_re() -> &'static Regex { }) } -fn number_re() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"\b\d+(?:[.,]\d+)?\b").unwrap()) -} - -fn date_re() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"(?i)\b(?:\d{4}-\d{1,2}-\d{1,2}|\d{1,2}/\d{1,2}/\d{2,4}|jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b").unwrap()) -} - -fn short_numeric_date_re() -> &'static Regex { +fn url_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"\b(?P\d{1,2})/(?P\d{1,2})(?:/\d{2,4})?\b").unwrap()) + RE.get_or_init(|| { + Regex::new( + r"(?ix)\b(?:https?://(?:localhost(?::\d{1,5})?|(?:\d{1,3}\.){3}\d{1,3}(?::\d{1,5})?|(?:[a-z0-9-]+\.)+[a-z]{2,})(?:[\x2f:\x3f\x23][^\s<>()]*)?|localhost:\d{1,5}\b)", + ) + .unwrap() + }) } -fn weekday_re() -> &'static Regex { +fn email_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"(?i)\b(?:mondays?|tuesdays?|wednesdays?|thursdays?|fridays?|saturdays?|sundays?)\b").unwrap()) + RE.get_or_init(|| { + Regex::new(r"(?i)\b[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9-]+(?:\.[a-z0-9-]+)+\b") + .unwrap() + }) } -fn clock_time_re() -> &'static Regex { +fn inline_code_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"(?i)\b(?:[01]?\d|2[0-3])(?::[0-5]\d)?\s*(?:am|pm|a\.m\.|p\.m\.)\b").unwrap()) + RE.get_or_init(|| Regex::new(r#"`[^`\n]{1,120}`"#).unwrap()) } -fn clock_time_capture_re() -> &'static Regex { +fn file_name_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?i)\b(?P[01]?\d|2[0-3])(?::(?P[0-5]\d))?\s*(?Pam|pm|a\.m\.|p\.m\.)\b", + r#"(?i)(?:^|[\s("'`])(?:\.[a-z][a-z0-9_-]*|[a-z][a-z0-9_.-]*\.[a-z][a-z0-9_.-]*)(?:$|[\s)"'`,;:.])"#, ) .unwrap() }) } -fn duration_re() -> &'static Regex { +fn file_path_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?ix) - \b(?:for\s+)? - (?: - \d+(?:[.,]\d+)? - | - one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve - ) - \s*(?:seconds?|minutes?|hours?|days?|weeks?|months?|years?)\b - ", + r#"(?ix)(?:^|[\s("'`])(?:[a-z]:[\\/](?:[a-z0-9_.-]+[\\/])*[a-z0-9_.-]+|\.{1,2}[\\/](?:[a-z0-9_.-]+[\\/])*[a-z0-9_.-]+|/(?:[a-z0-9_.-]+[\\/])*[a-z0-9_.-]+|(?:[a-z0-9_.-]+[\\/]){1,}[a-z0-9_.-]+)(?:$|[\s)"'`,;:.])"#, ) .unwrap() }) } -fn cadence_re() -> &'static Regex { +fn has_code_fence(content: &str) -> bool { + content + .lines() + .any(|line| line.trim_start().starts_with("```")) +} + +fn number_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"\b\d+(?:[.,]\d+)?\b").unwrap()) +} + +fn measurement_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?ix)\b(?:(?:once|twice)|(?:one|two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times?|\d+(?:[.,]\d+)?\s+hours?)\s+(?:a|an|per|each|every)\s+(?Pday|week|month|year)s?\b", + r"(?ix)\b(?P\d+(?:[.,]\d+)?|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\s*(?Pmicroseconds?|milliseconds?|nanoseconds?|seconds?|minutes?|hours?|days?|bytes?|kilobytes?|megabytes?|gigabytes?|terabytes?|kilograms?|grams?|milligrams?|pounds?|degrees?\s*[cf]|°\s*[cf]|celsius|fahrenheit|kelvin|us|μs|µs|ms|ns|sec|s|min|hr|h|d|b|kb|mb|gb|tb|kg|g|mg|lbs?)\b", ) .unwrap() }) } -fn every_unit_re() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"(?i)\bevery\s+(?Pday|week|month|year)\b").unwrap()) -} - -fn age_year_old_re() -> &'static Regex { +fn percentage_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { - Regex::new(r"(?i)\b(?P\d{1,3})\s*[- ]\s*years?\s*[- ]\s*old\b").unwrap() + Regex::new(r"(?i)\b(?P\d+(?:[.,]\d+)?)\s*(?:%|percent)(?:\b|[\s.,;:!?]|$)") + .unwrap() }) } -fn current_age_re() -> &'static Regex { +fn between_range_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?i)\b(?:(?:i|we)\s*(?:am|are|'m|'re|’m|’re)\s+(?:(?:a|an)\s+)?|as\s+(?:a|an)\s+|currently\s+)(?P\d{1,3})\s*[- ]\s*years?\s*[- ]\s*old\b", + r"(?ix)\bbetween\s+(?P\d+(?:[.,]\d+)?)\s+and\s+(?P\d+(?:[.,]\d+)?)(?:\s*(?Pmilliseconds?|seconds?|minutes?|hours?|days?|kilograms?|grams?|pounds?|degrees?\s*[cf]|°\s*[cf]|celsius|fahrenheit|kelvin|ms|s|min|hr|h|d|kg|g|lbs?))?\b", ) .unwrap() }) } -fn event_age_re() -> &'static Regex { +fn from_range_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?i)\b(?:at\s+(?:the\s+)?age\s+of|at\s+age|by\s+age|when\s+(?:i|we|he|she|they|you)\s+(?:was|were))\s+(?P\d{1,3})\b", + r"(?ix)\bfrom\s+(?P\d+(?:[.,]\d+)?)\s+to\s+(?P\d+(?:[.,]\d+)?)(?:\s*(?Pmilliseconds?|seconds?|minutes?|hours?|days?|kilograms?|grams?|pounds?|degrees?\s*[cf]|°\s*[cf]|celsius|fahrenheit|kelvin|ms|s|min|hr|h|d|kg|g|lbs?))?\b", ) .unwrap() }) } -fn metadata_date_re() -> &'static Regex { +fn uuid_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new(r"\b(?P\d{4})[-/](?P\d{1,2})[-/](?P\d{1,2})\b") - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b").unwrap()) } -fn first_person_completed_action_re() -> &'static Regex { +fn semver_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)(?:\s+|'ve\s+|’ve\s+|'d\s+|’d\s+) - (?: - (?:(?:just|recently|finally|already)\s+)[a-z]{3,}(?:ed|ied) | - (?:(?:just|recently|finally|already|also)\s+)? - (?:went | gone | visited | attended | joined | participated | - completed | finished | started | planted | bought | purchased | - booked | watched | read | wrote | made | built | created | - cooked | baked | ran | walked | hiked | traveled | travelled | - played | tried) | - (?:(?:just|recently|finally|already)\s+)?got\s+back\s+from | - (?:(?:have|had|'ve|’ve)\s+)?been\s+to - )\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"\bv?\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\b").unwrap()) } -fn first_person_did_named_event_re() -> &'static Regex { +fn issue_reference_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\s+ - (?:(?:just|recently|finally|already|also)\s+)? - did\s+ - (?:(?:the|a|an)\s+)? - .{0,90} - \b(?:event|walk|run|ride|drive|gala|fundraiser|fund-raiser|workshop|tour|class)\b - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?i)(?:\b(?:pr|gh)\s*)?#(?P\d+)\b|\b(?:gh|pr)-(?P\d+)\b").unwrap()) } -fn first_person_acquired_re() -> &'static Regex { +fn ip_address_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\s+ - (?:(?:just|recently|finally|already|also|actually)\s+)? - (?:got|bought|purchased|ordered|picked\s+up)\s+ - (?:(?:him|her|them|someone|somebody)\s+)? - (?:(?:a|an|the|my|our|new|some|\d+)\b) - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"\b(?:\d{1,3}\.){3}\d{1,3}\b").unwrap()) } -fn first_person_acquisition_source_re() -> &'static Regex { +fn port_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\b - .{0,120}? - \b(?:got|bought|purchased|ordered|picked\s+up)\b - .{0,120}? - \b(?:from|at|via|through)\s+ - (?:a|an|the|my|our|new|some)?\s* - [A-Za-z0-9][A-Za-z0-9'&.-]{1,} - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?i)\bport\s+(?P\d{1,5})\b|\b(?:localhost|(?:\d{1,3}\.){3}\d{1,3}|(?:[a-z0-9-]+\.)+[a-z]{2,}):(?P\d{1,5})\b").unwrap()) } -fn ownership_source_re() -> &'static Regex { +fn domain_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:(?:my|our|the)\s+)? - (?:new|current|latest|recent)\s+ - [A-Za-z0-9][A-Za-z0-9'&.-]{2,}(?:\s+[A-Za-z0-9][A-Za-z0-9'&.-]{2,}){0,3} - \s+(?:is|are|was|were|came)\s+from\s+ - [A-Za-z0-9][A-Za-z0-9'&.-]{1,} - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?i)\b(?:[a-z0-9-]+\.)+[a-z]{2,}\b").unwrap()) } -fn first_person_competition_event_re() -> &'static Regex { +fn environment_variable_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\s+ - (?: - (?:just\s+|recently\s+)?(?:completed|finished|ran|raced|joined|entered|played|participated\s+in|participate\s+in) | - (?:will|am\s+going\s+to|are\s+going\s+to|'m\s+going\s+to|'re\s+going\s+to|’m\s+going\s+to|’re\s+going\s+to|plan\s+to|planning\s+to)\s+(?:participate\s+in|play|run|race|join|enter) | - (?:am|are|'m|'re|’m|’re)\s+(?:participating\s+in|playing|running|racing) - ) - .{0,80} - \b(?:tournament|triathlon|marathon|half\s+marathon|race|run|5k|10k|match|game|competition|bike\s+ride)\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"\b[A-Z][A-Z0-9]*_[A-Z0-9_]+\b").unwrap()) } -fn first_person_with_companion_re() -> &'static Regex { +fn user_mention_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\b - .{0,140}? - \b(?:went|go|going|attended|saw|seen|visited|joined|participated|did|completed|watched|traveled|travelled|been)\b - .{0,140}? - \bwith\s+ - (?:(?:my|our|a|an|the|some|a\s+group\s+of)\s+)? - [A-Za-z0-9][A-Za-z0-9'&.-]{1,} - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"@[A-Za-z][A-Za-z0-9_.-]{1,63}\b").unwrap()) } -fn companion_query_re() -> &'static Regex { +fn hashtag_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \bwho\s+(?:(?:did|do|will)\s+)?(?:i|we)\b.{0,90}\bwith\b - | - \bwho\s+(?:was|were)\s+with\s+(?:me|us)\b - | - \bwith\s+whom\b - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"#[A-Za-z][A-Za-z0-9_-]{1,63}\b").unwrap()) } -fn first_person_completed_clean_re() -> &'static Regex { +fn commit_hash_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\b - (?: - .{0,50}?\b(?:cleaned|washed|polished|conditioned)\b - | - .{0,70}?\b(?:got\s+around\s+to|finished|finally\s+finished|remember\s+I|remember\s+we)\s+ - (?:cleaning|washing|polishing|conditioning)\b - ) - .{0,100}? - \b(?:my|our|the|a|an|this|that|these|those)\b - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"\b[0-9a-fA-F]{7,40}\b").unwrap()) } -fn completed_clean_query_re() -> &'static Regex { +fn negation_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?ix) - \b(?:what|which|when|where|did|have|had)\b - .{0,80}? - \b(?:i|we)\b - .{0,60}? - \b(?:clean|cleaned|cleaning|wash|washed|washing|polish|polished|polishing|condition|conditioned|conditioning)\b - ", + r"(?i)\b(?:not|never|neither|no|without|cannot|can't|don't|doesn't|didn't|won't|wouldn't|shouldn't|isn't|aren't|wasn't|weren't|haven't|hasn't|hadn't|mustn't)\b", ) .unwrap() }) } -fn co_residence_with_self_re() -> &'static Regex { +fn json_object_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:live|lived|living|stay|stayed|staying)\s+with\s+(?:me|us)\b - | - \b[A-Za-z][A-Za-z'-]{0,24}(?:(?:\s+(?:has|have|had))|(?:['’](?:ve|d)))\s+ - been\s+(?:living|staying)\s+with\s+(?:me|us)\b - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r#"(?s)\{\s*["']?[A-Za-z0-9_.-]+["']?\s*:"#).unwrap()) } -fn co_residence_query_re() -> &'static Regex { +fn key_value_line_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:live|lived|living|stay|stayed|staying)\s+with\s+(?:me|us)\b - | - \bbeen\s+(?:living|staying)\s+with\s+(?:me|us)\b - ", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?m)^\s*[A-Za-z][A-Za-z0-9_.-]*\s*:\s*\S+").unwrap()) } -fn first_person_project_work_re() -> &'static Regex { +fn xml_element_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?: - (?:i|we)(?:\s+)? - (?: - (?:am|are|'m|'re|’m|’re|was|were|have\s+been|has\s+been|'ve\s+been|’ve\s+been)\s+ - (?:working\s+on|leading|managing|running|building|developing|researching|presenting) - | - (?:led|managed|ran|built|developed|presented|researched|participated\s+in) - ) - .{0,80} - \b(?:project|research|campaign|initiative|case\s+competition|poster|feature)\b - | - (?:my|our)\s+(?:current\s+|latest\s+|new\s+|solo\s+|research\s+)?(?:project|research|campaign|initiative) - )\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?s)<[A-Za-z][A-Za-z0-9_.:-]*(?:\s[^>]*)?>.*?").unwrap()) } -fn decision_selection_re() -> &'static Regex { +fn markdown_table_separator_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?: - (?:i|we)\s+(?:choose|chose|pick|picked|select|selected|decided|settled\s+on|went\s+with) | - (?:let's|lets)\s+(?:go\s+with|call\s+it|name\s+it) | - [a-z0-9][a-z0-9'_-]*(?:\s+[a-z0-9][a-z0-9'_-]*){0,5}\s+ - is\s+(?:a|an|the)?\s*(?:[a-z0-9'_-]+\s+){0,5}(?:one|choice|name|pick) - )\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?m)^\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*$").unwrap()) } -fn prefix_re() -> &'static Regex { +fn stack_trace_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"(?m)^\s*([A-Za-z][A-Za-z0-9_-]{1,32})\s*:\s+").unwrap()) + RE.get_or_init(|| Regex::new(r"(?im)^\s*(?:traceback\s*\(|at\s+[A-Za-z0-9_.$/-]+(?:\([^\n]*\)|:\d+)|caused by:)").unwrap()) } -fn quoted_re() -> &'static Regex { +fn diff_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r#""([^"\n]{2,80})"|'([^'\n]{2,80})'"#).unwrap()) + RE.get_or_init(|| Regex::new(r"(?m)^\s*(?:diff --git |\+\+\+ |--- |@@\s+-\d+)").unwrap()) } -fn proper_noun_re() -> &'static Regex { +fn heading_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"\b[A-Z][a-zA-Z0-9]*(?:\s+[A-Z][a-zA-Z0-9]*){0,3}\b").unwrap()) + RE.get_or_init(|| Regex::new(r"(?m)^\s{0,3}(?P#{1,6})\s+\S+").unwrap()) } -fn entity_class_relation_re() -> &'static Regex { +fn checklist_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"\b(?P(?:[A-Z][A-Za-z'-]{2,}|[a-z][A-Za-z'-]{2,})(?:\s+(?:[A-Z][A-Za-z'-]{2,}|[a-z][A-Za-z'-]{2,})){0,3})\s+(?i:like|such\s+as)\s+(?P[A-Z][A-Za-z'-]{1,40})\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?m)^\s*(?:[-*]|\d+[.)])\s+\[[ xX]\]\s+\S+").unwrap()) } -fn preferred_attribute_relation_re() -> &'static Regex { +fn block_quote_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"\b(?P[A-Z][A-Za-z0-9'&.-]{1,40}(?:\s+[A-Z][A-Za-z0-9'&.-]{1,40}){0,3})\s+(?i:has\s+been|have\s+been|is|are|was|were|became|remains)\s+(?:(?:my|our|the)\s+)?(?i:favou?rite|preferred)\s+(?P[A-Za-z][A-Za-z'-]{2,}(?:\s+[A-Za-z][A-Za-z'-]{2,}){0,2})\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?m)^\s*>\s+\S+").unwrap()) } -fn titled_person_re() -> &'static Regex { +fn markdown_link_re() -> &'static Regex { static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new(r"\b(?PDr|Prof)\.?\s+(?P<name>[A-Z][A-Za-z'-]{1,40})\b").unwrap() - }) + RE.get_or_init(|| Regex::new(r"\[[^\]\n]{1,120}\]\([^\)\n]{1,240}\)").unwrap()) } -fn role_before_title_re() -> &'static Regex { +fn fenced_language_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?m)(?:^|[.!?;]\s+|,\s+(?:(?i:and|but)\s+)?|(?i:\band|but)\s+)(?P<role>(?:[A-Z]{2,8}|[A-Za-z][A-Za-z'-]{1,})(?:\s+(?:[A-Z]{2,8}|[A-Za-z][A-Za-z'-]{1,})){0,3})\s+(?i:Dr|Prof)\.?\s+[A-Z][A-Za-z'-]{1,40}\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?m)^\s*```(?P<language>[A-Za-z][A-Za-z0-9_+-]*)\b").unwrap()) } -fn possessed_role_before_title_re() -> &'static Regex { +fn date_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { - Regex::new( - r"\b(?i:my|our|the|a|an)\s+(?P<role>[A-Za-z][A-Za-z'-]{1,}(?:\s+[A-Za-z][A-Za-z'-]{1,}){0,3})\s+(?i:Dr|Prof)\.?\s+[A-Z][A-Za-z'-]{1,40}\b", - ) - .unwrap() + Regex::new(r"(?i)\b(?:\d{4}-\d{1,2}-\d{1,2}|\d{1,2}/\d{1,2}/\d{2,4}|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b").unwrap() }) } -fn role_before_name_re() -> &'static Regex { +fn short_numeric_date_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?m)(?:^|[.!?;]\s+|,\s+(?:(?i:and|but)\s+)?|(?i:\band|but)\s+)(?P<role>[A-Za-z][A-Za-z'-]{1,}(?:\s+[A-Za-z][A-Za-z'-]{1,}){0,3})\s+[A-Z][A-Za-z'-]{1,40}\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"\b(?P<month>\d{1,2})/(?P<day>\d{1,2})(?:/\d{2,4})?\b").unwrap()) } -fn sibling_count_re() -> &'static Regex { +fn weekday_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?i)\b(?:(?:i|we)\s+(?:have|had|have\s+got|'ve\s+got|’ve\s+got|got)\s+(?:a|an|one|two|three|four|five|six|seven|eight|nine|ten|\d+)\s+(?:older\s+|younger\s+|little\s+|big\s+|half\s+|step\s+)?(?:brothers?|sisters?|siblings?)|(?:come\s+from|grew\s+up\s+in|part\s+of)\s+(?:a\s+)?family\s+with\s+(?:a|an|one|two|three|four|five|six|seven|eight|nine|ten|\d+)\s+(?:older\s+|younger\s+|little\s+|big\s+|half\s+|step\s+)?(?:brothers?|sisters?|siblings?))\b", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?i)\b(?:mondays?|tuesdays?|wednesdays?|thursdays?|fridays?|saturdays?|sundays?)\b").unwrap()) } -fn self_sibling_re() -> &'static Regex { +fn clock_time_12_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { - Regex::new( - r"(?i)\b(?:(?:my|our)\s+(?:older\s+|younger\s+|little\s+|big\s+|half\s+|step\s+)?(?:brothers?|sisters?|siblings?)|(?:i|we)\s+(?:have|had|have\s+got|'ve\s+got|’ve\s+got|got)\s+.{0,40}\b(?:brothers?|sisters?|siblings?)|(?:come\s+from|grew\s+up\s+in|part\s+of)\s+(?:a\s+)?family\s+with\s+.{0,40}\b(?:brothers?|sisters?|siblings?))\b", - ) - .unwrap() + Regex::new(r"(?i)\b(?:0?[1-9]|1[0-2])(?::[0-5]\d)?\s*(?:am|pm|a\.m\.|p\.m\.)\b") + .unwrap() }) } -fn possessive_object_re() -> &'static Regex { +fn clock_time_capture_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { - Regex::new( - r"(?i)\b(?:my|our)\s+(?P<object>[A-Za-z0-9][A-Za-z0-9'-]*(?:\s+[A-Za-z0-9][A-Za-z0-9'-]*){0,6})", - ) - .unwrap() + Regex::new(r"(?i)\b(?P<hour>0?[1-9]|1[0-2])(?::(?P<minute>[0-5]\d))?\s*(?P<meridiem>am|pm|a\.m\.|p\.m\.)\b").unwrap() }) } -fn owned_object_re() -> &'static Regex { +fn clock_time_24_capture_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:i|we)\s+(?:currently\s+)? - (?: - (?:have|got)\s+(?:a|an|the|this|that|these|those|my|our|some|\d+)\s+ - (?P<object_det>[A-Za-z0-9][A-Za-z0-9'-]*(?:\s+[A-Za-z0-9][A-Za-z0-9'-]*){0,4}) - | - (?:own|use|keep|maintain|set\s+up)\s+ - (?P<object_direct>[A-Za-z0-9][A-Za-z0-9'-]*(?:\s+[A-Za-z0-9][A-Za-z0-9'-]*){0,4}) - )", - ) - .unwrap() + Regex::new(r"\b(?P<hour>[01]\d|2[0-3]):(?P<minute>[0-5]\d)\b").unwrap() }) } -fn owned_object_capture_text<'a>(cap: &'a regex::Captures<'a>) -> Option<&'a str> { - cap.name("object_det") - .or_else(|| cap.name("object_direct")) - .map(|m| m.as_str()) +fn has_clock_time(content: &str) -> bool { + clock_time_12_re().is_match(content) || clock_time_24_capture_re().is_match(content) } -fn first_person_possession_re() -> &'static Regex { +fn duration_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { Regex::new( - r"(?ix) - \b(?: - (?:i|we)(?:\s+have|\s+had|'ve|’ve)\s+had\s+(?:my|our)\s+ | - (?:i|we)(?:'ve|’ve|\s+have)?\s+been\s+(?:playing|using|wearing|driving|riding|keeping|maintaining)\s+(?:my|our)\s+ | - (?:i|we)(?:'m|’m|\s+am|\s+was|\s+are|\s+were)?\s*(?:thinking\s+of|planning\s+to|trying\s+to|looking\s+to)?\s*(?:sell|selling|sold)\s+(?:my|our)\s+ | - which\s+(?:i|we)(?:'ve|’ve|\s+have|\s+had)?\s+had\b - )", + r"(?ix)\b(?:for\s+)?(?:\d+(?:[.,]\d+)?|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\s*(?:seconds?|minutes?|hours?|days?|weeks?|months?|years?)\b", ) .unwrap() }) } -fn homegrown_source_re() -> &'static Regex { +fn cadence_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { - Regex::new( - r"(?ix) - \bhome[-\s]?grown\b | - \b(?:harvest(?:ed|ing)?|grew|grown|growing|picked|planted)\b.{0,80} - \b(?:garden|yard|backyard|planters?|raised\s+beds?|greenhouse|farm|balcony)\b | - \b(?:garden|yard|backyard|planters?|raised\s+beds?|greenhouse|farm|balcony)\b.{0,80} - \b(?:harvest(?:ed|ing)?|grew|grown|growing|picked|planted)\b - ", - ) - .unwrap() + Regex::new(r"(?ix)\b(?:(?:once|twice)|(?:one|two|three|four|five|six|seven|eight|nine|ten|\d+)\s+times?|\d+(?:[.,]\d+)?\s+hours?)\s+(?:a|an|per|each|every)\s+(?P<unit>day|week|month|year)s?\b").unwrap() }) } -fn ingredient_context_re() -> &'static Regex { +fn every_unit_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?ix)\b(?:ingredients?|recipes?|cooking|cook(?:ing|ed)?|baking|baked|meals?|dinner|lunch|breakfast|dish(?:es)?|served?)\b", - ) - .unwrap() - }) -} - -fn is_entity_noise(raw: &str) -> bool { - let normalized = raw.trim().to_ascii_lowercase(); - if crate::nl::get_stopwords().contains(normalized.as_str()) { - return true; - } - matches!( - normalized.as_str(), - "i" | "the" - | "a" - | "an" - | "this" - | "that" - | "here" - | "there" - | "what" - | "when" - | "where" - | "which" - | "who" - | "whom" - | "whose" - | "why" - | "how" - | "do" - | "does" - | "did" - | "can" - | "could" - | "should" - | "would" - | "will" - | "please" - | "tell" - | "user" - | "assistant" - | "system" - | "human" - | "bot" - | "agent" - ) + RE.get_or_init(|| Regex::new(r"(?i)\bevery\s+(?P<unit>day|week|month|year)\b").unwrap()) } -fn is_temporal_month_entity(content: &str, raw: &str) -> bool { - let month = raw.trim().to_ascii_lowercase(); - if raw.split_whitespace().count() != 1 || month_name_number(&month).is_none() { - return false; - } - - let tokens = crate::nl::normalize_text(content) - .split_whitespace() - .map(str::to_string) - .collect::<Vec<_>>(); - for (index, token) in tokens.iter().enumerate() { - if token != &month { - continue; - } - let temporal_context = index - .checked_sub(1) - .and_then(|previous| tokens.get(previous)) - .map(|previous| { - matches!( - previous.as_str(), - "after" - | "before" - | "by" - | "during" - | "for" - | "from" - | "in" - | "since" - | "the" - | "through" - | "until" - ) - }) - .unwrap_or(false); - let dated_suffix = tokens - .get(index + 1) - .map(|next| next.parse::<u32>().is_ok()) - .unwrap_or(false); - if temporal_context || dated_suffix { - return true; - } - } - - false +fn metadata_date_re() -> &'static Regex { + static RE: OnceLock<Regex> = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"\b(?P<year>\d{4})[-/](?P<month>\d{1,2})[-/](?P<day>\d{1,2})\b").unwrap()) } -fn product_re() -> &'static Regex { +fn prefix_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new(r"\b[A-Za-z]{1,12}[- ]?[A-Z]?\d[A-Za-z0-9-]{1,12}(?:\s+[A-Z]{1,4})?\b").unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?m)^\s*([A-Za-z][A-Za-z0-9_-]{1,32})\s*:\s+").unwrap()) } -fn list_item_re() -> &'static Regex { +fn quoted_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"^\d+[\.)]\s+").unwrap()) + RE.get_or_init(|| Regex::new(r#""([^"\n]{2,80})"|'([^'\n]{2,80})'"#).unwrap()) } -fn inline_list_item_re() -> &'static Regex { +fn proper_noun_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| Regex::new(r"(?:^|\s)\d{1,2}[\.)]\s+\S").unwrap()) + RE.get_or_init(|| Regex::new(r"\b[A-Z][a-zA-Z0-9]*(?:\s+[A-Z][a-zA-Z0-9]*){0,3}\b").unwrap()) } -fn quantity_unit_object_re() -> &'static Regex { +fn product_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?i)\b(?P<value>\d+(?:[.,]\d+)?)\s*[- ]\s*(?P<unit>[A-Za-z][A-Za-z]{1,20})\s+(?P<object>[A-Za-z][A-Za-z'-]{2,}(?:\s+[A-Za-z][A-Za-z'-]{2,}){0,3})", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"\b[A-Za-z]{1,12}[- ]?[A-Z]?\d[A-Za-z0-9-]{1,12}(?:\s+[A-Z]{1,4})?\b").unwrap()) } -fn quantity_object_re() -> &'static Regex { +fn list_item_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?i)\b(?P<value>\d+(?:[.,]\d+)?)\s+(?P<object>[A-Za-z][A-Za-z'-]{2,}(?:\s+[A-Za-z][A-Za-z'-]{2,}){0,3})", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"^\d+[\.)]\s+").unwrap()) } -fn contained_singular_object_re() -> &'static Regex { +fn inline_list_item_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?i)\b(?:has|have|contains|contain|includes|include)\s+(?:my|our|a|an|one)\s+(?P<object>[A-Za-z][A-Za-z'-]{2,}(?:\s+[A-Za-z][A-Za-z'-]{2,}){0,3})", - ) - .unwrap() - }) + RE.get_or_init(|| Regex::new(r"(?:^|\s)\d{1,2}[\.)]\s+\S").unwrap()) } -fn completed_count_object_re() -> &'static Regex { +fn temporal_event_relation_re() -> &'static Regex { static RE: OnceLock<Regex> = OnceLock::new(); RE.get_or_init(|| { - Regex::new( - r"(?ix) - \b(?:completed|finished|passed|took|taken)\s+ - (?P<value>a|an|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|\d+)\s+ - (?P<object>[A-Za-z][A-Za-z'-]{2,}(?:\s+[A-Za-z][A-Za-z'-]{2,}){0,3}) - ", - ) - .unwrap() + Regex::new(r"(?is)\b(?P<relation>after|before)\s+(?:the\s+)?(?P<anchor>[A-Za-z0-9][A-Za-z0-9'_-]*(?:\s+[A-Za-z0-9][A-Za-z0-9'_-]*){0,3})").unwrap() }) } @@ -703,1175 +368,895 @@ fn normalize_value(value: &str) -> Option<String> { } } -fn is_role_phrase_connector(part: &str) -> bool { - matches!(part, "and" | "for" | "in" | "of" | "to") +fn normalized_numeric_value(value: &str) -> Option<String> { + let lowercase = value.to_ascii_lowercase(); + let normalized = match lowercase.as_str() { + "one" => "1", + "two" => "2", + "three" => "3", + "four" => "4", + "five" => "5", + "six" => "6", + "seven" => "7", + "eight" => "8", + "nine" => "9", + "ten" => "10", + "eleven" => "11", + "twelve" => "12", + value => value, + }; + normalize_value(normalized).or_else(|| { + (normalized.len() == 1 && normalized.chars().all(|character| character.is_ascii_digit())) + .then(|| normalized.to_string()) + }) } -fn normalize_role_phrase(value: &str) -> Option<String> { - let mut parts = Vec::new(); - - for raw in value.split_whitespace() { - let part = raw - .trim_matches(|c: char| !c.is_alphanumeric()) - .to_ascii_lowercase(); - if part.len() < 2 { - return None; - } - parts.push(part); +fn canonical_quantity_unit(unit: &str) -> Option<&'static str> { + match unit.trim().to_ascii_lowercase().replace('°', "").as_str() { + "microsecond" | "microseconds" | "us" | "μs" | "µs" => Some("us"), + "millisecond" | "milliseconds" | "ms" => Some("ms"), + "nanosecond" | "nanoseconds" | "ns" => Some("ns"), + "second" | "seconds" | "sec" | "s" => Some("s"), + "minute" | "minutes" | "min" => Some("min"), + "hour" | "hours" | "hr" | "h" => Some("h"), + "day" | "days" | "d" => Some("d"), + "byte" | "bytes" | "b" => Some("b"), + "kilobyte" | "kilobytes" | "kb" => Some("kb"), + "megabyte" | "megabytes" | "mb" => Some("mb"), + "gigabyte" | "gigabytes" | "gb" => Some("gb"), + "terabyte" | "terabytes" | "tb" => Some("tb"), + "kilogram" | "kilograms" | "kg" => Some("kg"), + "gram" | "grams" | "g" => Some("g"), + "milligram" | "milligrams" | "mg" => Some("mg"), + "pound" | "pounds" | "lb" | "lbs" => Some("lb"), + "degree c" | "degrees c" | "celsius" | "c" => Some("celsius"), + "degree f" | "degrees f" | "fahrenheit" | "f" => Some("fahrenheit"), + "kelvin" | "k" => Some("kelvin"), + _ => None, } +} - if parts.is_empty() || parts.len() > 4 { - return None; - } +fn normalized_identifier(value: &str) -> Option<String> { + normalize_value(value.trim_matches(|character: char| { + !character.is_ascii_alphanumeric() + })) +} - for (index, part) in parts.iter().enumerate() { - if !crate::nl::get_stopwords().contains(part.as_str()) { - continue; - } - let internal_connector = - index > 0 && index + 1 < parts.len() && is_role_phrase_connector(part); - if !internal_connector { - return None; - } +fn push_unique(out: &mut Vec<String>, seen: &mut HashSet<String>, cue: impl Into<String>) { + if out.len() >= MAX_FACETS { + return; } - - let phrase = parts.join("_"); - if phrase.len() < 3 || phrase.len() > 64 { - None - } else { - Some(phrase) - } -} - -fn normalize_person_role_phrase(value: &str) -> Option<String> { - let role = normalize_role_phrase(value)?; - let tail = role.rsplit('_').next()?; - let lemma = crate::nl::stem_word(tail); - - // A role phrase should end in its nominal head. If lemmatization changes the - // final token, the candidate is more likely a clause ending in an inflected - // verb ("my friend saw Dr. Patel") than a role immediately before a name. - let known_noun = matches!( - crate::nl::get_known_pos_tag(tail), - Some("NN" | "NNS" | "NNP" | "NNPS") - ); - let connector_scoped_plural = role.split('_').any(is_role_phrase_connector) - && (tail == format!("{lemma}s") - || tail == format!("{lemma}es") - || (tail.ends_with("ies") - && lemma.ends_with('y') - && tail.strip_suffix("ies") == lemma.strip_suffix('y'))); - if lemma != tail && !known_noun && !connector_scoped_plural { - return None; + let cue = cue.into(); + if seen.insert(cue.to_lowercase()) { + out.push(cue); } - - Some(role) } -fn quantity_object_stopwords() -> &'static HashSet<&'static str> { - static STOPWORDS: OnceLock<HashSet<&'static str>> = OnceLock::new(); - STOPWORDS.get_or_init(|| { - HashSet::from([ - "a", "an", "and", "ago", "am", "at", "by", "day", "days", "dollar", "dollars", "for", - "from", "episode", "episodes", "gallon", "gallons", "hour", "hours", "in", "inch", - "inches", "meter", "meters", "minute", "minutes", "month", "months", "named", "of", - "on", "or", "percent", "pm", "season", "seasons", "second", "seconds", "the", "to", - "week", "weeks", "with", "year", "years", - ]) +fn metadata_string<'a>(metadata: &'a HashMap<String, Value>, keys: &[&str]) -> Option<&'a str> { + keys.iter().find_map(|key| { + metadata.get(*key).and_then(Value::as_str).filter(|value| !value.trim().is_empty()) }) } -fn inventory_object_stopwords() -> &'static HashSet<&'static str> { - static STOPWORDS: OnceLock<HashSet<&'static str>> = OnceLock::new(); - STOPWORDS.get_or_init(|| { - let mut stopwords = quantity_object_stopwords().clone(); - stopwords.extend([ - "few", "issue", "issues", "level", "levels", "lot", "lots", "many", "more", "new", - "old", "problem", "problems", "some", - ]); - stopwords - }) +fn parse_date_text(raw: &str) -> Option<NaiveDate> { + let cap = metadata_date_re().captures(raw)?; + let year = cap.name("year")?.as_str().parse::<i32>().ok()?; + let month = cap.name("month")?.as_str().parse::<u32>().ok()?; + let day = cap.name("day")?.as_str().parse::<u32>().ok()?; + NaiveDate::from_ymd_opt(year, month, day) } -fn normalize_quantity_token(raw: &str) -> Option<String> { - let normalized = normalize_value(raw)?; - if quantity_object_stopwords().contains(normalized.as_str()) { - return None; - } - let stemmed = crate::nl::stem_word(&normalized); - if stemmed.len() < 2 || quantity_object_stopwords().contains(stemmed.as_str()) { - None - } else { - Some(stemmed) +fn parse_date_value(value: &Value) -> Option<NaiveDate> { + if let Some(raw) = value.as_str() { + return parse_date_text(raw); } + let seconds = value.as_f64()?; + let days = (seconds / 86_400.0).floor() as i64; + NaiveDate::from_ymd_opt(1970, 1, 1)?.checked_add_signed(Duration::days(days)) } -fn normalize_quantity_unit(raw: &str) -> Option<String> { - let normalized = normalize_value(raw)?; - let stemmed = crate::nl::stem_word(&normalized); - if stemmed.len() < 2 { - None - } else { - Some(stemmed) - } +fn metadata_date(metadata: &HashMap<String, Value>) -> Option<NaiveDate> { + [ + "source_date", + "source_timestamp", + "timestamp", + "created_at", + "datetime", + "date", + ] + .iter() + .find_map(|key| metadata.get(*key).and_then(parse_date_value)) } -fn quantity_object_tokens(phrase: &str) -> Vec<String> { - let mut tokens = Vec::new(); - let mut seen = HashSet::new(); - - for raw in phrase.split_whitespace().take(4) { - let normalized = normalize_value(raw); - if normalized - .as_deref() - .map(|token| quantity_object_stopwords().contains(token)) - .unwrap_or(false) - { - break; - } +fn source_date_facet(date: NaiveDate) -> String { + format!("source_date:{:04}_{:02}_{:02}", date.year(), date.month(), date.day()) +} - let Some(token) = normalize_quantity_token(raw) else { - break; - }; - if seen.insert(token.clone()) { - tokens.push(token); - } - } +fn source_week_facet(date: NaiveDate) -> String { + let week = date.iso_week(); + format!("source_week:{:04}_w{:02}", week.year(), week.week()) +} - tokens +fn source_month_facet(date: NaiveDate) -> String { + format!("source_month:{:04}_{:02}", date.year(), date.month()) } -fn inventory_object_tokens(phrase: &str) -> Vec<String> { - let mut tokens = Vec::new(); - let mut seen = HashSet::new(); - let stopwords = inventory_object_stopwords(); +fn source_year_facet(date: NaiveDate) -> String { + format!("source_year:{:04}", date.year()) +} - for raw in phrase.split_whitespace().take(5) { - let Some(token) = normalize_quantity_token(raw) else { - continue; - }; - if stopwords.contains(token.as_str()) { - continue; - } - if seen.insert(token.clone()) { - tokens.push(token); - } +fn month_name_number(token: &str) -> Option<u32> { + match token.to_ascii_lowercase().as_str() { + "jan" | "january" => Some(1), + "feb" | "february" => Some(2), + "mar" | "march" => Some(3), + "apr" | "april" => Some(4), + "may" => Some(5), + "jun" | "june" => Some(6), + "jul" | "july" => Some(7), + "aug" | "august" => Some(8), + "sep" | "sept" | "september" => Some(9), + "oct" | "october" => Some(10), + "nov" | "november" => Some(11), + "dec" | "december" => Some(12), + _ => None, } +} - tokens +fn surface_tokens(value: &str) -> Vec<String> { + value + .split(|ch: char| !ch.is_alphanumeric()) + .filter(|token| !token.is_empty()) + .map(str::to_string) + .collect() } -fn numeric_value_looks_like_year(value: &str) -> bool { - let integer = value.split(['.', ',']).next().unwrap_or(value); - integer.len() == 4 - && integer - .parse::<u16>() - .map(|year| (1900..=2100).contains(&year)) - .unwrap_or(false) +fn query_tokens(value: &str) -> Vec<String> { + surface_tokens(value) + .into_iter() + .map(|token| token.to_ascii_lowercase()) + .collect() } -fn numeric_value_looks_like_age(value: &str) -> bool { - value - .split(['.', ',']) - .next() - .unwrap_or(value) - .parse::<u16>() - .map(|age| (1..=130).contains(&age)) - .unwrap_or(false) +fn is_numeric_date_token(token: &str) -> bool { + let lower = token.to_ascii_lowercase(); + let digits = lower + .strip_suffix("st") + .or_else(|| lower.strip_suffix("nd")) + .or_else(|| lower.strip_suffix("rd")) + .or_else(|| lower.strip_suffix("th")) + .unwrap_or(&lower); + !digits.is_empty() && digits.chars().all(|ch| ch.is_ascii_digit()) } -fn has_valid_age_match(regex: &Regex, content: &str) -> bool { - regex.captures_iter(content).any(|cap| { - cap.name("age") - .map(|m| numeric_value_looks_like_age(m.as_str())) - .unwrap_or(false) - }) +fn is_temporal_context_word(token: &str) -> bool { + matches!( + token.to_ascii_lowercase().as_str(), + "in" + | "on" + | "by" + | "since" + | "during" + | "from" + | "until" + | "before" + | "after" + | "around" + | "near" + | "last" + | "next" + | "early" + | "late" + | "mid" + ) } -fn add_numeric_object_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - for cap in quantity_unit_object_re().captures_iter(content) { - if cap - .name("value") - .map(|m| numeric_value_looks_like_year(m.as_str())) - .unwrap_or(false) - { - continue; - } +fn is_pronoun_or_auxiliary(token: &str) -> bool { + matches!( + token.to_ascii_lowercase().as_str(), + "i" + | "you" + | "he" + | "she" + | "we" + | "they" + | "it" + | "me" + | "him" + | "her" + | "us" + | "them" + | "this" + | "that" + | "these" + | "those" + | "am" + | "are" + | "is" + | "was" + | "were" + | "be" + | "been" + | "being" + | "do" + | "does" + | "did" + | "have" + | "has" + | "had" + | "can" + | "could" + | "may" + | "might" + | "must" + | "shall" + | "should" + | "will" + | "would" + ) +} - let Some(unit) = cap - .name("unit") - .and_then(|m| normalize_quantity_unit(m.as_str())) - else { - continue; - }; - let object_tokens = cap - .name("object") - .map(|m| quantity_object_tokens(m.as_str())) - .unwrap_or_default(); - if object_tokens.is_empty() { - continue; - } +fn known_pos_tag(token: &str) -> Option<&'static str> { + crate::nl::get_known_pos_tag(token) + .or_else(|| crate::nl::get_known_pos_tag(&token.to_ascii_lowercase())) +} - push_unique(out, seen, format!("quantity_unit:{}", unit)); - for object in &object_tokens { - push_unique(out, seen, format!("quantity_object:{}", object)); - } - if let Some(head) = object_tokens.last() { - push_unique(out, seen, format!("quantity_unit_object:{}_{}", unit, head)); - } - } +fn is_noun_or_adjective(token: Option<&String>) -> bool { + token + .and_then(|token| known_pos_tag(token)) + .is_some_and(|tag| matches!(tag, "NN" | "NNS" | "NNP" | "NNPS" | "JJ" | "JJR" | "JJS")) +} - for cap in quantity_object_re().captures_iter(content) { - if cap - .name("value") - .map(|m| numeric_value_looks_like_year(m.as_str())) - .unwrap_or(false) - { - continue; - } +fn is_strong_preceding_temporal_context(token: &str) -> bool { + is_temporal_context_word(token) +} - let object_tokens = cap - .name("object") - .map(|m| quantity_object_tokens(m.as_str())) - .unwrap_or_default(); - if !object_tokens.is_empty() { - push_unique(out, seen, "quantity_count:object"); - let preceding = &content[..cap.get(0).map(|m| m.start()).unwrap_or(0)]; - let preceding_window = preceding - .chars() - .rev() - .take(80) - .collect::<String>() - .chars() - .rev() - .collect::<String>() - .to_lowercase(); - if has_any( - &format!(" {} ", preceding_window), - &[ - " has ", - " have ", - " contains ", - " contain ", - " includes ", - " include ", - ], - ) { - push_unique(out, seen, "inventory_count:contained"); - } - } - for object in object_tokens { - push_unique(out, seen, format!("quantity_object:{}", object)); - } - } +fn is_ambiguous_month(month: u32) -> bool { + matches!(month, 3 | 4 | 5 | 6 | 8) +} - for cap in contained_singular_object_re().captures_iter(content) { - let object_tokens = cap - .name("object") - .map(|m| quantity_object_tokens(m.as_str())) - .unwrap_or_default(); - if object_tokens.is_empty() { - continue; - } - push_unique(out, seen, "quantity_count:object"); - push_unique(out, seen, "inventory_count:contained"); - for object in object_tokens { - push_unique(out, seen, format!("quantity_object:{}", object)); - } +fn classify_capitalized_ambiguous_month( + previous: Option<&String>, + next: Option<&String>, +) -> bool { + if previous.is_some_and(|token| is_strong_preceding_temporal_context(token)) + || previous.is_some_and(|token| is_numeric_date_token(token)) + || next.is_some_and(|token| is_numeric_date_token(token)) + { + return true; } - for cap in completed_count_object_re().captures_iter(content) { - if cap - .name("value") - .and_then(|m| small_number_value(&m.as_str().to_ascii_lowercase())) - .is_none() - { - continue; - } - let object_tokens = cap - .name("object") - .map(|m| quantity_object_tokens(m.as_str())) - .unwrap_or_default(); - if object_tokens.is_empty() { - continue; - } - push_unique(out, seen, "quantity_count:object"); - push_unique(out, seen, "completion_count:object"); - for object in object_tokens { - push_unique(out, seen, format!("quantity_object:{}", object)); - } + // A capitalized month at the end of a sentence is normally a date. When + // followed by a known verb/auxiliary/preposition, it is more likely to be + // a name, command, or adjective ("March toward...", "June joined..."). + if next.is_none() { + return true; } -} - -fn add_inventory_object_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let lower = format!(" {} ", content.to_lowercase()); - let has_ownership_signal = first_person_acquired_re().is_match(&lower) - || first_person_possession_re().is_match(&lower) - || owned_object_re().is_match(content); - - for cap in possessive_object_re().captures_iter(content) { - let object_tokens = cap - .name("object") - .map(|m| inventory_object_tokens(m.as_str())) - .unwrap_or_default(); - for object in object_tokens { - push_unique(out, seen, format!("inventory_object:{}", object)); - } + if next.is_some_and(|token| is_pronoun_or_auxiliary(token)) { + return false; } - - for cap in owned_object_re().captures_iter(content) { - let object_tokens = owned_object_capture_text(&cap) - .map(inventory_object_tokens) - .unwrap_or_default(); - for object in object_tokens { - push_unique(out, seen, format!("inventory_object:{}", object)); - } + if next + .and_then(|token| known_pos_tag(token)) + .is_some_and(|tag| matches!(tag, "VB" | "VBD" | "VBG" | "VBN" | "VBP" | "VBZ" | "MD" | "IN" | "TO" | "RB")) + { + return false; } - if has_ownership_signal { - for cap in quantity_unit_object_re().captures_iter(content) { - let object_tokens = cap - .name("object") - .map(|m| inventory_object_tokens(m.as_str())) - .unwrap_or_default(); - for object in object_tokens { - push_unique(out, seen, format!("inventory_object:{}", object)); - } - } + is_noun_or_adjective(next) +} - for cap in quantity_object_re().captures_iter(content) { - let object_tokens = cap - .name("object") - .map(|m| inventory_object_tokens(m.as_str())) - .unwrap_or_default(); - for object in object_tokens { - push_unique(out, seen, format!("inventory_object:{}", object)); - } +fn is_temporal_month_at(tokens: &[String], index: usize) -> bool { + let Some(month) = month_name_number(&tokens[index]) else { + return false; + }; + let previous = index.checked_sub(1).and_then(|index| tokens.get(index)); + let next = tokens.get(index + 1); + let raw = &tokens[index]; + let capitalized = raw.chars().next().is_some_and(char::is_uppercase); + + // Lowercase ambiguous month names are only accepted after explicit + // temporal syntax. This prevents "It may in fact work" and "Version 5 + // may fail" from becoming calendar evidence. + if !capitalized { + if month == 5 { + return previous.is_some_and(|token| is_strong_preceding_temporal_context(token)); + } + if is_ambiguous_month(month) { + return previous.is_some_and(|token| is_strong_preceding_temporal_context(token)) + || next.is_some_and(|token| is_numeric_date_token(token)); } + return true; } -} -fn add_age_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let has_year_old = has_valid_age_match(age_year_old_re(), content); - let has_current_age = has_valid_age_match(current_age_re(), content); - let has_event_age = has_valid_age_match(event_age_re(), content); - - if has_year_old || has_current_age || has_event_age { - push_unique(out, seen, "has:age"); - } - if has_current_age { - push_unique(out, seen, "age:current"); + if month == 5 && known_pos_tag(raw) == Some("MD") { + return false; } - if has_event_age { - push_unique(out, seen, "age:event"); + if next.is_some_and(|token| is_pronoun_or_auxiliary(token)) { + return false; } - if has_year_old && !has_current_age { - push_unique(out, seen, "age:mentioned"); + + if !is_ambiguous_month(month) { + return true; } + + classify_capitalized_ambiguous_month(previous, next) } -fn add_education_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let lower = format!(" {} ", content.to_lowercase()); - let undergraduate = has_any( - &lower, - &[ - " undergraduate", - " undergrad", - " bachelor's", - " bachelors", - " bachelor ", - ], - ); - let degree = has_any( - &lower, - &[ - " degree", - " bachelor's", - " bachelors", - " bachelor ", - " undergraduate", - " undergrad", - " master's", - " masters", - " master ", - " mba", - " ph.d", - " phd", - " doctorate", - " diploma", - " associate degree", - ], - ); - let institution = has_any(&lower, &[" college", " university", " school"]); - let completion = has_any( - &lower, - &[ - " graduated", - " graduation", - " completed", - " finished", - " earned", - " received", - ], - ); - let graduation = has_any( - &lower, - &[ - " graduated from", - " graduation from", - " finished college", - " finished university", - " completed college", - " completed university", - ], - ) || (degree && completion); +fn has_temporal_month(content: &str) -> bool { + let tokens = surface_tokens(content); + tokens + .iter() + .enumerate() + .any(|(index, _)| is_temporal_month_at(&tokens, index)) +} - if degree { - push_unique(out, seen, "education:degree"); - } - if undergraduate { - push_unique(out, seen, "education:undergraduate"); - } - if institution { - push_unique(out, seen, "education:college"); - } - if graduation { - push_unique(out, seen, "education:graduation"); - } +fn is_entity_noise(raw: &str) -> bool { + let normalized = raw.trim().to_ascii_lowercase(); + crate::nl::get_stopwords().contains(normalized.as_str()) + || matches!( + normalized.as_str(), + "i" | "the" | "a" | "an" | "this" | "that" | "here" | "there" | "what" + | "when" | "where" | "which" | "who" | "whom" | "whose" | "why" | "how" + | "do" | "does" | "did" | "can" | "could" | "should" | "would" | "will" + | "please" | "tell" | "user" | "assistant" | "system" | "human" | "bot" + | "agent" + ) } -fn family_relation_for_token(token: &str) -> Option<(&'static str, Option<&'static str>)> { - match token { - "brother" | "brothers" => Some(("sibling", Some("brother"))), - "sister" | "sisters" => Some(("sibling", Some("sister"))), - "sibling" | "siblings" => Some(("sibling", None)), - "mother" | "mom" | "mum" | "father" | "dad" | "parent" | "parents" => { - Some(("parent", None)) - } - "son" | "sons" | "daughter" | "daughters" | "child" | "children" | "kid" - | "kids" => Some(("child", None)), - "husband" | "wife" | "spouse" => Some(("spouse", None)), - "cousin" | "cousins" => Some(("cousin", None)), - "aunt" | "aunts" | "uncle" | "uncles" => Some(("aunt_uncle", None)), - "niece" | "nieces" | "nephew" | "nephews" => Some(("niece_nephew", None)), - "grandmother" | "grandma" | "grandfather" | "grandpa" | "grandparent" - | "grandparents" => Some(("grandparent", None)), - _ => None, +fn is_temporal_month_entity(content: &str, raw: &str) -> bool { + if raw.split_whitespace().count() != 1 || month_name_number(raw.trim()).is_none() { + return false; } + let tokens = surface_tokens(content); + tokens.iter().enumerate().any(|(index, token)| { + token.eq_ignore_ascii_case(raw.trim()) && is_temporal_month_at(&tokens, index) + }) } -fn add_family_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let lower = content.to_lowercase(); - let tokens = query_tokens(&lower); - let mut saw_sibling = false; +fn add_metadata_temporal_facets( + metadata: Option<&HashMap<String, Value>>, + out: &mut Vec<String>, + seen: &mut HashSet<String>, +) { + let Some(date) = metadata.and_then(metadata_date) else { + return; + }; + push_unique(out, seen, "source_time:dated"); + push_unique(out, seen, source_date_facet(date)); + push_unique(out, seen, source_week_facet(date)); + push_unique(out, seen, source_month_facet(date)); + push_unique(out, seen, source_year_facet(date)); +} - for token in tokens { - let Some((relation, kind)) = family_relation_for_token(token.as_str()) else { - continue; - }; - push_unique(out, seen, format!("family_relation:{}", relation)); - if relation == "sibling" { - saw_sibling = true; - } - if let Some(kind) = kind { - if relation == "sibling" { - push_unique(out, seen, format!("sibling_kind:{}", kind)); +fn add_source_facets( + content: &str, + metadata: Option<&HashMap<String, Value>>, + existing_cues: &[String], + out: &mut Vec<String>, + seen: &mut HashSet<String>, +) { + if let Some(metadata) = metadata { + for (keys, target) in [ + (&["source_role", "role", "speaker", "author_role"][..], "source_role"), + (&["source_channel", "channel", "conversation", "room"][..], "source_channel"), + (&["source_type", "source", "kind"][..], "source_type"), + (&["source_session_id", "session_id", "conversation_id", "thread_id"][..], "source_session"), + ] { + if let Some(value) = metadata_string(metadata, keys).and_then(normalize_value) { + push_unique(out, seen, format!("{target}:{value}")); } } } - if saw_sibling && self_sibling_re().is_match(content) { - push_unique(out, seen, "family_scope:self"); - } - if saw_sibling && sibling_count_re().is_match(content) { - push_unique(out, seen, "family_count:sibling"); - push_unique(out, seen, "family_scope:self"); - } -} - -fn family_relation_query_facets(query: &str) -> Vec<String> { - let mut facets = Vec::new(); - let mut seen = HashSet::new(); - for token in query_tokens(query) { - let Some((relation, kind)) = family_relation_for_token(token.as_str()) else { + for cue in existing_cues { + let Some((key, value)) = cue.split_once(':') else { continue; }; - let relation_facet = format!("family_relation:{}", relation); - if seen.insert(relation_facet.clone()) { - facets.push(relation_facet); - } - if relation == "sibling" { - if let Some(kind) = kind { - let kind_facet = format!("sibling_kind:{}", kind); - if seen.insert(kind_facet.clone()) { - facets.push(kind_facet); - } + let target = match key { + "role" | "speaker" => Some("source_role"), + "channel" => Some("source_channel"), + "source" | "category" => Some("source_type"), + "session" | "conversation" | "thread" => Some("source_session"), + _ => None, + }; + if let Some(target) = target { + if let Some(value) = normalize_value(value) { + push_unique(out, seen, format!("{target}:{value}")); } } } - facets -} -fn push_unique(out: &mut Vec<String>, seen: &mut HashSet<String>, cue: impl Into<String>) { - if out.len() >= MAX_FACETS { - return; - } - let cue = cue.into(); - if cue.is_empty() { - return; + if let Some(value) = prefix_re().captures(content).and_then(|cap| cap.get(1)).and_then(|m| normalize_value(m.as_str())) { + push_unique(out, seen, format!("source_role:{value}")); } - let key = cue.to_lowercase(); - if seen.insert(key) { - out.push(cue); +} + +fn clock_hour_24(hour: u32, meridiem: &str) -> u32 { + let meridiem = meridiem.to_ascii_lowercase(); + if meridiem.starts_with('p') { + if hour == 12 { 12 } else { hour + 12 } + } else if hour == 12 { + 0 + } else { + hour } } -fn standing_always_when_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\balways\s+(?P<action>.{3,220}?)\s+when\s+(?:i|we|the user|users?|someone|people)\s+(?:ask|asks|asked|am asking|are asking)\s+about\s+(?P<trigger>.{3,220})(?:[.!?]|$)", - ) - .unwrap() - }) -} - -fn standing_when_always_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\bwhen\s+(?:i|we|the user|users?|someone|people)\s+(?:ask|asks|asked|am asking|are asking)\s+about\s+(?P<trigger>.{3,220}?)\s*,?\s+(?:always|please|make sure to|remember to)\s+(?P<action>.{3,220})(?:[.!?]|$)", - ) - .unwrap() - }) -} - -fn standing_make_sure_when_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?:make sure to|remember to|please)\s+(?P<action>.{3,220}?)\s+when\s+(?:discussing|covering|answering|responding to|talking about|i\s+ask\s+about|we\s+ask\s+about)\s+(?P<trigger>.{3,220})(?:[.!?]|$)", - ) - .unwrap() - }) +fn time_of_day_for_hour(hour: u32) -> &'static str { + match hour { + 5..=11 => "morning", + 12..=16 => "afternoon", + 17..=21 => "evening", + _ => "night", + } } -fn standing_instruction_clause_trim(clause: &str) -> String { - clause - .split("->->") - .next() - .unwrap_or(clause) - .split("```") - .next() - .unwrap_or(clause) - .lines() - .next() - .unwrap_or(clause) - .trim_matches(|ch: char| ch.is_whitespace() || ch == '"' || ch == '\'' || ch == ',' || ch == ';' || ch == ':') - .trim() - .to_string() +fn add_time_of_day_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let lower = format!(" {} ", content.to_ascii_lowercase()); + for (term, facet) in [ + (" morning ", "morning"), + (" afternoon ", "afternoon"), + (" evening ", "evening"), + (" tonight ", "evening"), + (" night ", "night"), + (" bedtime ", "night"), + ] { + if lower.contains(term) { + push_unique(out, seen, format!("time_of_day:{facet}")); + } + } + for cap in clock_time_capture_re().captures_iter(content) { + let Some(hour) = cap.name("hour").and_then(|m| m.as_str().parse::<u32>().ok()) else { + continue; + }; + let Some(meridiem) = cap.name("meridiem").map(|m| m.as_str()) else { + continue; + }; + push_unique(out, seen, format!("time_of_day:{}", time_of_day_for_hour(clock_hour_24(hour, meridiem)))); + } + for cap in clock_time_24_capture_re().captures_iter(content) { + let Some(hour) = cap.name("hour").and_then(|m| m.as_str().parse::<u32>().ok()) else { + continue; + }; + push_unique(out, seen, format!("time_of_day:{}", time_of_day_for_hour(hour))); + } } -fn standing_instruction_cue_is_generic(cue: &str) -> bool { - if cue.starts_with("type:") - || cue.starts_with("has:") - || cue.starts_with("temporal:") - || cue.starts_with("source_") - { - return true; +fn add_cadence_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let mut matched = false; + for unit in cadence_re().captures_iter(content).chain(every_unit_re().captures_iter(content)) { + let Some(unit) = unit.name("unit").and_then(|m| normalize_value(m.as_str())) else { + continue; + }; + matched = true; + push_unique(out, seen, "has:frequency"); + push_unique(out, seen, "schedule:frequency"); + push_unique(out, seen, format!("frequency_unit:{unit}")); + if unit == "week" { + push_unique(out, seen, "schedule:weekly"); + } } - - if cue.contains('_') { - let parts = cue.split('_').collect::<Vec<_>>(); - return parts.iter().all(|part| standing_instruction_cue_is_generic(part)); + if matched && has_clock_time(content) { + push_unique(out, seen, "has:time"); } - - crate::nl::get_stopwords().contains(cue) - || matches!( - cue, - "always" - | "ask" - | "asked" - | "asking" - | "about" - | "when" - | "provide" - | "include" - | "specify" - | "confirm" - | "explain" - | "remember" - | "make" - | "sure" - | "please" - | "tell" - | "show" - | "help" - | "advice" - | "thing" - | "things" - | "way" - | "ways" - ) } -fn add_standing_instruction_clause_cues( - prefix: &str, - clause: &str, - limit: usize, - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - let clause = standing_instruction_clause_trim(clause); - if clause.len() < 3 { - return; +fn add_quantity_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + for caps in measurement_re().captures_iter(content) { + let Some(value) = caps + .name("value") + .and_then(|value| normalized_numeric_value(value.as_str())) + else { + continue; + }; + let Some(unit) = caps + .name("unit") + .and_then(|unit| canonical_quantity_unit(unit.as_str())) + else { + continue; + }; + push_unique(out, seen, "has:measurement"); + push_unique(out, seen, format!("quantity_unit:{unit}")); + push_unique(out, seen, format!("measurement:{value}_{unit}")); } - let mut emitted = HashSet::new(); - for cue in crate::nl::tokenize_to_cues(&clause) { - let cue = cue.trim().to_lowercase(); - if cue.len() <= 2 - || standing_instruction_cue_is_generic(&cue) - || !emitted.insert(cue.clone()) - { + for caps in percentage_re().captures_iter(content) { + let Some(value) = caps + .name("value") + .and_then(|value| normalized_numeric_value(value.as_str())) + else { continue; - } - push_unique(out, seen, format!("{prefix}:{cue}")); - if emitted.len() >= limit { - break; - } + }; + push_unique(out, seen, "has:percentage"); + push_unique(out, seen, format!("percentage:{value}")); } -} -fn add_standing_instruction_dynamic_facets( - content: &str, - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - let patterns = [ - standing_always_when_re(), - standing_when_always_re(), - standing_make_sure_when_re(), - ]; - - for re in patterns { - let Some(caps) = re.captures(content) else { + for caps in between_range_re() + .captures_iter(content) + .chain(from_range_re().captures_iter(content)) + { + let Some(min) = caps + .name("min") + .and_then(|value| normalized_numeric_value(value.as_str())) + else { continue; }; - if let Some(trigger) = caps.name("trigger") { - add_standing_instruction_clause_cues( - "instruction_trigger", - trigger.as_str(), - 10, - out, - seen, - ); - } - if let Some(action) = caps.name("action") { - add_standing_instruction_clause_cues( - "instruction_action", - action.as_str(), - 8, - out, - seen, - ); + let Some(max) = caps + .name("max") + .and_then(|value| normalized_numeric_value(value.as_str())) + else { + continue; + }; + push_unique(out, seen, "has:numeric_range"); + push_unique(out, seen, "has:comparator"); + push_unique(out, seen, "comparison:between"); + push_unique(out, seen, format!("range_min:{min}")); + push_unique(out, seen, format!("range_max:{max}")); + if let Some(unit) = caps + .name("unit") + .and_then(|unit| canonical_quantity_unit(unit.as_str())) + { + push_unique(out, seen, format!("quantity_unit:{unit}")); + push_unique(out, seen, format!("range:{min}_{max}_{unit}")); } - break; } -} - -fn preference_over_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?:i|we)\s+(?:really\s+|usually\s+|generally\s+|strongly\s+)?(?:prefer|like|love|enjoy)\s+(?P<value>.{3,180}?)\s+\b(?:over|rather than|instead of)\b\s+(?P<contrast>.{3,180})(?:[.!?]|$)", - ) - .unwrap() - }) -} - -fn preference_for_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?:i|we)\s+(?:really\s+|usually\s+|generally\s+|strongly\s+)?(?:prefer|like|love|enjoy)\s+(?P<value>.{3,180}?)\s+\b(?:for|when|while|with|in|during)\b\s+(?P<topic>.{3,180})(?:[.!?]|$)", - ) - .unwrap() - }) -} - -fn preference_simple_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?:i|we)\s+(?:really\s+|usually\s+|generally\s+|strongly\s+)?(?:prefer|like|love|enjoy)\s+(?P<value>.{3,180})(?:[.!?]|$)", - ) - .unwrap() - }) -} - -fn preference_rather_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?:i|we)\s+would\s+rather\s+(?P<value>.{3,180})(?:[.!?]|$)", - ) - .unwrap() - }) -} - -fn preference_negative_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?:i|we)\s+(?:do\s+not|don't|dont|dislike|avoid|can't\s+stand|cannot\s+stand)\s+(?P<value>.{3,180})(?:[.!?]|$)", - ) - .unwrap() - }) -} -fn personal_transition_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?s)\b(?P<subject>(?:I|We|[A-Z][A-Za-z'-]{1,40}))\s+(?i:(?:have\s+|has\s+|had\s+)?(?:switched|changed|moved))\s+from\s+(?P<previous>.{1,120}?)\s+to\s+(?P<value>.{1,120}?)(?:\s+\b(?:after|before|during|since|until|when|while|because)\b|[.!?]|$)", - ) - .unwrap() - }) + let lower = content.to_ascii_lowercase(); + let less_than = query_has_any( + &lower, + &["under", "below", "less than", "at most", "no more than"], + ); + let greater_than = query_has_any( + &lower, + &["over", "above", "more than", "at least", "no less than"], + ); + let approximately = query_has_any( + &lower, + &["approximately", "approx", "about", "around", "roughly"], + ); + if less_than || greater_than || approximately { + push_unique(out, seen, "has:comparator"); + } + if less_than { + push_unique(out, seen, "comparison:less_than"); + } + if greater_than { + push_unique(out, seen, "comparison:greater_than"); + } + if approximately { + push_unique(out, seen, "comparison:approximately"); + } } -fn preference_clause_trim(clause: &str) -> String { - clause - .split("->->") - .next() - .unwrap_or(clause) - .split("```") - .next() - .unwrap_or(clause) - .lines() - .next() - .unwrap_or(clause) - .split(|ch| ch == ',' || ch == ';') - .next() - .unwrap_or(clause) - .trim_matches(|ch: char| { - ch.is_whitespace() || ch == '"' || ch == '\'' || ch == ',' || ch == ';' || ch == ':' +fn valid_ip_address(raw: &str) -> bool { + let octets = raw.split('.').collect::<Vec<_>>(); + octets.len() == 4 + && octets.iter().all(|octet| { + octet.parse::<u16>().is_ok_and(|value| value <= 255) }) - .trim() - .to_string() } -fn preference_cue_is_generic(cue: &str) -> bool { - if cue.starts_with("type:") - || cue.starts_with("has:") - || cue.starts_with("temporal:") - || cue.starts_with("source_") - { - return true; +fn add_identifier_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + for matched in uuid_re().find_iter(content) { + let Some(value) = normalized_identifier(matched.as_str()) else { + continue; + }; + push_unique(out, seen, "has:uuid"); + push_unique(out, seen, format!("uuid:{value}")); } - if cue.contains('_') { - let parts = cue.split('_').collect::<Vec<_>>(); - return parts.iter().all(|part| preference_cue_is_generic(part)); + for matched in semver_re().find_iter(content) { + let raw = matched + .as_str() + .strip_prefix('v') + .or_else(|| matched.as_str().strip_prefix('V')) + .unwrap_or(matched.as_str()); + let Some(value) = normalized_identifier(raw) else { + continue; + }; + push_unique(out, seen, "has:semver"); + push_unique(out, seen, format!("version:{value}")); } - crate::nl::get_stopwords().contains(cue) - || matches!( - cue, - "prefer" - | "preference" - | "like" - | "love" - | "enjoy" - | "want" - | "need" - | "rather" - | "over" - | "instead" - | "for" - | "when" - | "while" - | "with" - | "about" - | "because" - | "can" - | "could" - | "would" - | "should" - | "show" - | "tell" - | "help" - | "explain" - | "recommend" - | "suggest" - | "use" - | "using" - | "get" - | "make" - | "thing" - | "things" - | "way" - | "ways" - ) -} - -fn add_preference_clause_cues( - prefix: &str, - clause: &str, - limit: usize, - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - let clause = preference_clause_trim(clause); - if clause.len() < 3 { - return; + for caps in issue_reference_re().captures_iter(content) { + let Some(value) = caps + .name("issue") + .or_else(|| caps.name("hyphen_issue")) + .and_then(|value| normalized_identifier(value.as_str())) + else { + continue; + }; + push_unique(out, seen, "has:issue_reference"); + push_unique(out, seen, format!("issue:{value}")); } - let mut emitted = HashSet::new(); - for cue in crate::nl::tokenize_to_cues(&clause) { - let cue = cue.trim().to_lowercase(); - if cue.len() <= 2 || preference_cue_is_generic(&cue) || !emitted.insert(cue.clone()) { + for matched in ip_address_re().find_iter(content) { + if !valid_ip_address(matched.as_str()) { continue; } - push_unique(out, seen, format!("{prefix}:{cue}")); - if emitted.len() >= limit { - break; - } + let Some(value) = normalized_identifier(matched.as_str()) else { + continue; + }; + push_unique(out, seen, "has:ip_address"); + push_unique(out, seen, format!("ip:{value}")); } -} -fn add_personal_transition_facets( - content: &str, - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - let Some(caps) = personal_transition_re().captures(content) else { - return; - }; - let Some(previous) = caps.name("previous") else { - return; - }; - let Some(value) = caps.name("value") else { - return; - }; + for caps in port_re().captures_iter(content) { + let Some(raw) = caps + .name("port") + .or_else(|| caps.name("host_port")) + .map(|value| value.as_str()) + else { + continue; + }; + let Some(value) = raw.parse::<u32>().ok().filter(|value| *value <= 65_535) else { + continue; + }; + push_unique(out, seen, "has:port"); + push_unique(out, seen, format!("port:{value}")); + } - push_unique(out, seen, "type:update"); - push_unique(out, seen, "type:selection"); - push_unique(out, seen, "type:preference"); - push_unique(out, seen, "preference:changed"); - add_preference_clause_cues("preference_contrast", previous.as_str(), 8, out, seen); - add_preference_clause_cues("preference_value", value.as_str(), 10, out, seen); -} + for matched in domain_re().find_iter(content) { + let Some(value) = normalized_identifier(matched.as_str()) else { + continue; + }; + // The ingestion agent already owns `domain:*`; use a distinct + // namespace for content-derived domain structure. + push_unique(out, seen, "has:domain"); + push_unique(out, seen, format!("domain_name:{value}")); + } -fn add_preference_dynamic_facets( - content: &str, - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - let mut matched = false; + for matched in environment_variable_re().find_iter(content) { + let Some(value) = normalized_identifier(matched.as_str()) else { + continue; + }; + push_unique(out, seen, "has:environment_variable"); + push_unique(out, seen, format!("env:{value}")); + } - if let Some(caps) = preference_over_re().captures(content) { - if let Some(value) = caps.name("value") { - add_preference_clause_cues("preference_value", value.as_str(), 10, out, seen); - } - if let Some(contrast) = caps.name("contrast") { - add_preference_clause_cues("preference_contrast", contrast.as_str(), 8, out, seen); - } - matched = true; + for matched in user_mention_re().find_iter(content) { + let Some(value) = normalized_identifier(matched.as_str().trim_start_matches('@')) else { + continue; + }; + push_unique(out, seen, "has:user_mention"); + push_unique(out, seen, format!("mention:{value}")); } - if let Some(caps) = preference_for_re().captures(content) { - if let Some(value) = caps.name("value") { - add_preference_clause_cues("preference_value", value.as_str(), 10, out, seen); - } - if let Some(topic) = caps.name("topic") { - add_preference_clause_cues("preference_topic", topic.as_str(), 10, out, seen); - } - matched = true; + for matched in hashtag_re().find_iter(content) { + let Some(value) = normalized_identifier(matched.as_str().trim_start_matches('#')) else { + continue; + }; + push_unique(out, seen, "has:hashtag"); + push_unique(out, seen, format!("hashtag:{value}")); } - if let Some(caps) = preference_rather_re().captures(content) { - if let Some(value) = caps.name("value") { - add_preference_clause_cues("preference_value", value.as_str(), 10, out, seen); + for matched in commit_hash_re().find_iter(content) { + let raw = matched.as_str(); + if !raw.chars().any(|character| matches!(character, 'a'..='f' | 'A'..='F')) { + continue; } - matched = true; + let Some(value) = normalized_identifier(raw) else { + continue; + }; + push_unique(out, seen, "has:commit_hash"); + push_unique(out, seen, format!("commit:{value}")); } +} - if let Some(caps) = preference_negative_re().captures(content) { - if let Some(value) = caps.name("value") { - add_preference_clause_cues("preference_contrast", value.as_str(), 10, out, seen); - } - matched = true; +fn clean_file_reference(raw: &str) -> String { + let mut value = raw + .trim() + .trim_matches(|character: char| matches!(character, '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | ',' | ';' | ':' | '!' | '?')) + .to_string(); + while value.ends_with('.') && !value.starts_with('.') { + value.pop(); + } + value +} + +fn add_one_file_reference(reference: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let reference = clean_file_reference(reference); + if reference.is_empty() || reference.contains("://") { + return; } + let normalized_separators = reference.replace('\\', "/"); + let segments = normalized_separators + .split('/') + .filter(|segment| !segment.is_empty() && *segment != "." && *segment != "..") + .filter(|segment| !segment.ends_with(':')) + .collect::<Vec<_>>(); + let Some(basename) = segments.last().copied() else { + return; + }; - if !matched { - if let Some(caps) = preference_simple_re().captures(content) { - if let Some(value) = caps.name("value") { - add_preference_clause_cues("preference_value", value.as_str(), 10, out, seen); + if segments.len() > 1 { + push_unique(out, seen, "has:directory_path"); + for segment in &segments[..segments.len() - 1] { + if let Some(value) = normalize_value(segment) { + push_unique(out, seen, format!("path_segment:{value}")); } } } -} -fn metadata_string<'a>(metadata: &'a HashMap<String, Value>, keys: &[&str]) -> Option<&'a str> { - for key in keys { - if let Some(value) = metadata.get(*key).and_then(|v| v.as_str()) { - if !value.trim().is_empty() { - return Some(value); + if let Some(value) = normalize_value(basename) { + push_unique(out, seen, format!("file_name:{value}")); + } + if !basename.starts_with('.') { + if let Some(extension) = basename.rsplit_once('.').map(|(_, extension)| extension) { + if let Some(value) = normalize_value(extension) { + push_unique(out, seen, format!("file_extension:{value}")); } } } - None -} - -fn parse_date_text(raw: &str) -> Option<NaiveDate> { - let cap = metadata_date_re().captures(raw)?; - let year = cap.name("year")?.as_str().parse::<i32>().ok()?; - let month = cap.name("month")?.as_str().parse::<u32>().ok()?; - let day = cap.name("day")?.as_str().parse::<u32>().ok()?; - NaiveDate::from_ymd_opt(year, month, day) } -fn parse_date_value(value: &Value) -> Option<NaiveDate> { - if let Some(raw) = value.as_str() { - return parse_date_text(raw); +fn add_file_structure_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + for matched in file_path_re().find_iter(content) { + add_one_file_reference(matched.as_str(), out, seen); } - if let Some(seconds) = value.as_f64() { - let days = (seconds / 86_400.0).floor() as i64; - return NaiveDate::from_ymd_opt(1970, 1, 1)?.checked_add_signed(Duration::days(days)); + for matched in file_name_re().find_iter(content) { + add_one_file_reference(matched.as_str(), out, seen); } - None } -fn metadata_date(metadata: &HashMap<String, Value>) -> Option<NaiveDate> { - for key in [ - "source_date", - "source_timestamp", - "timestamp", - "created_at", - "datetime", - "date", - ] { - if let Some(date) = metadata.get(key).and_then(parse_date_value) { - return Some(date); - } - } - None +fn looks_like_csv(content: &str) -> bool { + content + .lines() + .filter(|line| !line.trim().is_empty()) + .filter(|line| line.split(',').count() >= 2) + .take(3) + .count() + >= 2 } -fn source_date_facet(date: NaiveDate) -> String { - format!("source_date:{:04}_{:02}_{:02}", date.year(), date.month(), date.day()) -} - -fn source_week_facet(date: NaiveDate) -> String { - let week = date.iso_week(); - format!("source_week:{:04}_w{:02}", week.year(), week.week()) -} - -fn source_month_facet(date: NaiveDate) -> String { - format!("source_month:{:04}_{:02}", date.year(), date.month()) -} - -fn source_year_facet(date: NaiveDate) -> String { - format!("source_year:{:04}", date.year()) -} - -fn content_month_facet(month: u32) -> Option<String> { - if (1..=12).contains(&month) { - Some(format!("content_month:{:02}", month)) - } else { - None +fn add_document_structure_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + if json_object_re().is_match(content) { + push_unique(out, seen, "has:json"); } -} - -fn month_name_number(token: &str) -> Option<u32> { - match token.to_ascii_lowercase().as_str() { - "jan" | "january" => Some(1), - "feb" | "february" => Some(2), - "mar" | "march" => Some(3), - "apr" | "april" => Some(4), - "may" => Some(5), - "jun" | "june" => Some(6), - "jul" | "july" => Some(7), - "aug" | "august" => Some(8), - "sep" | "sept" | "september" => Some(9), - "oct" | "october" => Some(10), - "nov" | "november" => Some(11), - "dec" | "december" => Some(12), - _ => None, + let key_value_lines = key_value_line_re().find_iter(content).count(); + if key_value_lines >= 2 { + push_unique(out, seen, "has:key_value_pairs"); + push_unique(out, seen, "has:yaml"); } -} - -fn explicit_query_months(lower: &str) -> Vec<u32> { - let mut months = Vec::new(); - let mut seen = HashSet::new(); - for token in query_tokens(lower) { - if let Some(month) = month_name_number(&token) { - if seen.insert(month) { - months.push(month); - } + if xml_element_re().is_match(content) { + push_unique(out, seen, "has:xml"); + } + if looks_like_csv(content) { + push_unique(out, seen, "has:csv"); + } + let table_rows = content.lines().filter(|line| line.contains('|')).count(); + if markdown_table_separator_re().is_match(content) && table_rows >= 2 { + push_unique(out, seen, "has:markdown_table"); + } + if stack_trace_re().is_match(content) { + push_unique(out, seen, "has:stack_trace"); + } + if diff_re().is_match(content) { + push_unique(out, seen, "has:diff"); + } + for caps in heading_re().captures_iter(content) { + let Some(level) = caps.name("hashes").map(|hashes| hashes.as_str().len()) else { + continue; + }; + push_unique(out, seen, "has:heading"); + push_unique(out, seen, format!("heading_level:{level}")); + } + if checklist_re().is_match(content) { + push_unique(out, seen, "has:checklist"); + } + if block_quote_re().is_match(content) { + push_unique(out, seen, "has:block_quote"); + } + if markdown_link_re().is_match(content) { + push_unique(out, seen, "has:markdown_link"); + } + for caps in fenced_language_re().captures_iter(content) { + let Some(language) = caps.name("language").map(|language| language.as_str().to_ascii_lowercase().replace(|character: char| !character.is_ascii_alphanumeric(), "_")) else { + continue; + }; + if !language.is_empty() { + push_unique(out, seen, "has:code"); + push_unique(out, seen, format!("code_language:{language}")); } } - months -} - -fn previous_calendar_month_date(reference_date: NaiveDate) -> Option<NaiveDate> { - let (year, month) = if reference_date.month() == 1 { - (reference_date.year() - 1, 12) - } else { - (reference_date.year(), reference_date.month() - 1) - }; - NaiveDate::from_ymd_opt(year, month, 1) } -fn most_recent_weekend_dates(reference_date: NaiveDate) -> Option<(NaiveDate, NaiveDate)> { - let days_since_sunday = reference_date.weekday().num_days_from_sunday() as i64; - let sunday = reference_date.checked_sub_signed(Duration::days(days_since_sunday))?; - let saturday = sunday.checked_sub_signed(Duration::days(1))?; - Some((saturday, sunday)) +fn is_emoji(character: char) -> bool { + let code = character as u32; + (0x1f300..=0x1faff).contains(&code) || (0x2600..=0x27bf).contains(&code) } -fn add_metadata_temporal_facets( - metadata: Option<&HashMap<String, Value>>, - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - let Some(date) = metadata.and_then(metadata_date) else { - return; - }; - - push_unique(out, seen, "source_time:dated"); - push_unique(out, seen, source_date_facet(date)); - push_unique(out, seen, source_week_facet(date)); - push_unique(out, seen, source_month_facet(date)); - push_unique(out, seen, source_year_facet(date)); +fn add_emoji_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let mut emoji = false; + for character in content.chars() { + emoji |= is_emoji(character); + } + if emoji { + push_unique(out, seen, "has:emoji"); + } } -fn add_source_facets( - content: &str, - metadata: Option<&HashMap<String, Value>>, - existing_cues: &[String], - out: &mut Vec<String>, - seen: &mut HashSet<String>, -) { - if let Some(metadata) = metadata { - if let Some(role) = - metadata_string(metadata, &["source_role", "role", "speaker", "author_role"]) - { - if let Some(value) = normalize_value(role) { - push_unique(out, seen, format!("source_role:{}", value)); - } - } - if let Some(channel) = metadata_string( - metadata, - &["source_channel", "channel", "conversation", "room"], - ) { - if let Some(value) = normalize_value(channel) { - push_unique(out, seen, format!("source_channel:{}", value)); - } - } - if let Some(source_type) = metadata_string(metadata, &["source_type", "source", "kind"]) { - if let Some(value) = normalize_value(source_type) { - push_unique(out, seen, format!("source_type:{}", value)); - } - } - if let Some(session) = metadata_string( - metadata, - &[ - "source_session_id", - "session_id", - "conversation_id", - "thread_id", - ], - ) { - if let Some(value) = normalize_value(session) { - push_unique(out, seen, format!("source_session:{}", value)); - } - } +fn add_discourse_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let lower = content.to_ascii_lowercase(); + let has_negation = negation_re().is_match(&lower); + if has_negation { + push_unique(out, seen, "has:negation"); } - - for cue in existing_cues { - if let Some((key, value)) = cue.split_once(':') { - let target = match key { - "role" | "speaker" => Some("source_role"), - "channel" => Some("source_channel"), - "source" | "category" => Some("source_type"), - "session" | "conversation" | "thread" => Some("source_session"), - _ => None, - }; - if let Some(target) = target { - if let Some(value) = normalize_value(value) { - push_unique(out, seen, format!("{}:{}", target, value)); - } - } - } + if query_has_any( + &lower, + &["but", "however", "although", "whereas", "instead of", "rather than"], + ) { + push_unique(out, seen, "has:contrast"); } - - if let Some(cap) = prefix_re().captures(content) { - if let Some(value) = cap.get(1).and_then(|m| normalize_value(m.as_str())) { - push_unique(out, seen, format!("source_role:{}", value)); - } + if query_has_any(&lower, &["actually", "correction", "to clarify", "i mean", "rather than"]) + || (has_negation && query_has_any(&lower, &["but"])) + { + push_unique(out, seen, "has:correction"); + } + if query_has_any( + &lower, + &[ + "used to", + "no longer", + "no more", + "changed my mind", + "instead of", + "replaced by", + "superseded by", + ], + ) { + push_unique(out, seen, "has:supersession"); } } fn add_evidence_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - if number_re().is_match(content) { - push_unique(out, seen, "has:number"); - } - if money_re().is_match(content) { - push_unique(out, seen, "has:money"); - } - let has_short_numeric_date = short_numeric_date_re().is_match(content); - if date_re().is_match(content) || has_short_numeric_date { + if number_re().is_match(content) { push_unique(out, seen, "has:number"); } + if money_re().is_match(content) { push_unique(out, seen, "has:money"); } + if url_re().is_match(content) { push_unique(out, seen, "has:url"); } + if email_re().is_match(content) { push_unique(out, seen, "has:email"); } + if quoted_re().is_match(content) { push_unique(out, seen, "has:quote"); } + if inline_code_re().is_match(content) || has_code_fence(content) { + push_unique(out, seen, "has:code"); + } + let has_file_path = file_path_re().is_match(content); + if has_file_path || file_name_re().is_match(content) { + push_unique(out, seen, "has:file_name"); + } + if has_file_path { push_unique(out, seen, "has:file_path"); } + if date_re().is_match(content) + || short_numeric_date_re().is_match(content) + || has_temporal_month(content) + { push_unique(out, seen, "has:date"); } - for token in query_tokens(content) { - if let Some(month) = month_name_number(&token) { - if let Some(facet) = content_month_facet(month) { - push_unique(out, seen, facet); - } - } - } - for cap in short_numeric_date_re().captures_iter(content) { - let Some(month) = cap - .name("month") - .and_then(|m| m.as_str().parse::<u32>().ok()) - else { - continue; - }; - let Some(day) = cap - .name("day") - .and_then(|m| m.as_str().parse::<u32>().ok()) - else { - continue; - }; - if day <= 31 { - if let Some(facet) = content_month_facet(month) { - push_unique(out, seen, facet); + let tokens = surface_tokens(content); + for (index, token) in tokens.iter().enumerate() { + if let Some(month) = month_name_number(token) { + if is_temporal_month_at(&tokens, index) { + push_unique(out, seen, format!("content_month:{month:02}")); } } } - if duration_re().is_match(content) { - push_unique(out, seen, "has:duration"); - } - let has_weekday = weekday_re().is_match(content); - if has_weekday { + if duration_re().is_match(content) { push_unique(out, seen, "has:duration"); } + if weekday_re().is_match(content) { push_unique(out, seen, "has:weekday"); push_unique(out, seen, "schedule:weekly"); } - if clock_time_re().is_match(content) { - push_unique(out, seen, "has:time"); - } + if has_clock_time(content) { push_unique(out, seen, "has:time"); } add_time_of_day_facets(content, out, seen); add_cadence_facets(content, out, seen); + add_quantity_facets(content, out, seen); + add_identifier_facets(content, out, seen); + add_file_structure_facets(content, out, seen); + add_document_structure_facets(content, out, seen); + add_emoji_facets(content, out, seen); + add_discourse_facets(content, out, seen); let list_markers = content .lines() .filter(|line| { let trimmed = line.trim_start(); - trimmed.starts_with("- ") - || trimmed.starts_with("* ") - || list_item_re().is_match(trimmed) + trimmed.starts_with("- ") || trimmed.starts_with("* ") || list_item_re().is_match(trimmed) }) .take(3) .count(); @@ -1881,4622 +1266,472 @@ fn add_evidence_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet< } } -fn clock_hour_24(hour: u32, meridiem: &str) -> u32 { - let meridiem = meridiem.to_ascii_lowercase(); - if meridiem.starts_with('p') { - if hour == 12 { - 12 - } else { - hour + 12 +fn add_surface_entities(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let mut candidates = Vec::new(); + for cap in quoted_re().captures_iter(content) { + if let Some(raw) = cap.get(1).or_else(|| cap.get(2)).map(|m| m.as_str()) { + candidates.push(raw.to_string()); } - } else if hour == 12 { - 0 - } else { - hour - } -} - -fn time_of_day_for_hour(hour: u32) -> &'static str { - match hour { - 5..=11 => "morning", - 12..=16 => "afternoon", - 17..=21 => "evening", - _ => "night", } -} + candidates.extend(product_re().find_iter(content).map(|m| m.as_str().to_string())); + candidates.extend(proper_noun_re().find_iter(content).filter_map(|m| { + let raw = m.as_str(); + (!is_entity_noise(raw) && !is_temporal_month_entity(content, raw)).then(|| raw.to_string()) + })); -fn time_of_day_terms(lower: &str) -> Vec<&'static str> { - let mut out = Vec::new(); - let mut seen = HashSet::new(); - for (needle, facet) in [ - (" morning", "morning"), - (" afternoon", "afternoon"), - (" evening", "evening"), - (" tonight", "evening"), - (" night", "night"), - (" bedtime", "night"), - (" later part of the day", "evening"), - (" later part of day", "evening"), - ] { - if lower.contains(needle) && seen.insert(facet) { - out.push(facet); + let mut entity_seen = HashSet::new(); + for candidate in candidates { + if entity_seen.len() >= MAX_ENTITIES { break; } + if let Some(value) = normalize_value(&candidate) { + if entity_seen.insert(value.clone()) { + push_unique(out, seen, format!("entity:{value}")); + } } } - out } -fn add_time_of_day_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let lower = format!(" {} ", content.to_lowercase()); - for facet in time_of_day_terms(&lower) { - push_unique(out, seen, format!("time_of_day:{}", facet)); - } - for cap in clock_time_capture_re().captures_iter(content) { - let Some(hour) = cap - .name("hour") - .and_then(|m| m.as_str().parse::<u32>().ok()) - else { +fn temporal_event_relations(content: &str) -> Vec<(String, String)> { + let mut relations = Vec::new(); + let mut seen = HashSet::new(); + for caps in temporal_event_relation_re().captures_iter(content) { + let Some(relation) = caps.name("relation").map(|m| m.as_str().to_ascii_lowercase()) else { continue; }; - let Some(meridiem) = cap.name("meridiem").map(|m| m.as_str()) else { + let Some(anchor) = caps.name("anchor") else { continue; }; + let cues = crate::nl::tokenize_to_cues(anchor.as_str()); + let Some(anchor) = cues.iter().filter(|cue| cue.contains('_')).max_by_key(|cue| (cue.split('_').count(), cue.len())).or_else(|| cues.first()) else { continue; }; - let hour = clock_hour_24(hour, meridiem); - push_unique( - out, - seen, - format!("time_of_day:{}", time_of_day_for_hour(hour)), - ); + let pair = (relation, anchor.clone()); + if seen.insert(pair.clone()) { relations.push(pair); } } + relations } -fn add_cadence_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let mut matched = false; - for cap in cadence_re().captures_iter(content) { - let Some(unit) = cap.name("unit").and_then(|m| normalize_value(m.as_str())) else { - continue; - }; - matched = true; - push_unique(out, seen, "has:frequency"); - push_unique(out, seen, "schedule:frequency"); - push_unique(out, seen, format!("frequency_unit:{}", unit)); - if unit == "week" { - push_unique(out, seen, "schedule:weekly"); - } +fn add_temporal_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { + let lower = content.to_ascii_lowercase(); + for (needle, facet) in [ + ("currently", "temporal:current"), + ("right now", "temporal:current"), + ("latest", "temporal:current"), + ("recently", "temporal:recent"), + ("lately", "temporal:recent"), + ("last week", "temporal:last_week"), + ("past week", "temporal:last_week"), + ("previous week", "temporal:last_week"), + ("yesterday", "temporal:yesterday"), + ("today", "temporal:today"), + ("tomorrow", "temporal:tomorrow"), + ("last month", "temporal:last_month"), + ("last year", "temporal:last_year"), + ] { + if lower.contains(needle) { push_unique(out, seen, facet); } } - for cap in every_unit_re().captures_iter(content) { - let Some(unit) = cap.name("unit").and_then(|m| normalize_value(m.as_str())) else { - continue; - }; - matched = true; - push_unique(out, seen, "has:frequency"); - push_unique(out, seen, "schedule:frequency"); - push_unique(out, seen, format!("frequency_unit:{}", unit)); - if unit == "week" { - push_unique(out, seen, "schedule:weekly"); - } + if lower.contains("last ") || lower.contains("next ") || lower.contains("ago") || lower.contains("past ") { + push_unique(out, seen, "temporal:relative"); } - if matched && clock_time_re().is_match(content) { - push_unique(out, seen, "has:time"); + for (relation, anchor) in temporal_event_relations(content) { + push_unique(out, seen, format!("temporal_relation:{relation}")); + push_unique(out, seen, format!("temporal_anchor:{anchor}")); } } -fn has_any(lower: &str, needles: &[&str]) -> bool { - needles.iter().any(|needle| lower.contains(needle)) +pub fn extract_memory_facets_core( + content: &str, + metadata: Option<&HashMap<String, Value>>, + existing_cues: &[String], +) -> Vec<String> { + let mut facets = Vec::new(); + let mut seen = HashSet::new(); + add_source_facets(content, metadata, existing_cues, &mut facets, &mut seen); + add_metadata_temporal_facets(metadata, &mut facets, &mut seen); + add_evidence_facets(content, &mut facets, &mut seen); + add_temporal_facets(content, &mut facets, &mut seen); + add_surface_entities(content, &mut facets, &mut seen); + facets } -fn purchase_consideration_signal(lower: &str) -> bool { - let first_person_planning = has_any( - lower, - &[ - " i am considering ", - " i'm considering ", - " im considering ", - " i was considering ", - " we are considering ", - " we're considering ", - " we were considering ", - " i am thinking about ", - " i'm thinking about ", - " im thinking about ", - " we are thinking about ", - " we're thinking about ", - " i am looking to ", - " i'm looking to ", - " im looking to ", - " we are looking to ", - " we're looking to ", - " i plan to ", - " i'm planning to ", - " im planning to ", - " we plan to ", - " we're planning to ", - " i want to ", - " i'd like to ", - " we want to ", - " we'd like to ", - ], - ); - let acquisition_or_change = has_any( - lower, - &[ - " buy ", - " buying ", - " purchase ", - " purchasing ", - " get ", - " getting ", - " order ", - " ordering ", - " upgrade ", - " upgrading ", - " replace ", - " replacing ", - " switch ", - " switching ", - ], - ); - - (first_person_planning && acquisition_or_change) - || has_any( - lower, - &[ - " i am in the market for ", - " i'm in the market for ", - " im in the market for ", - " we are in the market for ", - " we're in the market for ", - ], - ) +pub fn extract_memory_facets( + content: &str, + metadata: Option<&HashMap<String, Value>>, + existing_cues: &[String], +) -> Vec<String> { + extract_memory_facets_core(content, metadata, existing_cues) } -fn iteration_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " another ", - " a different ", - " different version ", - " revised ", - " revision ", - " updated version ", - " alternative version ", - " second ", - " next version ", - " new version ", - " here's a more ", - " here is a more ", - ], - ) +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct StructuralQueryPlan { + /// Query-shape labels are structural only; semantic intent labels are not emitted. + /// Perspective and query-shape heuristics are currently English-specific. + pub labels: Vec<String>, + pub weighted_cues: Vec<(String, f64)>, + #[serde(default)] + pub cue_weight_adjustments: Vec<(String, f64)>, + pub suppress_generic: bool, } -fn inspiration_source_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " get inspiration from ", - " gets inspiration from ", - " getting inspiration from ", - " got inspiration from ", - " find inspiration from ", - " finds inspiration from ", - " finding inspiration from ", - " found inspiration from ", - " draw inspiration from ", - " draws inspiration from ", - " drawing inspiration from ", - " drew inspiration from ", - " take inspiration from ", - " takes inspiration from ", - " taking inspiration from ", - " took inspiration from ", - " inspired by ", - ], - ) +fn push_weighted_if_available<F>( + weighted: &mut Vec<(String, f64)>, + seen: &mut HashSet<String>, + available: &F, + cue: impl Into<String>, + weight: f64, +) where + F: Fn(&str) -> bool, +{ + let cue = cue.into(); + if available(&cue) && seen.insert(cue.clone()) { + weighted.push((cue, weight)); + } } -fn charity_event_signal(lower: &str) -> bool { - lower.contains(" charity ") - && has_any( - lower, - &[ - " event ", - " walk ", - " run ", - " ride ", - " drive ", - " gala ", - " fundraiser ", - " fund-raiser ", - " raise money ", - " raised money ", - ], - ) +fn push_adjustment(out: &mut Vec<(String, f64)>, cue: &str, multiplier: f64) { + if let Some((_, existing)) = out.iter_mut().find(|(existing, _)| existing == cue) { + *existing *= multiplier; + } else { + out.push((cue.to_string(), multiplier)); + } } -fn wake_time_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " wake up at ", - " wake up around ", - " wake up by ", - " wake up before ", - " wake up after ", - " waking up at ", - " waking up around ", - " waking up by ", - " waking up before ", - " waking up after ", - " wake-up time ", - " wake-up times ", - " wakeup time ", - " wakeup times ", - ], - ) +fn add_label(out: &mut Vec<String>, label: &str) { + if !out.iter().any(|existing| existing == label) { + out.push(label.to_string()); + } } -fn bed_time_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " go to bed at ", - " go to bed around ", - " go to bed by ", - " go to bed before ", - " go to bed after ", - " went to bed at ", - " went to bed around ", - " went to bed by ", - " went to bed before ", - " went to bed after ", - " get to bed at ", - " get to bed around ", - " get to bed by ", - " get to bed before ", - " get to bed after ", - " get to bed until ", - " got to bed at ", - " got to bed around ", - " got to bed by ", - " got to bed before ", - " got to bed after ", - " got to bed until ", - " bedtime at ", - " bedtime around ", - " bedtime was ", - ], - ) +fn query_has_any(query: &str, needles: &[&str]) -> bool { + let tokens = query_tokens(query); + needles.iter().any(|needle| { + let needle_tokens = query_tokens(needle); + !needle_tokens.is_empty() + && tokens + .windows(needle_tokens.len()) + .any(|window| window == needle_tokens.as_slice()) + }) } -fn add_type_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let lower = format!(" {} ", content.to_lowercase()); - let first_person_acquired = first_person_acquired_re().is_match(&lower); - let acquisition_source = - first_person_acquisition_source_re().is_match(&lower) || ownership_source_re().is_match(&lower); - let decision_selection = has_decision_selection_language(content); - let homegrown_source = homegrown_source_re().is_match(&lower); - let ingredient_context = ingredient_context_re().is_match(&lower); - let inspiration_source = inspiration_source_signal(&lower); - let charity_event = charity_event_signal(&lower); - let wake_time = wake_time_signal(&lower); - let bed_time = bed_time_signal(&lower); - - if has_any( - &lower, - &[ - "favorite", - "favourite", - "prefer", - "preference", - " i like ", - " i love ", - " enjoy ", - " fan of ", - "would rather", - ], - ) { - push_unique(out, seen, "type:preference"); +fn add_answer_shape_label(query: &str, plan: &mut StructuralQueryPlan) { + let tokens = query_tokens(query); + if tokens.is_empty() { + return; } - if has_any( - &lower, - &[ - "don't like", - "do not like", - "dislike", - "hate", - "avoid", - "can't stand", - "not a fan", - ], - ) { - push_unique(out, seen, "type:dislike"); - } - if owned_object_re().is_match(content) - || first_person_acquired - || first_person_possession_re().is_match(&lower) - { - push_unique(out, seen, "type:ownership"); - } - if first_person_acquired { - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "type:event"); - push_unique(out, seen, "purchase:acquired"); - } - if acquisition_source { - push_unique(out, seen, "type:ownership"); - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "type:event"); - push_unique(out, seen, "purchase:source"); - } - if purchase_consideration_signal(&lower) { - push_unique(out, seen, "type:purchase_consideration"); - } - if iteration_signal(&lower) { - push_unique(out, seen, "type:iteration"); - } - if inspiration_source { - push_unique(out, seen, "type:inspiration_source"); - push_unique(out, seen, "type:interest"); - } - if first_person_competition_event_re().is_match(&lower) { - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "type:event"); - push_unique(out, seen, "type:competition_event"); - push_unique(out, seen, "activity_domain:sport"); - } - if first_person_with_companion_re().is_match(&lower) { - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "companion:with"); - } - if first_person_completed_clean_re().is_match(&lower) { - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "completed_action:clean"); - } - if co_residence_with_self_re().is_match(&lower) { - push_unique(out, seen, "co_residence:with_self"); - } - if first_person_project_work_re().is_match(&lower) { - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "type:project_work"); - } - if first_person_completed_action_re().is_match(&lower) - || first_person_did_named_event_re().is_match(&lower) + + let shape = if query_has_any(query, &["who", "whose"]) { + Some("person") + } else if query_has_any(query, &["where"]) { + Some("location") + } else if query_has_any(query, &["when", "what time"]) { + Some("time") + } else if query_has_any(query, &["how many", "number of", "count of"]) { + Some("count") + } else if query_has_any(query, &["how much", "amount of", "cost of", "price of"]) { + Some("amount") + } else if query_has_any(query, &["why"]) { + Some("reason") + } else if query_has_any(query, &["how long"]) { + Some("duration") + } else if query_has_any(query, &["which", "what options", "what choices"]) { + Some("selection") + } else if query_has_any(query, &["what kind", "what type", "what category"]) { + Some("category") + } else if is_question_auxiliary(&tokens[0]) + && tokens.get(1).is_none_or(|token| token != "you") + && !tokens.iter().any(|token| is_question_word(token)) { - push_unique(out, seen, "type:activity"); - push_unique(out, seen, "type:event"); - } - if charity_event { - push_unique(out, seen, "type:event"); - push_unique(out, seen, "event_domain:charity"); - } - if wake_time { - push_unique(out, seen, "type:routine"); - push_unique(out, seen, "routine:wake_time"); - } - if bed_time { - push_unique(out, seen, "type:routine"); - push_unique(out, seen, "routine:bed_time"); - } - if decision_selection { - push_unique(out, seen, "type:decision"); - push_unique(out, seen, "type:selection"); - if has_any( - &lower, - &[" name ", " names ", " named ", " call it ", " called "], - ) { - push_unique(out, seen, "type:naming"); - } - } - if has_any( - &lower, - &[ - " milestone ", - " first client", - " first customer", - " first sale", - " first contract", - " signed a contract", - " landed my first", - " landed our first", - " launched my ", - " launched our ", - " launched the ", - " opened my ", - " opened our ", - ], - ) { - push_unique(out, seen, "type:milestone"); - } - if has_any( - &lower, - &[ - " actually ", - " changed to ", - " changed from ", - " switched to ", - " switched from ", - " no longer ", - " instead ", - " updated ", - " just wrapped up ", - " wrapped up ", - ], - ) { - push_unique(out, seen, "type:update"); - } - if has_any( - &lower, - &[ - "recommend", - "suggest", - "suggestion", - "you should", - "try ", - "option", - "would be good", - ], - ) { - push_unique(out, seen, "type:recommendation"); - } - if has_any( - &lower, - &[ - " working in the field", - " work in the field", - " works in the field", - " my field", - " our field", - " field of research", - " research area", - " research interests", - " i specialize in", - " i specialize on", - " we specialize in", - " we specialize on", - " i'm specializing in", - " im specializing in", - " we're specializing in", - " were specializing in", - ], - ) { - push_unique(out, seen, "type:expertise"); - push_unique(out, seen, "type:interest"); - } - if has_any( - &lower, - &[ - "recipe", - "ingredient", - "preheat", - "tablespoon", - "teaspoon", - "bake", - "simmer", - "saute", - "cook for", - ], - ) { - push_unique(out, seen, "type:recipe"); - } - if ingredient_context { - push_unique(out, seen, "type:ingredient"); - } - if homegrown_source { - push_unique(out, seen, "type:homegrown"); - } - if has_any( - &lower, - &[ - "answer is", - "the answer", - "correct answer", - "here's", - "here is", - "you can ", - "you could ", - ], - ) { - push_unique(out, seen, "type:answer"); - } - if has_any( - &lower, - &[ - "usually", - "always", - "every morning", - "every night", - "daily", - "weekly", - "typical week", - "per week", - "a week", - "each week", - "routine", - "habit", - "wind down", - ], - ) { - push_unique(out, seen, "type:routine"); + Some("boolean") + } else { + None + }; + + if let Some(shape) = shape { + add_label(&mut plan.labels, &format!("answer_shape_{shape}")); } } -pub fn has_decision_selection_language(content: &str) -> bool { - let lower = format!(" {} ", content.to_lowercase()); - has_any( - &lower, - &[ - " decided ", - " decide ", - " chose ", - " choose ", - " picked ", - " pick ", - " selected ", - " select ", - " settled on ", - " went with ", - " go with ", - " call it ", - " name it ", - ], - ) || decision_selection_re().is_match(&lower) +fn query_perspective_tokens(query: &str) -> Vec<String> { + let normalized = query + .to_ascii_lowercase() + .replace('’', "'") + // Expand only grammatical contractions. Predicate/content words are + // intentionally not inspected or classified. + .replace("n't", " not") + .replace("'re", " are") + .replace("'ve", " have") + .replace("'m", " am") + .replace("'ll", " will") + .replace("'d", " would") + .replace("'s", " is"); + query_tokens(&normalized) } -fn navigation_route_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " getting around ", - " get around ", - " how to get to ", - " how do i get to ", - " how can i get to ", - " best way to get ", - " way to get there ", - " way to get to ", - " route to ", - " route from ", - " directions to ", - " direction to ", - " navigate ", - " navigation ", - " meeting point ", - " travel to ", - " travel from ", - " transfer to ", - " transfer at ", - ], +fn is_question_auxiliary(token: &str) -> bool { + matches!( + token, + "am" + | "are" + | "is" + | "was" + | "were" + | "do" + | "does" + | "did" + | "have" + | "has" + | "had" + | "can" + | "could" + | "may" + | "might" + | "must" + | "shall" + | "should" + | "will" + | "would" ) } -fn navigation_transit_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " public transport", - " public transportation", - " public transit", - " mass transit", - " take the train", - " take a train", - " by train", - " train from ", - " train to ", - " train station", - " subway", - " metro", - " take the bus", - " take a bus", - " by bus", - " bus from ", - " bus to ", - " bus station", - " tram", - " ferry", - " taxi", - " rideshare", - " ride share", - " airport shuttle", - ], - ) +fn is_question_word(token: &str) -> bool { + matches!(token, "what" | "which" | "when" | "where" | "why" | "how" | "who") } -fn navigation_station_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " station", - " airport", - " terminal", - " platform", - " ticket gate", - " departure gate", - " arrival gate", - " gate when entering", - " gate when exiting", - ], - ) +fn is_embedded_question_marker(token: &str) -> bool { + is_question_word(token) || matches!(token, "if" | "whether") } -fn navigation_fare_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " fare", - " ticket", - " tickets", - " travel time", - " ride time", - " journey time", - " transfer time", - " approximate cost", - " cost using", - ], - ) +fn perspective_after_auxiliary(tokens: &[String], auxiliary_index: usize) -> Option<&'static str> { + let subject_index = if tokens.get(auxiliary_index + 1).is_some_and(|token| token == "not") { + auxiliary_index + 2 + } else { + auxiliary_index + 1 + }; + tokens + .get(subject_index) + .and_then(|token| query_perspective_for_subject(token)) } -fn navigation_pass_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " transit card", - " transport card", - " transportation card", - " prepaid card", - " rail pass", - " train pass", - " travel pass", - " metrocard", - " metro card", - ], - ) +fn query_perspective_for_subject(token: &str) -> Option<&'static str> { + match token { + "i" | "we" | "me" | "us" | "my" | "our" | "mine" | "ours" => { + Some("first_person") + } + "you" | "your" | "yours" => Some("second_person"), + "he" | "she" | "they" | "it" | "him" | "her" | "them" | "his" | "their" + | "its" | "theirs" => Some("third_person"), + _ => None, + } } -fn navigation_app_signal(lower: &str) -> bool { - has_any( - lower, - &[ - " transit app", - " transport app", - " transportation app", - " travel app", - " trip app", - " route app", - " maps app", - " itinerary app", - " tripit app", - " google maps", - " apple maps", - " citymapper", - " moovit", - " downloaded the app", - " downloaded an app", - ], - ) +fn is_request_wrapper(tokens: &[String]) -> bool { + match tokens.first().map(String::as_str) { + Some("please") => true, + Some("can" | "could" | "will" | "would") => { + tokens.get(1).is_some_and(|token| token == "you") + } + Some("do") => { + tokens.get(1).is_some_and(|token| token == "you") + && tokens.get(2).is_some_and(|token| token == "remember") + } + _ => false, + } } -fn temporal_event_relation_re() -> &'static Regex { - static RE: OnceLock<Regex> = OnceLock::new(); - RE.get_or_init(|| { - Regex::new( - r"(?is)\b(?P<relation>after|before)\s+(?:the\s+)?(?P<anchor>[A-Za-z0-9][A-Za-z0-9'_-]*(?:\s+[A-Za-z0-9][A-Za-z0-9'_-]*){0,3})", - ) - .unwrap() - }) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EmbeddedPerspective { + None, + One(&'static str), + Conflicting, } -fn temporal_event_relations(content: &str) -> Vec<(String, String)> { - let mut relations = Vec::new(); - let mut seen = HashSet::new(); +fn embedded_query_perspective(tokens: &[String]) -> EmbeddedPerspective { + let mut found = None; + for index in 1..tokens.len() { + let Some(perspective) = (|| { + if !is_embedded_question_marker(&tokens[index]) { + return None; + } - for caps in temporal_event_relation_re().captures_iter(content) { - let Some(relation) = caps.name("relation") else { - continue; - }; - let Some(anchor) = caps.name("anchor") else { - continue; - }; - let anchor_cues = crate::nl::tokenize_to_cues(anchor.as_str()); - let anchor_cue = anchor_cues - .iter() - .filter(|cue| cue.contains('_')) - .max_by_key(|cue| (cue.split('_').count(), cue.len())) - .or_else(|| anchor_cues.first()); - let Some(anchor_cue) = anchor_cue else { + if let Some(perspective) = tokens + .get(index + 1) + .and_then(|token| query_perspective_for_subject(token)) + { + return Some(perspective); + } + + if tokens + .get(index + 1) + .is_some_and(|token| is_question_auxiliary(token)) + { + return perspective_after_auxiliary(tokens, index + 1); + } + + None + })() else { continue; }; - let pair = (relation.as_str().to_lowercase(), anchor_cue.clone()); - if seen.insert(pair.clone()) { - relations.push(pair); - } - } - - relations -} -fn add_temporal_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let lower = content.to_lowercase(); - if has_any( - &lower, - &[ - "currently", - "current ", - "right now", - "now ", - "latest", - "newest", - "recently updated", - ], - ) { - push_unique(out, seen, "temporal:current"); - } - if has_any( - &lower, - &[ - "recently", - "lately", - "the other day", - "past few", - "last few", - ], - ) { - push_unique(out, seen, "temporal:recent"); - } - if has_any(&lower, &["last week", "past week", "previous week"]) { - push_unique(out, seen, "temporal:last_week"); - } - if has_any( - &lower, - &[ - "yesterday", - "today", - "tomorrow", - "last ", - "next ", - "ago", - "past ", - ], - ) { - push_unique(out, seen, "temporal:relative"); - } - for marker in [ - ("yesterday", "temporal:yesterday"), - ("today", "temporal:today"), - ("tomorrow", "temporal:tomorrow"), - ("last month", "temporal:last_month"), - ("last year", "temporal:last_year"), - ] { - if lower.contains(marker.0) { - push_unique(out, seen, marker.1); + match found { + None => found = Some(perspective), + Some(existing) if existing == perspective => {} + Some(_) => return EmbeddedPerspective::Conflicting, } } - for (relation, anchor) in temporal_event_relations(content) { - push_unique(out, seen, format!("temporal_relation:{relation}")); - push_unique(out, seen, format!("temporal_anchor:{anchor}")); + + match found { + Some(perspective) => EmbeddedPerspective::One(perspective), + None => EmbeddedPerspective::None, } } -fn add_entity_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - let mut entities = Vec::new(); - for cap in quoted_re().captures_iter(content) { - if let Some(raw) = cap.get(1).or_else(|| cap.get(2)).map(|m| m.as_str()) { - entities.push(raw.to_string()); - } - } - for mat in product_re().find_iter(content) { - entities.push(mat.as_str().to_string()); - } - for mat in proper_noun_re().find_iter(content) { - let raw = mat.as_str(); - if is_entity_noise(raw) || is_temporal_month_entity(content, raw) { - continue; +fn outer_query_perspective(tokens: &[String]) -> Option<&'static str> { + if tokens.len() >= 2 && is_question_auxiliary(&tokens[0]) { + if let Some(perspective) = perspective_after_auxiliary(&tokens, 0) { + return Some(perspective); } - entities.push(raw.to_string()); } - let mut entity_seen = HashSet::new(); - for entity in entities { - if entity_seen.len() >= MAX_ENTITIES { - break; - } - if let Some(value) = normalize_value(&entity) { - if entity_seen.insert(value.clone()) { - push_unique(out, seen, format!("entity:{}", value)); - } + if tokens.len() >= 3 + && is_question_word(&tokens[0]) + && is_question_auxiliary(&tokens[1]) + { + if let Some(perspective) = perspective_after_auxiliary(&tokens, 1) { + return Some(perspective); } } + + None } -fn add_entity_attribute_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - for cap in entity_class_relation_re().captures_iter(content) { - let Some(class_text) = cap.name("class").map(|m| m.as_str()) else { - continue; +/// Detects who the query is grammatically asking about without inspecting the +/// predicate or topic. Embedded perspective overrides the outer clause only +/// for recognized request wrappers; otherwise disagreement is left unweighted. +fn query_perspective(query: &str) -> Option<&'static str> { + let tokens = query_perspective_tokens(query); + let outer = outer_query_perspective(&tokens); + let embedded = embedded_query_perspective(&tokens); + + if is_request_wrapper(&tokens) { + return match embedded { + EmbeddedPerspective::One(perspective) => Some(perspective), + EmbeddedPerspective::None | EmbeddedPerspective::Conflicting => None, }; - if normalize_role_phrase(class_text).is_none() { - continue; - } - push_unique(out, seen, "type:entity_attribute"); - push_unique(out, seen, "attribute:class_relation"); - return; } - for cap in preferred_attribute_relation_re().captures_iter(content) { - let Some(class_text) = cap.name("class").map(|m| m.as_str()) else { - continue; - }; - if normalize_role_phrase(class_text).is_none() { - continue; - } - push_unique(out, seen, "type:entity_attribute"); - push_unique(out, seen, "attribute:class_relation"); - return; + + match embedded { + EmbeddedPerspective::None => outer, + EmbeddedPerspective::One(inner) => match outer { + None => Some(inner), + Some(outer) if outer == inner => Some(outer), + Some(_) => None, + }, + EmbeddedPerspective::Conflicting => None, } } -fn add_person_role_facets(content: &str, out: &mut Vec<String>, seen: &mut HashSet<String>) { - for cap in titled_person_re().captures_iter(content) { - let Some(title) = cap.name("title").and_then(|m| normalize_value(m.as_str())) else { - continue; - }; - push_unique(out, seen, format!("person_title:{}", title)); - push_unique(out, seen, "person_ref:named"); +pub fn compile_query_plan<F>(query: &str, available: F) -> StructuralQueryPlan +where + F: Fn(&str) -> bool, +{ + compile_query_plan_with_reference_time(query, None, available) +} + +pub fn compile_query_plan_with_reference_time<F>( + query: &str, + reference_time: Option<&str>, + available: F, +) -> StructuralQueryPlan +where + F: Fn(&str) -> bool, +{ + let lower = query.to_ascii_lowercase(); + let mut plan = StructuralQueryPlan::default(); + let mut seen = HashSet::new(); + let structural = extract_memory_facets_core(query, None, &[]); + + for facet in structural { + let label = facet.replace(':', "_"); + add_label(&mut plan.labels, &label); + push_weighted_if_available( + &mut plan.weighted_cues, + &mut seen, + &available, + facet, + 2.0, + ); } - for cap in possessed_role_before_title_re().captures_iter(content) { - if let Some(role) = cap - .name("role") - .and_then(|m| normalize_person_role_phrase(m.as_str())) - { - push_unique(out, seen, format!("person_role_phrase:{}", role)); - push_unique(out, seen, "person_ref:named"); + let explicit_source_role = plan.labels.iter().any(|label| { + label == "source_role_user" || label == "source_role_assistant" + }); + if let Some(perspective) = query_perspective(query) { + add_label( + &mut plan.labels, + &format!("query_perspective_{perspective}"), + ); + + if !explicit_source_role { + // Grammatical perspective and stored-message authorship are + // different dimensions. In this deployment, first-person + // questions usually ask about user-authored memories and + // second-person questions usually ask about assistant-authored + // memories, so retain those as soft retrieval preferences. + let source_role = match perspective { + "first_person" => Some(("source_user", "user")), + "second_person" => Some(("source_assistant", "assistant")), + "third_person" => None, + _ => None, + }; + if let Some((label, role)) = source_role { + add_label(&mut plan.labels, label); + push_weighted_if_available( + &mut plan.weighted_cues, + &mut seen, + &available, + format!("source_role:{role}"), + QUERY_PERSPECTIVE_SOURCE_ROLE_WEIGHT, + ); + } } } - for cap in role_before_title_re().captures_iter(content) { - if let Some(role) = cap - .name("role") - .and_then(|m| normalize_person_role_phrase(m.as_str())) - { - push_unique(out, seen, format!("person_role_phrase:{}", role)); - push_unique(out, seen, "person_ref:named"); - } + if query_has_any(&lower, &["timeline", "sequence", "in order", "chronological", "what order"]) { + add_label(&mut plan.labels, "ordered_reconstruction"); + } + if query_has_any(&lower, &["summarize", "summary", "overview", "recap", "main points", "key points"]) { + add_label(&mut plan.labels, "multi_evidence_summary"); } + if query_has_any(&lower, &["list", "all", "several", "multiple", "different options", "examples"]) { + add_label(&mut plan.labels, "multi_evidence_collection"); + push_weighted_if_available(&mut plan.weighted_cues, &mut seen, &available, "has:list", 1.2); + } + add_answer_shape_label(query, &mut plan); - for cap in role_before_name_re().captures_iter(content) { - let Some(role) = cap - .name("role") - .and_then(|m| normalize_person_role_phrase(m.as_str())) - else { - continue; - }; - if role.contains('_') { - push_unique(out, seen, format!("person_role_phrase:{}", role)); - push_unique(out, seen, "person_ref:named"); + if let Some(reference_time) = reference_time.and_then(parse_date_text) { + if lower.contains("today") { + push_weighted_if_available(&mut plan.weighted_cues, &mut seen, &available, source_date_facet(reference_time), 2.0); + push_adjustment(&mut plan.cue_weight_adjustments, "today", 0.35); + add_label(&mut plan.labels, "temporal_resolved_date"); + } else if lower.contains("yesterday") { + if let Some(date) = reference_time.checked_sub_signed(Duration::days(1)) { + push_weighted_if_available(&mut plan.weighted_cues, &mut seen, &available, source_date_facet(date), 2.0); + push_adjustment(&mut plan.cue_weight_adjustments, "yesterday", 0.35); + add_label(&mut plan.labels, "temporal_resolved_date"); + } } } -} - -const TRANSPORT_MODES: &[(&str, &[&str])] = &[ - ("bus", &["bus", "buses"]), - ("train", &["train", "trains"]), - ("plane", &["plane", "planes", "flight", "flights"]), - ("car", &["car", "cars"]), - ("taxi", &["taxi", "taxis", "cab", "cabs"]), - ("subway", &["subway", "subways", "metro", "metros"]), - ("tram", &["tram", "trams"]), - ("ferry", &["ferry", "ferries"]), - ("bike", &["bike", "bikes", "bicycle", "bicycles"]), - ("walk", &["walk", "walking"]), -]; - -const RELIGIOUS_CONTEXT_TERMS: &[&str] = &[ - "abbey", - "ashram", - "baptist", - "bible", - "buddhist", - "cathedral", - "catholic", - "chapel", - "christian", - "church", - "convent", - "episcopal", - "gurdwara", - "hindu", - "islamic", - "jewish", - "lutheran", - "methodist", - "monastery", - "mosque", - "muslim", - "orthodox", - "parish", - "presbyterian", - "rabbi", - "shrine", - "synagogue", - "temple", -]; - -const RELIGIOUS_ACTIVITY_TERMS: &[&str] = &[ - "bible study", - "communion", - "eucharist", - "liturgy", - "mass", - "maundy", - "prayer", - "sabbath service", - "sermon", - "service", - "sunday school", - "worship", -]; - -fn padded_contains_word(lower: &str, word: &str) -> bool { - lower.contains(&format!(" {word} ")) -} - -fn has_padded_term(lower: &str, terms: &[&str]) -> bool { - terms.iter().any(|term| padded_contains_word(lower, term)) -} - -pub fn extract_memory_facets_core( - content: &str, - metadata: Option<&HashMap<String, Value>>, - existing_cues: &[String], -) -> Vec<String> { - let mut facets = Vec::new(); - let mut seen = HashSet::new(); - add_source_facets(content, metadata, existing_cues, &mut facets, &mut seen); - add_metadata_temporal_facets(metadata, &mut facets, &mut seen); - add_evidence_facets(content, &mut facets, &mut seen); - add_numeric_object_facets(content, &mut facets, &mut seen); - add_inventory_object_facets(content, &mut facets, &mut seen); - add_age_facets(content, &mut facets, &mut seen); - add_education_facets(content, &mut facets, &mut seen); - add_family_facets(content, &mut facets, &mut seen); - add_type_facets(content, &mut facets, &mut seen); - add_personal_transition_facets(content, &mut facets, &mut seen); - add_temporal_facets(content, &mut facets, &mut seen); - add_entity_facets(content, &mut facets, &mut seen); - add_entity_attribute_facets(content, &mut facets, &mut seen); - add_person_role_facets(content, &mut facets, &mut seen); - - facets + plan.suppress_generic = false; + plan } -pub fn extract_memory_facets( - content: &str, - metadata: Option<&HashMap<String, Value>>, - existing_cues: &[String], -) -> Vec<String> { - extract_memory_facets_with_cuepacks( - content, - metadata, - existing_cues, - crate::cuepacks::default_registry(), - None, - ) -} - -pub fn extract_memory_facets_with_cuepacks( - content: &str, - metadata: Option<&HashMap<String, Value>>, - existing_cues: &[String], - cuepacks: &crate::cuepacks::CuePackRegistry, - cuepack_selection: Option<&[String]>, -) -> Vec<String> { - let mut facets = extract_memory_facets_core(content, metadata, existing_cues); - let mut seen: HashSet<String> = facets.iter().map(|facet| facet.to_lowercase()).collect(); - let pack_output = cuepacks.extract_memory_facets(content, cuepack_selection); - let emits_standing_instruction = pack_output - .facets - .iter() - .any(|facet| facet == "type:standing_instruction"); - let emits_explicit_preference = pack_output - .facets - .iter() - .any(|facet| facet == "preference:explicit"); - for facet in pack_output.facets { - if seen.insert(facet.to_lowercase()) { - facets.push(facet); - } - } - if emits_standing_instruction { - add_standing_instruction_dynamic_facets(content, &mut facets, &mut seen); - } - if emits_explicit_preference { - add_preference_dynamic_facets(content, &mut facets, &mut seen); - } - facets -} - -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -pub struct QueryIntent { - pub labels: Vec<String>, - pub weighted_cues: Vec<(String, f64)>, - #[serde(default)] - pub cue_weight_adjustments: Vec<(String, f64)>, - #[serde(default)] - pub cuepack_rules: Vec<String>, - pub suppress_generic: bool, -} - -fn push_weighted_if_available<F>( - out: &mut Vec<(String, f64)>, - seen: &mut HashSet<String>, - available: &F, - cue: &str, - weight: f64, -) where - F: Fn(&str) -> bool, -{ - if !available(cue) { - return; - } - if seen.insert(cue.to_string()) { - out.push((cue.to_string(), weight)); - } else if let Some((_, existing_weight)) = out.iter_mut().find(|(existing, _)| existing == cue) - { - if *existing_weight < weight { - *existing_weight = weight; - } - } -} - -fn add_label(out: &mut Vec<String>, label: &str) { - if !out.iter().any(|existing| existing == label) { - out.push(label.to_string()); - } -} - -fn push_adjustment(out: &mut Vec<(String, f64)>, cue: &str, multiplier: f64) { - if let Some((_, existing)) = out.iter_mut().find(|(existing, _)| existing == cue) { - if *existing > multiplier { - *existing = multiplier; - } - } else { - out.push((cue.to_string(), multiplier)); - } -} - -fn normalized_phrase_initialism(value: &str) -> Option<String> { - let parts = value - .split('_') - .filter(|part| { - part.len() >= 2 - && part.chars().all(|ch| ch.is_ascii_alphabetic()) - && !crate::nl::get_stopwords().contains(*part) - }) - .collect::<Vec<_>>(); - - if parts.len() < 2 || parts.len() > 6 { - return None; - } - - let initials = parts - .iter() - .filter_map(|part| part.chars().next()) - .collect::<String>(); - if initials.len() < 2 || initials.len() > 8 || initials == value { - None - } else { - Some(initials) - } -} - -fn recommendation_topic_is_generic(cue: &str) -> bool { - if cue.contains('_') { - return cue.split('_').all(recommendation_topic_is_generic); - } - - matches!( - cue, - "can" - | "could" - | "would" - | "should" - | "please" - | "recommend" - | "recommendation" - | "suggest" - | "suggestion" - | "idea" - | "ideas" - | "tip" - | "tips" - | "advice" - | "help" - | "helpful" - | "think" - | "try" - | "recipe" - | "recipes" - | "useful" - | "good" - | "best" - | "some" - | "any" - | "make" - | "making" - | "choose" - | "choosing" - | "chosen" - | "choice" - | "decide" - | "deciding" - | "decision" - | "select" - | "selecting" - | "selection" - | "not" - | "sure" - | "one" - | "look" - | "looking" - | "thing" - | "something" - | "show" - | "shows" - | "movie" - | "movies" - | "watch" - | "tonight" - | "new" - | "current" - | "recent" - | "lately" - | "upcoming" - | "weekend" - | "today" - | "tomorrow" - | "later" - | "soon" - | "trip" - | "travel" - | "accessory" - | "setup" - ) -} - -fn recommendation_topic_cues(query: &str) -> Vec<(String, f64)> { - let mut seen = HashSet::new(); - let mut cues = Vec::new(); - let tokenized = crate::nl::tokenize_to_cues(query); - let mut head_topic = None; - let lower = query.to_lowercase(); - - for marker in [ - "making a ", - "making an ", - "making some ", - "make a ", - "make an ", - "make some ", - "what to ", - "how to ", - ] { - let Some((_, tail)) = lower.split_once(marker) else { - continue; - }; - if let Some(action) = crate::nl::tokenize_to_cues(tail) - .into_iter() - .map(|cue| cue.trim().to_lowercase()) - .find(|cue| cue.len() > 2 && !cue.contains('_') && !recommendation_topic_is_generic(cue)) - { - head_topic = Some(action); - break; - } - } - - if head_topic.is_none() { - for idx in 0..tokenized.len() { - let cue = tokenized[idx].trim().to_lowercase(); - if cue.len() <= 2 || cue.contains('_') || recommendation_topic_is_generic(&cue) { - continue; - } - - let right = tokenized - .iter() - .skip(idx + 1) - .take(3) - .map(|part| part.trim().to_lowercase()) - .collect::<Vec<_>>(); - if right - .iter() - .any(|part| matches!(part.as_str(), "recipe" | "recipes" | "idea" | "ideas" | "recommendation" | "recommendations" | "suggestion" | "suggestions")) - { - head_topic = Some(cue); - } - } - } - - for cue in tokenized { - let cue = cue.trim().to_lowercase(); - if cue.len() <= 2 || recommendation_topic_is_generic(&cue) || !seen.insert(cue.clone()) { - continue; - } - let weight = if Some(cue.as_str()) == head_topic.as_deref() { - 4.0 - } else if head_topic.is_some() && !cue.contains('_') { - 1.4 - } else { - 2.4 - }; - cues.push((cue, weight)); - if cues.len() >= 8 { - break; - } - } - cues -} - -fn query_transport_modes(query: &str) -> Vec<&'static str> { - let lower = format!(" {} ", crate::nl::normalize_text(query)); - let mut modes = Vec::new(); - for (mode, variants) in TRANSPORT_MODES { - if variants - .iter() - .any(|variant| padded_contains_word(&lower, variant)) - { - modes.push(*mode); - } - } - modes -} - -fn add_person_query_intent<F>( - lower: &str, - is_count: bool, - intent: &mut QueryIntent, - seen: &mut HashSet<String>, - available: &F, -) where - F: Fn(&str) -> bool, -{ - let doctor_event_context = has_any( - lower, - &[ - "doctor appointment", - "doctor's appointment", - "doctor appointments", - "doctor visit", - "doctor's visit", - ], - ); - let title_queries = [ - ( - has_any(lower, &["doctor", "doctors", "dr. ", " dr "]) - && (is_count || !doctor_event_context), - "dr", - "person_title:dr", - ), - ( - has_any(lower, &["professor", "professors", "prof. ", " prof "]), - "prof", - "person_title:prof", - ), - ]; - let mut saw_person_title_query = false; - - for (matches_query, lexical_title, title_facet) in title_queries { - if !matches_query { - continue; - } - saw_person_title_query = true; - add_label(&mut intent.labels, "person_role"); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - lexical_title, - 2.2, - ); - push_weighted_if_available(&mut intent.weighted_cues, seen, available, title_facet, 2.4); - } - - if is_count && saw_person_title_query { - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - "person_ref:named", - 1.6, - ); - } -} - -fn has_person_title_query(lower: &str) -> bool { - has_any( - lower, - &["doctor", "doctors", "dr. ", " dr ", "professor", "professors", "prof. ", " prof "], - ) -} - -fn query_tokens(lower: &str) -> Vec<String> { - lower - .chars() - .map(|ch| if ch.is_alphanumeric() { ch } else { ' ' }) - .collect::<String>() - .split_whitespace() - .map(|token| token.to_string()) - .collect() -} - -fn query_has_phrase(tokens: &[String], phrase: &[&str]) -> bool { - !phrase.is_empty() - && tokens - .windows(phrase.len()) - .any(|window| window.iter().map(String::as_str).eq(phrase.iter().copied())) -} - -fn count_object_stopwords() -> &'static HashSet<&'static str> { - static STOPWORDS: OnceLock<HashSet<&'static str>> = OnceLock::new(); - STOPWORDS.get_or_init(|| { - HashSet::from([ - "a", - "an", - "and", - "current", - "currently", - "different", - "existing", - "my", - "new", - "old", - "one", - "own", - "kind", - "kinds", - "the", - "total", - "type", - "types", - ]) - }) -} - -fn count_object_boundary(token: &str) -> bool { - matches!( - token, - "am" | "are" - | "can" - | "could" - | "did" - | "do" - | "does" - | "had" - | "has" - | "have" - | "include" - | "including" - | "is" - | "should" - | "was" - | "were" - | "will" - | "would" - ) -} - -fn collect_count_object(tokens: &[String], start: usize) -> Option<Vec<String>> { - let stopwords = count_object_stopwords(); - let mut objects = Vec::new(); - let mut idx = start; - - while idx < tokens.len() && objects.len() < 4 { - let token = tokens[idx].as_str(); - if count_object_boundary(token) { - break; - } - if !stopwords.contains(token) { - if let Some(object) = normalize_quantity_token(token) { - objects.push(object); - } - } - idx += 1; - } - - if objects.is_empty() { - None - } else { - Some(objects) - } -} - -fn count_query_objects(lower: &str) -> Vec<String> { - let tokens = query_tokens(lower); - let mut objects = Vec::new(); - let mut seen = HashSet::new(); - - for idx in 0..tokens.len() { - let start = if tokens[idx] == "many" && idx > 0 && tokens[idx - 1] == "how" { - Some(idx + 1) - } else if tokens[idx] == "of" - && idx > 0 - && matches!(tokens[idx - 1].as_str(), "number" | "count" | "total") - { - Some(idx + 1) - } else { - None - }; - - if let Some(start) = start { - if let Some(candidates) = collect_count_object(&tokens, start) { - for candidate in candidates { - if seen.insert(candidate.clone()) { - objects.push(candidate); - } - } - } - } - } - - objects -} - -fn count_scope_preposition(token: &str) -> bool { - matches!( - token, - "in" | "inside" | "within" | "from" | "for" | "across" | "among" | "on" - ) -} - -fn count_scope_boundary(token: &str) -> bool { - matches!( - token, - "and" | "but" - | "in" - | "or" - | "from" - | "for" - | "because" - | "when" - | "where" - | "which" - | "who" - | "what" - | "that" - | "then" - | "than" - | "with" - | "without" - | "before" - | "after" - | "while" - | "during" - ) -} - -fn count_scope_determiner(token: &str) -> bool { - matches!( - token, - "my" | "our" | "the" | "this" | "that" | "these" | "those" | "a" | "an" - ) -} - -fn count_query_scope_tokens(lower: &str) -> Vec<String> { - let tokens = query_tokens(lower); - let mut scoped = Vec::new(); - let mut seen = HashSet::new(); - - for idx in 0..tokens.len().saturating_sub(1) { - if !count_scope_preposition(tokens[idx].as_str()) { - continue; - } - - let mut cursor = idx + 1; - if cursor < tokens.len() && count_scope_determiner(tokens[cursor].as_str()) { - cursor += 1; - } - - let mut phrase_tokens = Vec::new(); - while cursor < tokens.len() && phrase_tokens.len() < 4 { - let token = tokens[cursor].as_str(); - if count_scope_boundary(token) || count_object_boundary(token) { - break; - } - if !count_object_stopwords().contains(token) { - if let Some(normalized) = normalize_quantity_token(token) { - phrase_tokens.push(normalized); - } - } - cursor += 1; - } - - if phrase_tokens.is_empty() { - continue; - } - - for token in &phrase_tokens { - if seen.insert(token.clone()) { - scoped.push(token.clone()); - } - } - if phrase_tokens.len() >= 2 { - let phrase = phrase_tokens.join("_"); - if seen.insert(phrase.clone()) { - scoped.push(phrase); - } - } - } - - scoped -} - -fn purchase_query_object_tokens(lower: &str) -> Vec<String> { - let tokens = query_tokens(lower); - let mut objects = Vec::new(); - let mut seen = HashSet::new(); - - for idx in 0..tokens.len() { - if !matches!( - tokens[idx].as_str(), - "acquire" - | "acquired" - | "buy" - | "bought" - | "get" - | "got" - | "order" - | "ordered" - | "purchase" - | "purchased" - | "receive" - | "received" - ) { - continue; - } - - let mut cursor = idx + 1; - while cursor < tokens.len() - && matches!(tokens[cursor].as_str(), "my" | "our" | "the" | "a" | "an" | "some") - { - cursor += 1; - } - - let mut phrase = Vec::new(); - while cursor < tokens.len() && phrase.len() < 4 { - let token = tokens[cursor].as_str(); - if matches!(token, "from" | "at" | "via" | "through" | "for" | "with") - || count_object_boundary(token) - || count_scope_boundary(token) - { - break; - } - if !matches!(token, "new" | "current" | "latest" | "same") { - if let Some(normalized) = normalize_quantity_token(token) { - phrase.push(normalized); - } - } - cursor += 1; - } - - for token in &phrase { - if seen.insert(token.clone()) { - objects.push(token.clone()); - } - } - if phrase.len() >= 2 { - let phrase_cue = phrase.join("_"); - if seen.insert(phrase_cue.clone()) { - objects.push(phrase_cue); - } - } - } - - objects -} - -fn small_number_value(token: &str) -> Option<i64> { - match token { - "a" | "an" | "one" => Some(1), - "two" => Some(2), - "three" => Some(3), - "four" => Some(4), - "five" => Some(5), - "six" => Some(6), - "seven" => Some(7), - "eight" => Some(8), - "nine" => Some(9), - "ten" => Some(10), - "eleven" => Some(11), - "twelve" => Some(12), - _ => token.parse::<i64>().ok(), - } -} - -fn temporal_quantity_unit_to_days(quantity: i64, unit: &str) -> Option<i64> { - match unit { - "day" | "days" => Some(quantity), - "week" | "weeks" => Some(quantity * 7), - "month" | "months" => Some(quantity * 30), - "year" | "years" => Some(quantity * 365), - _ => None, - } -} - -fn relative_query_target_date(lower: &str, reference_date: NaiveDate) -> Option<NaiveDate> { - let tokens = query_tokens(lower); - - if tokens.iter().any(|token| token == "yesterday") { - return reference_date.checked_sub_signed(Duration::days(1)); - } - if tokens.iter().any(|token| token == "today") { - return Some(reference_date); - } - if query_has_phrase(&tokens, &["last", "week"]) - || query_has_phrase(&tokens, &["previous", "week"]) - { - return reference_date.checked_sub_signed(Duration::days(7)); - } - - for idx in 0..tokens.len().saturating_sub(2) { - let Some(quantity) = small_number_value(tokens[idx].as_str()) else { - continue; - }; - if tokens[idx + 2] != "ago" { - continue; - } - let Some(days) = temporal_quantity_unit_to_days(quantity, tokens[idx + 1].as_str()) else { - continue; - }; - return reference_date.checked_sub_signed(Duration::days(days)); - } - - None -} - -fn add_relative_temporal_query_cues<F>( - lower: &str, - reference_time: Option<&str>, - intent: &mut QueryIntent, - seen: &mut HashSet<String>, - available: &F, -) -> bool -where - F: Fn(&str) -> bool, -{ - let Some(reference_date) = reference_time.and_then(parse_date_text) else { - return false; - }; - - if lower.contains("this year") || lower.contains("current year") { - add_label(&mut intent.labels, "temporal_resolved_year"); - for cue in ["this", "current", "year", "years"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_year_facet(reference_date), - 1.4, - ); - return true; - } - - if lower.contains("this month") || lower.contains("current month") { - add_label(&mut intent.labels, "temporal_resolved_month"); - for cue in ["this", "current", "month", "months"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_month_facet(reference_date), - 1.5, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_year_facet(reference_date), - 0.7, - ); - return true; - } - - if lower.contains("last month") - || lower.contains("previous month") - || lower.contains("past month") - { - let Some(target_month) = previous_calendar_month_date(reference_date) else { - return false; - }; - add_label(&mut intent.labels, "temporal_resolved_month"); - for cue in ["last", "previous", "past", "month", "months"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_month_facet(target_month), - 1.9, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_year_facet(target_month), - 0.7, - ); - if lower.contains("past month") && target_month.month() != reference_date.month() { - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_month_facet(reference_date), - 0.9, - ); - } - return true; - } - - if lower.contains("this week") || lower.contains("current week") { - add_label(&mut intent.labels, "temporal_resolved_week"); - for cue in ["this", "current", "week", "weeks"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_week_facet(reference_date), - 1.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_month_facet(reference_date), - 0.8, - ); - return true; - }; - let tokens = query_tokens(lower); - if query_has_phrase(&tokens, &["last", "weekend"]) - || query_has_phrase(&tokens, &["past", "weekend"]) - || query_has_phrase(&tokens, &["previous", "weekend"]) - { - let Some((saturday, sunday)) = most_recent_weekend_dates(reference_date) else { - return false; - }; - add_label(&mut intent.labels, "temporal_resolved_weekend"); - for cue in ["last", "past", "previous", "weekend", "weekends"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_date_facet(saturday), - 1.9, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_date_facet(sunday), - 1.9, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_week_facet(sunday), - 0.8, - ); - return true; - } - if query_has_phrase(&tokens, &["last", "week"]) - || query_has_phrase(&tokens, &["previous", "week"]) - { - let Some(target_date) = reference_date.checked_sub_signed(Duration::days(7)) else { - return false; - }; - add_label(&mut intent.labels, "temporal_resolved_week"); - for cue in ["last", "previous", "week", "weeks"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_week_facet(target_date), - 1.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_month_facet(target_date), - 0.8, - ); - return true; - } - let Some(target_date) = relative_query_target_date(lower, reference_date) else { - return false; - }; - - add_label(&mut intent.labels, "temporal_resolved_date"); - for cue in [ - "ago", "day", "days", "week", "weeks", "month", "months", "year", "years", "today", - "yesterday", "last", "previous", "one", "two", "three", "four", "five", "six", - "seven", "eight", "nine", "ten", "eleven", "twelve", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_date_facet(target_date), - 1.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_week_facet(target_date), - 0.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - &source_month_facet(target_date), - 0.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - seen, - available, - "temporal:today", - 1.6, - ); - true -} - -fn is_inventory_query(lower: &str, is_count: bool, is_duration: bool) -> bool { - if is_duration { - return false; - } - - has_any( - lower, - &[ - "current setup", - "currently have", - "currently own", - "currently use", - "do i have", - "do i own", - "my setup", - ], - ) || (is_count - && has_any( - lower, - &[ - "do i have", - "do we have", - "i have", - "we have", - " own", - " set up", - " use", - "including the one", - "include the one", - "plus the one", - ], - )) - || (is_count - && has_any(lower, &[" my ", " our "]) - && has_any( - lower, - &[ - " in ", - " inside ", - " within ", - " across ", - " among ", - " between ", - " both ", - " all ", - " total ", - ], - )) -} - -pub fn compile_query_intent<F>(query: &str, available: F) -> QueryIntent -where - F: Fn(&str) -> bool, -{ - compile_query_intent_with_reference_time(query, None, available) -} - -pub fn compile_query_intent_with_reference_time<F>( - query: &str, - reference_time: Option<&str>, - available: F, -) -> QueryIntent -where - F: Fn(&str) -> bool, -{ - let lower = query.to_lowercase(); - let lower_padded = format!(" {} ", lower); - let normalized_padded = format!(" {} ", crate::nl::normalize_text(query)); - let mut intent = QueryIntent::default(); - let mut seen = HashSet::new(); - - let asks_time_amount = has_any( - &lower, - &[ - "how much time", - "how many seconds", - "how many minutes", - "how many hours", - "how many days", - "how many weeks", - "how many months", - "how many years", - ], - ); - let is_count = has_any( - &lower, - &[ - "how many", - "number of", - "count ", - "total number", - "different ", - ], - ); - let is_completion_count_query = is_count - && has_any( - &normalized_padded, - &[ - " completed ", - " complete ", - " finished ", - " finish ", - " passed ", - " took ", - " taken ", - ], - ); - let is_age_difference = has_any( - &lower, - &[ - "years older", - "year older", - "years younger", - "year younger", - "older am i", - "younger am i", - ], - ) || (lower.contains("how many years") - && has_any( - &lower, - &[ - "how old", - "what age", - "when i", - "when we", - "graduated", - "graduation", - "college", - "degree", - ], - )); - let is_age_query = is_age_difference - || has_any( - &lower, - &[ - "how old", - "what age", - "at what age", - "age was i", - "age am i", - "when i was", - "when we were", - ], - ); - let is_undergraduate_education_query = has_any( - &lower_padded, - &[ - " bachelor's", - " bachelors", - " bachelor ", - " undergraduate", - " undergrad", - ], - ); - let is_education_query = is_age_query - || is_undergraduate_education_query - || has_any( - &lower_padded, - &[ - " degree ", - " degrees ", - " graduated ", - " graduation ", - " graduate ", - " college ", - " university ", - " alma mater ", - ], - ); - let is_money = has_any( - &lower, - &[ - "cost", "costs", "spent", "paid", "price", "money", "dollars", "usd", "$", - ], - ) || (lower.contains("how much") && !asks_time_amount); - let relative_time_operator = has_any( - &lower, - &[ - " ago", - "last ", - "previous ", - "yesterday", - "today", - "tomorrow", - ], - ); - let asks_activity_duration_amount = asks_time_amount - && (has_any( - &lower, - &[ - "how many seconds of", - "how many minutes of", - "how many hours of", - "how many days of", - "how many weeks of", - "how many months of", - "how many years of", - ], - ) || has_any( - &lower, - &[ - "how many seconds did i spend", - "how many seconds did we spend", - "how many minutes did i spend", - "how many minutes did we spend", - "how many hours did i spend", - "how many hours did we spend", - "how many days did i spend", - "how many days did we spend", - "how many weeks did i spend", - "how many weeks did we spend", - "how many months did i spend", - "how many months did we spend", - "how many years did i spend", - "how many years did we spend", - ], - )); - let is_duration = !is_age_query - && ((asks_time_amount && (!relative_time_operator || asks_activity_duration_amount)) - || has_any( - &lower, - &["how long", "duration", "for how many"], - )); - let count_objects = if is_count { - count_query_objects(&lower) - } else { - Vec::new() - }; - let count_scopes = if is_count { - count_query_scope_tokens(&lower) - } else { - Vec::new() - }; - let family_relation_query_facets = family_relation_query_facets(&lower); - let is_co_residence_query = co_residence_query_re().is_match(&lower); - let is_weekly_routine_query = has_any( - &lower, - &[ - "how often", - "typical week", - "usual week", - "normal week", - "per week", - "days a week", - "times a week", - "each week", - "weekly", - ], - ) && !has_any(&lower, &["last week", "past week", "previous week", "week ago"]); - let is_weekday_schedule_query = has_any( - &lower, - &[ - "what day of the week", - "which day of the week", - "what weekday", - "which weekday", - ], - ); - let is_wake_time_routine_query = has_any(&lower, &["what time", "when"]) - && has_any( - &normalized_padded, - &[ - " wake ", - " wake up ", - " waking ", - " waking up ", - ], - ) - && has_any( - &normalized_padded, - &[ - " morning ", - " mornings ", - " weekday ", - " weekdays ", - " weekend ", - " weekends ", - " monday ", - " tuesday ", - " wednesday ", - " thursday ", - " friday ", - " saturday ", - " sunday ", - ], - ) - && !has_any(&lower, &["yesterday", "last night", "last week", "ago"]); - let is_bed_time_query = has_any(&lower, &["what time", "when"]) - && has_any( - &normalized_padded, - &[ - " go to bed ", - " went to bed ", - " get to bed ", - " got to bed ", - " bedtime ", - ], - ); - let is_current = has_any( - &lower, - &[ - "current", - "currently", - "right now", - "latest", - "newest", - "recent", - ], - ); - let is_temporal_order = has_any( - &lower, - &[ - "earliest to latest", - "latest to earliest", - "oldest to newest", - "newest to oldest", - "first to last", - "last to first", - "chronological order", - "reverse chronological", - ], - ) || (lower.contains("order of") - && has_any( - &lower, - &[ - "earliest", - "latest", - "oldest", - "newest", - "first", - "last", - "chronological", - ], - )); - let transport_query_modes = query_transport_modes(query); - let is_transport_mode_comparison = transport_query_modes.len() >= 2 - && has_any( - &lower, - &[ - "mode of transport", - "mode of transportation", - "transport did i use", - "transportation did i use", - "which mode", - "which transport", - ], - ) - && has_any(&lower, &["recent", "recently", "latest", "last"]); - let is_streaming_service_query = has_any( - &normalized_padded, - &[ - " streaming service ", - " streaming services ", - " streaming platform ", - " streaming platforms ", - ], - ) || (has_any(&normalized_padded, &[" which service ", " what service "]) - && has_any( - &normalized_padded, - &[" watch ", " watching ", " show ", " shows "], - )) - || (has_any( - &normalized_padded, - &[ - " service did i start using ", - " service did we start using ", - " service did i use ", - " service did we use ", - ], - ) && has_any(&normalized_padded, &[" stream ", " streaming "])); - let is_music_streaming_service_query = is_streaming_service_query - && has_any( - &normalized_padded, - &[ - " music ", - " song ", - " songs ", - " track ", - " tracks ", - " album ", - " albums ", - " artist ", - " artists ", - " playlist ", - " playlists ", - ], - ); - let is_current_reading_query = has_any( - &normalized_padded, - &[ - " what book am i currently reading ", - " what book am i reading ", - " what book are we currently reading ", - " what book are we reading ", - " which book am i currently reading ", - " which book am i reading ", - " current book ", - " currently reading ", - ], - ); - let is_entity_attribute_query = has_any( - &normalized_padded, - &[ - " what breed ", - " which breed ", - " what type ", - " which type ", - " what model ", - " which model ", - " what brand ", - " which brand ", - " what color ", - " which color ", - " what size ", - " which size ", - " what name ", - " which name ", - ], - ) && has_any(&normalized_padded, &[" my ", " our ", " mine ", " ours "]); - let is_first_person_state = has_any( - &lower, - &[ - " am i ", - "am i ", - " do i ", - "do i ", - " does my ", - "does my ", - " my current ", - " my latest ", - " my newest ", - ], - ); - let is_preference = has_any( - &lower, - &[ - "favorite", - "favourite", - "prefer", - "preference", - "do i like", - "i like", - "recommend for me", - "would i like", - ], - ); - let is_source_answer = has_any( - &lower, - &[ - "what did", - "what was said", - "did you say", - "you said", - "did you tell", - "you told", - "did you suggest", - "you suggested", - "did you recommend", - "you recommended", - "you created", - "you made", - "you wrote", - "you generated", - "you composed", - "provided", - "answer", - "remind me of", - "remind us of", - "remind me what", - "remind us what", - "remind me which", - "remind us which", - "remind me who", - "remind us who", - "remind me where", - "remind us where", - "remind me the", - "remind us the", - ], - ); - let is_activity_event_query = has_any( - &lower, - &[ - "what did i do", - "what did we do", - "activity did i", - "activity did we", - "event did i", - "event did we", - "what event", - "what activity", - "where did i attend", - "where did we attend", - "where did i participate", - "where did we participate", - "visited", - "participated in", - "attended", - ], - ) || (is_temporal_order - && has_any( - &lower, - &[ - "visit", - "visited", - "attend", - "attended", - "participated", - "went to", - ], - )); - let is_charity_event_query = has_any( - &normalized_padded, - &[ - " charity event ", - " charity events ", - " charity walk ", - " charity run ", - " charity ride ", - " charity drive ", - " charity gala ", - " fundraiser ", - " fund raiser ", - ], - ); - let is_competition_event_query = has_any( - &normalized_padded, - &[ - " sport event ", - " sports event ", - " sport events ", - " sports events ", - " tournament ", - " tournaments ", - " race ", - " races ", - " triathlon ", - " triathlons ", - " marathon ", - " marathons ", - " 5k ", - " 10k ", - " match ", - " matches ", - " competition ", - " competitions ", - ], - ); - let is_first_person_activity_query = has_any( - &lower, - &[ - "what did i do", - "what did we do", - "activity did i", - "activity did we", - "event did i", - "event did we", - "where did i", - "where did we", - "did i attend", - "did we attend", - "i attended", - "we attended", - "i participated", - "we participated", - ], - ); - let is_companion_query = companion_query_re().is_match(&lower); - let is_completed_clean_query = completed_clean_query_re().is_match(&lower); - let is_religious_activity_query = has_padded_term( - &normalized_padded, - &[ - "religious activity", - "religious event", - "spiritual activity", - "spiritual event", - "faith activity", - "faith event", - "worship service", - ], - ) || (is_activity_event_query - && (padded_contains_word(&normalized_padded, "religious") - || padded_contains_word(&normalized_padded, "spiritual") - || has_padded_term(&normalized_padded, RELIGIOUS_CONTEXT_TERMS) - || has_padded_term(&normalized_padded, RELIGIOUS_ACTIVITY_TERMS))); - let is_milestone_query = has_any( - &lower, - &[ - "milestone", - "major achievement", - "big achievement", - "significant achievement", - "important achievement", - "major accomplishment", - "significant accomplishment", - ], - ); - let is_decision_query = has_any( - &lower, - &[ - "what did we decide", - "what did i decide", - "what did you decide", - "what did we finally decide", - "what did i finally decide", - "did we decide", - "did i decide", - "finally decided", - "finally decide", - "decided to", - "what did we choose", - "what did i choose", - "which did we choose", - "which did i choose", - "what did we pick", - "what did i pick", - "settled on", - "went with", - ], - ); - let is_naming_decision = is_decision_query - && has_any( - &lower, - &[ - " name", - " names", - " named", - " call it", - " called", - " what to call", - ], - ); - let is_assistant_source = has_any( - &lower, - &[ - "did you say", - "you said", - "did you tell", - "you told", - "you mentioned", - "did you suggest", - "you suggested", - "did you recommend", - "you recommended", - "you created", - "you made", - "you wrote", - "you generated", - "you composed", - "your answer", - "you provided", - "remind me of", - "remind us of", - "remind me what", - "remind us what", - "remind me which", - "remind us which", - "remind me who", - "remind us who", - "remind me where", - "remind us where", - "remind me the", - "remind us the", - ], - ); - let is_user_source = has_any( - &lower, - &[ - "i said", - "i told", - "i mention", - "i mentioned", - "i bring up", - "i brought up", - "i discuss", - "i discussed", - "i talk about", - "i talked about", - "i ask about", - "i asked about", - "i asked", - "my message", - "i wrote", - ], - ); - let is_iteration_reference = has_any( - &lower_padded, - &[ - " second ", - " third ", - " fourth ", - " another ", - " revised ", - " revision ", - " updated version ", - " different version ", - " alternative ", - " next version ", - ], - ); - let is_recipe = - has_any(&lower, &["recipe", "ingredient", "cook", "bake"]) || ingredient_context_re().is_match(&lower); - let is_homegrown = homegrown_source_re().is_match(&lower); - let is_recommendation = has_any( - &lower, - &[ - "recommend", - "suggest", - "should i", - "what should", - "tip", - "advice", - "any ideas", - "ideas on", - "idea for", - "idea about", - ], - ); - let is_personal_recommendation_context = is_recommendation - && !is_source_answer - && !is_assistant_source - && has_any( - &normalized_padded, - &[ - " for me ", - " for us ", - " my ", - " our ", - " i ", - " we ", - " me ", - " us ", - ], - ); - let has_explicit_recommendation_topic = has_any( - &normalized_padded, - &[ - " about ", - " regarding ", - " related to ", - " around ", - " on the topic of ", - " in the area of ", - ], - ); - let is_vague_interest_recommendation = is_personal_recommendation_context - && !has_explicit_recommendation_topic - && has_any( - &lower, - &[ - "find interesting", - "might find", - "would find", - "might like", - "would like", - "of interest", - ], - ); - let is_research_interest_recommendation = is_vague_interest_recommendation - && has_any( - &normalized_padded, - &[ - " publication ", - " publications ", - " conference ", - " conferences ", - " paper ", - " papers ", - " literature ", - " research ", - ], - ); - let is_media_watch_recommendation = is_recommendation - && has_any( - &normalized_padded, - &[ - " movie ", - " movies ", - " show ", - " shows ", - " something to watch ", - " watch tonight ", - " tv show ", - " tv shows ", - ], - ); - let is_inspiration_recommendation = is_personal_recommendation_context - && has_any( - &normalized_padded, - &[ - " inspiration ", - " inspired ", - " inspiring ", - ], - ); - let is_purchase_query = has_any( - &lower, - &[ - "what did i buy", - "what did we buy", - "where did i buy", - "where did we buy", - "where did i get", - "where did we get", - "where did i purchase", - "where did we purchase", - "where did i order", - "where did we order", - "did i buy", - "did we buy", - "what did i purchase", - "what did we purchase", - "what did i order", - "what did we order", - "what did i receive", - "what did we receive", - "what did i acquire", - "what did we acquire", - "what did i get", - "what did we get", - "who did i get", - "who did we get", - "who did i receive", - "who did we receive", - "who did i acquire", - "who did we acquire", - "from whom did i get", - "from whom did we get", - "from whom did i receive", - "from whom did we receive", - "from whom did i acquire", - "from whom did we acquire", - "i bought", - "i purchased", - "i ordered", - "i received", - "i acquired", - ], - ); - let is_purchase_source_query = is_purchase_query - && has_any( - &lower, - &[ - "where did i", - "where did we", - "where was it from", - "where were they from", - "from where", - "from whom", - "who did i", - "who did we", - ], - ); - let is_purchase_consideration_query = has_any( - &normalized_padded, - &[ - " what to look for ", - " what should i look for ", - " what should we look for ", - " tips on what to look for ", - " shopping for ", - " in the market for ", - " looking to buy ", - " looking to purchase ", - " looking to get ", - " planning to buy ", - " planning to purchase ", - " considering buying ", - " considering purchasing ", - " considering upgrading ", - " new one ", - ], - ) || (has_any( - &normalized_padded, - &[" new ", " newer ", " upgrade ", " upgrading "], - ) && has_any( - &normalized_padded, - &[ - " recommend ", - " recommendation ", - " suggest ", - " suggestion ", - " tips ", - " advice ", - " look for ", - ], - )); - let is_navigation = navigation_route_signal(&lower_padded) - || navigation_transit_signal(&lower_padded) - || navigation_station_signal(&lower_padded) - || navigation_fare_signal(&lower_padded) - || navigation_pass_signal(&lower_padded) - || navigation_app_signal(&lower_padded); - let is_sibling_relation_query = has_any( - &lower, - &[ - "sibling", - "siblings", - "brother", - "brothers", - "sister", - "sisters", - ], - ); - let is_family_relation_count = is_count - && is_sibling_relation_query - && has_any( - &lower, - &[ - "i have", - "do i have", - "number of", - "total number", - "how many", - "count of", - ], - ); - let has_temporal_marker = has_any( - &lower, - &[ - "when", - "last ", - "past ", - "ago", - "yesterday", - "today", - "tomorrow", - "week", - "month", - "year", - ], - ); - let is_future_scheduled_advice = has_any( - &lower, - &[ - "this weekend", - "next weekend", - "today", - "tonight", - "tomorrow", - "later", - "soon", - "upcoming", - ], - ) && (is_recommendation - || has_any( - &lower, - &[ - "any tips", - "tips", - "advice", - "planning to", - "thinking about", - "going to", - "looking for", - "want to", - "getting excited about", - ], - )); - let asks_past_or_time_window = has_any( - &lower, - &[ - "what did", - "what was", - "what were", - "where did", - "who did", - "which", - "when", - "how many", - "how much", - "last ", - "past ", - "ago", - "yesterday", - ], - ); - let is_temporal_distance_question = is_count - && lower.contains("ago") - && has_any( - &normalized_padded, - &[ - " day ", - " days ", - " week ", - " weeks ", - " month ", - " months ", - " year ", - " years ", - ], - ); - let is_temporal = !is_age_query - && !is_weekly_routine_query - && !is_weekday_schedule_query - && has_temporal_marker - && (!is_future_scheduled_advice || asks_past_or_time_window); - let is_inventory = !is_family_relation_count - && is_inventory_query(&lower, is_count, is_duration); - - if is_future_scheduled_advice { - for cue in [ - "weekend", - "today", - "tonight", - "tomorrow", - "later", - "soon", - "upcoming", - "trip", - "travel", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - } - - let query_time_of_day = time_of_day_terms(&lower_padded); - if !query_time_of_day.is_empty() { - add_label(&mut intent.labels, "time_of_day"); - for facet in query_time_of_day { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("time_of_day:{}", facet), - 3.4, - ); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:time", - 1.4, - ); - } - - if is_count { - add_label(&mut intent.labels, "count"); - for cue in ["many", "different", "number", "count", "total"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.25); - } - if !is_age_query && !is_duration && !has_person_title_query(&lower) { - for object in &count_objects { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - object, - 8.0, - ); - } - for scope in &count_scopes { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - scope, - 3.6, - ); - } - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:number", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:list", - 1.6, - ); - if is_completion_count_query { - add_label(&mut intent.labels, "completion_count"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "completion_count:object", - 7.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "quantity_count:object", - 5.2, - ); - for object in &count_objects { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("quantity_object:{}", object), - 4.8, - ); - } - } - intent.suppress_generic = true; - } - let is_project_count_query = is_count - && has_any(&normalized_padded, &[" project ", " projects "]) - && has_any( - &lower, - &[ - "have i led", - "have we led", - "am currently leading", - "are currently leading", - "currently leading", - "led or", - "leading", - "working on", - "worked on", - ], - ); - if is_project_count_query { - add_label(&mut intent.labels, "project_work_count"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:project_work", - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:activity", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 3.0, - ); - for cue in ["lead", "led", "leading", "work", "working", "project"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 1.8); - } - } - if is_weekly_routine_query { - add_label(&mut intent.labels, "weekly_routine"); - for cue in ["typical", "usual", "normal", "week", "weekly"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:frequency", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "schedule:frequency", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "frequency_unit:week", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "schedule:weekly", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:weekday", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:time", - 1.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:routine", - 2.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:activity", - 1.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.4, - ); - } - if is_weekday_schedule_query { - add_label(&mut intent.labels, "weekday_schedule"); - for cue in ["day", "week", "weekday"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:weekday", - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "schedule:weekly", - 3.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:routine", - 1.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.4, - ); - } - if is_wake_time_routine_query { - add_label(&mut intent.labels, "wake_time_routine"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "routine:wake_time", - 4.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:time", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:weekday", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:routine", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.2, - ); - } - if is_bed_time_query { - add_label(&mut intent.labels, "bed_time"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "routine:bed_time", - 4.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:time", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "time_of_day:night", - 2.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.0, - ); - } - if is_money { - add_label(&mut intent.labels, "money"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:money", - 3.5, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:number", - 1.5, - ); - intent.suppress_generic = true; - } - if !family_relation_query_facets.is_empty() { - add_label(&mut intent.labels, "family_relation"); - for facet in &family_relation_query_facets { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - facet, - 3.6, - ); - } - } - if is_co_residence_query { - add_label(&mut intent.labels, "co_residence"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "co_residence:with_self", - 4.2, - ); - } - if is_duration { - add_label(&mut intent.labels, "duration"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:duration", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:number", - 1.2, - ); - for object in &count_objects { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - object, - 6.0, - ); - } - for scope in &count_scopes { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - scope, - 4.0, - ); - } - intent.suppress_generic = true; - } - if is_age_query { - add_label(&mut intent.labels, "age_query"); - if is_age_difference { - add_label(&mut intent.labels, "age_difference"); - } - for cue in ["year", "years", "old", "older", "younger"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:age", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "age:current", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "age:event", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:graduation", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:degree", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:college", - 2.0, - ); - intent.suppress_generic = true; - } - if is_education_query && !is_age_query { - add_label(&mut intent.labels, "education_query"); - for cue in [ - "degree", - "degrees", - "bachelor", - "bachelors", - "undergraduate", - "undergrad", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.55); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:degree", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:undergraduate", - 3.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:graduation", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "education:college", - 1.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.2, - ); - } - if is_family_relation_count { - add_label(&mut intent.labels, "family_relation_count"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "family_count:sibling", - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "family_scope:self", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "family_relation:sibling", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "sibling_kind:brother", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "sibling_kind:sister", - 2.8, - ); - intent.suppress_generic = true; - } - if is_inventory { - add_label(&mut intent.labels, "inventory"); - push_adjustment(&mut intent.cue_weight_adjustments, "currently", 0.25); - push_adjustment(&mut intent.cue_weight_adjustments, "current", 0.25); - if has_any( - &lower, - &["including the one", "include the one", "plus the one"], - ) { - push_adjustment(&mut intent.cue_weight_adjustments, "one", 0.25); - push_adjustment(&mut intent.cue_weight_adjustments, "include", 0.45); - push_adjustment(&mut intent.cue_weight_adjustments, "including", 0.45); - push_adjustment(&mut intent.cue_weight_adjustments, "set", 0.45); - push_adjustment(&mut intent.cue_weight_adjustments, "friend", 0.45); - push_adjustment(&mut intent.cue_weight_adjustments, "kid", 0.45); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ownership", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "quantity_count:object", - 8.0, - ); - if !count_scopes.is_empty() { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "inventory_count:contained", - 7.2, - ); - } - for object in count_query_objects(&lower) { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("inventory_object:{}", object), - 3.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("quantity_object:{}", object), - 3.8, - ); - } - } - if is_temporal_order { - add_label(&mut intent.labels, "temporal_order"); - for cue in [ - "order", - "early", - "earliest", - "latest", - "oldest", - "newest", - "first", - "last", - "six", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_time:dated", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.2, - ); - } - if is_transport_mode_comparison { - add_label(&mut intent.labels, "transport_mode_comparison"); - add_label(&mut intent.labels, "temporal_order"); - for cue in ["mode", "transport", "transportation", "use", "recent", "recently"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_time:dated", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:activity", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:event", - 2.2, - ); - for mode in &transport_query_modes { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("transport_event:{mode}"), - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("transport_mode:{mode}"), - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - mode, - 2.0, - ); - } - } - if is_streaming_service_query { - add_label(&mut intent.labels, "streaming_service_usage"); - if is_music_streaming_service_query { - add_label(&mut intent.labels, "music_streaming_service_usage"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:music_streaming", - 4.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:music", - 2.4, - ); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:streaming", - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:usage", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:watching", - 1.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.4, - ); - for cue in ["service", "start", "started", "use", "using", "used", "recently"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.45); - } - } - if is_current_reading_query { - add_label(&mut intent.labels, "current_reading"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "reading:current", - 4.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:book_reading", - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:book", - 2.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:current", - 2.0, - ); - for cue in ["currently", "current", "read", "reading"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.55); - } - } - if is_current - && !is_inventory - && !is_temporal_order - && !is_transport_mode_comparison - && !is_streaming_service_query - && !is_current_reading_query - { - add_label(&mut intent.labels, "latest_current"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:current", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:recent", - 1.8, - ); - } - if (is_current - && !is_inventory - && !is_temporal_order - && !is_transport_mode_comparison - && !is_weekly_routine_query - && !is_personal_recommendation_context - && !is_streaming_service_query - && !is_current_reading_query) - || (is_first_person_state - && !is_age_query - && !is_inventory - && !is_temporal_order - && !is_transport_mode_comparison - && !is_weekly_routine_query - && !is_personal_recommendation_context - && !is_streaming_service_query - && !is_current_reading_query) - { - add_label(&mut intent.labels, "state_update"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:update", - 2.6, - ); - } - if lower.contains("what type of") { - push_adjustment(&mut intent.cue_weight_adjustments, "type", 0.25); - } - if is_preference { - add_label(&mut intent.labels, "preference"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:preference", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:dislike", - 2.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ownership", - 1.2, - ); - } - if is_purchase_query { - add_label(&mut intent.labels, "purchase"); - let purchase_objects = purchase_query_object_tokens(&lower); - for object in &purchase_objects { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - object, - if object.contains('_') { 4.4 } else { 3.4 }, - ); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ownership", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "purchase:acquired", - 4.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:activity", - 1.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:event", - 1.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.4, - ); - if is_purchase_source_query { - add_label(&mut intent.labels, "purchase_source"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "purchase:source", - 3.6, - ); - } - } - if is_purchase_consideration_query { - add_label(&mut intent.labels, "purchase_consideration"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:purchase_consideration", - 3.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ownership", - 1.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:preference", - 1.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.8, - ); - } - if is_entity_attribute_query { - add_label(&mut intent.labels, "entity_attribute"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:entity_attribute", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "attribute:class_relation", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - if is_preference { 3.2 } else { 2.0 }, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ownership", - 1.0, - ); - } - if is_source_answer { - add_label(&mut intent.labels, "source_answer"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:answer", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:recommendation", - 2.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:list", - 1.4, - ); - } - if is_iteration_reference { - add_label(&mut intent.labels, "iteration_reference"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:iteration", - 3.2, - ); - } - if is_activity_event_query || is_companion_query || is_completed_clean_query { - add_label(&mut intent.labels, "activity_event"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:activity", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:event", - 2.2, - ); - if is_charity_event_query { - add_label(&mut intent.labels, "charity_event"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "event_domain:charity", - 3.4, - ); - } - push_adjustment(&mut intent.cue_weight_adjustments, "activity", 0.6); - push_adjustment(&mut intent.cue_weight_adjustments, "event", 0.6); - if is_first_person_activity_query { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.4, - ); - } - if is_companion_query { - add_label(&mut intent.labels, "companion"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "companion:with", - 5.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.2, - ); - } - if is_completed_clean_query { - add_label(&mut intent.labels, "completed_action"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "completed_action:clean", - 5.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.2, - ); - } - } - if is_competition_event_query { - add_label(&mut intent.labels, "competition_event"); - for cue in [ - "participate", - "participated", - "participating", - "entered", - "joined", - "raced", - "ran", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 2.4); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:competition_event", - 4.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "activity_domain:sport", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:activity", - 2.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:event", - 2.0, - ); - } - if is_religious_activity_query { - add_label(&mut intent.labels, "religious_activity"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "activity_domain:religion", - 4.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "topic:religion", - 1.6, - ); - push_adjustment(&mut intent.cue_weight_adjustments, "religious", 0.7); - push_adjustment(&mut intent.cue_weight_adjustments, "spiritual", 0.7); - } - if is_milestone_query { - add_label(&mut intent.labels, "milestone"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:milestone", - 3.2, - ); - push_adjustment(&mut intent.cue_weight_adjustments, "significant", 0.6); - push_adjustment(&mut intent.cue_weight_adjustments, "important", 0.6); - push_adjustment(&mut intent.cue_weight_adjustments, "major", 0.6); - } - if is_decision_query { - add_label(&mut intent.labels, "decision_selection"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:decision", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:selection", - 2.6, - ); - if is_naming_decision { - add_label(&mut intent.labels, "naming_decision"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:naming", - 2.4, - ); - } - } - if is_assistant_source { - add_label(&mut intent.labels, "source_assistant"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:assistant", - 2.5, - ); - } - if is_user_source { - add_label(&mut intent.labels, "source_user"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.5, - ); - } - if is_temporal { - add_label(&mut intent.labels, "temporal_window"); - for month in explicit_query_months(&lower) { - if let Some(facet) = content_month_facet(month) { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &facet, - 4.0, - ); - } - } - let added_specific_source_window = add_relative_temporal_query_cues( - &lower, - reference_time, - &mut intent, - &mut seen, - &available, - ); - if !added_specific_source_window { - if is_temporal_distance_question { - add_label(&mut intent.labels, "temporal_distance"); - for cue in [ - "ago", "day", "days", "week", "weeks", "month", "months", "year", - "years", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:date", - 2.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:relative", - 1.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.1, - ); - } else { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "has:date", - 2.3, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:relative", - 2.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:last_week", - 2.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "temporal:recent", - 1.5, - ); - } - } - } - if is_recipe { - add_label(&mut intent.labels, "recipe"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:recipe", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ingredient", - 2.4, - ); - } - if is_homegrown { - add_label(&mut intent.labels, "homegrown"); - for cue in ["ingredient", "ingredients", "recipe", "recipes", "weekend"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.35); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:homegrown", - 3.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ingredient", - 2.2, - ); - } - if is_recommendation { - add_label(&mut intent.labels, "recommendation"); - for cue in [ - "can", - "could", - "would", - "should", - "please", - "recommend", - "recommendation", - "suggest", - "suggestion", - "think", - "try", - "new", - "recipe", - "recipes", - "useful", - "good", - "best", - "some", - "any", - ] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.45); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:recommendation", - 2.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:preference", - 1.8, - ); - if is_vague_interest_recommendation { - for cue in ["might", "find", "interest", "interesting"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.45); - } - } else { - for (cue, weight) in recommendation_topic_cues(query) { - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &cue, - weight, - ); - } - } - } - if is_personal_recommendation_context { - add_label(&mut intent.labels, "personal_recommendation_context"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 2.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:ownership", - 2.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:usage", - 1.8, - ); - } - if is_vague_interest_recommendation { - add_label(&mut intent.labels, "vague_interest_recommendation"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:interest", - 2.4, - ); - } - if is_research_interest_recommendation { - add_label(&mut intent.labels, "research_interest_recommendation"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:expertise", - 3.2, - ); - } - if is_media_watch_recommendation { - add_label(&mut intent.labels, "media_watch_recommendation"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "media:watching", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "source_role:user", - 1.4, - ); - for cue in ["show", "shows", "movie", "movies", "watch", "tonight"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.65); - } - } - if is_inspiration_recommendation { - add_label(&mut intent.labels, "inspiration_recommendation"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:inspiration_source", - 4.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:interest", - 2.2, - ); - } - if is_navigation { - add_label(&mut intent.labels, "navigation"); - for cue in ["bit", "got", "around", "helpful", "tip"] { - push_adjustment(&mut intent.cue_weight_adjustments, cue, 0.45); - } - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:navigation", - 3.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "travel:route", - 3.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "travel:transit", - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "travel:station", - 2.4, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "travel:fare", - 2.2, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "travel:pass", - 3.6, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "travel:app", - 3.6, - ); - } - - let is_state_transition_query = has_any( - &normalized_padded, - &[ - " switch ", - " switched ", - " change ", - " changed ", - " move ", - " moved ", - ], - ) && has_any(&lower_padded, &[" from ", " to ", " what ", " which "]); - if is_state_transition_query { - add_label(&mut intent.labels, "state_transition"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:update", - 3.0, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - "type:selection", - 2.4, - ); - } - - for (relation, anchor) in temporal_event_relations(query) { - add_label(&mut intent.labels, "temporal_event_relation"); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("temporal_relation:{relation}"), - 2.8, - ); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &format!("temporal_anchor:{anchor}"), - 3.0, - ); - } - - add_person_query_intent(&lower, is_count, &mut intent, &mut seen, &available); - - let mut entity_facets = Vec::new(); - let mut entity_seen = HashSet::new(); - add_entity_facets(query, &mut entity_facets, &mut entity_seen); - for cue in entity_facets { - push_weighted_if_available(&mut intent.weighted_cues, &mut seen, &available, &cue, 2.2); - if let Some(value) = cue.strip_prefix("entity:") { - if let Some(initials) = normalized_phrase_initialism(value) { - let initialism_cue = format!("entity:{}", initials); - push_weighted_if_available( - &mut intent.weighted_cues, - &mut seen, - &available, - &initialism_cue, - 2.8, - ); - } - } - } - - intent -} - -pub fn compile_query_intent_with_cuepacks<F>( - query: &str, - reference_time: Option<&str>, - available: F, - cuepacks: &crate::cuepacks::CuePackRegistry, - cuepack_selection: Option<&[String]>, -) -> QueryIntent -where - F: Fn(&str) -> bool, -{ - let mut intent = compile_query_intent_with_reference_time(query, reference_time, &available); - let pack_output = cuepacks.compile_query_intent(query, cuepack_selection, &available); - - for label in pack_output.labels { - add_label(&mut intent.labels, &label); - } - - let mut seen = intent - .weighted_cues - .iter() - .map(|(cue, _)| cue.clone()) - .collect::<HashSet<_>>(); - for (cue, weight) in pack_output.weighted_cues { - if seen.insert(cue.clone()) { - intent.weighted_cues.push((cue, weight)); - } - } - - for (cue, multiplier) in pack_output.cue_weight_adjustments { - push_adjustment(&mut intent.cue_weight_adjustments, &cue, multiplier); - } - - intent.cuepack_rules = pack_output.matched_rules; - intent.suppress_generic |= pack_output.suppress_generic; - intent -} - -pub fn is_weak_query_cue(cue: &str) -> bool { - matches!( - cue, - "many" - | "number" - | "count" - | "total" - | "different" - | "time" - | "times" - | "current" - | "currently" - | "latest" - | "newest" - | "recent" - | "recently" - | "past" - | "last" - | "ago" - | "week" - | "month" - | "year" - | "long" - | "much" - | "cost" - | "price" +pub fn is_weak_query_cue(cue: &str) -> bool { + matches!( + cue, + "many" | "number" | "count" | "total" | "different" | "time" | "times" | "current" + | "currently" | "latest" | "newest" | "recent" | "recently" | "past" | "last" + | "ago" | "week" | "month" | "year" | "long" | "much" | "cost" | "price" ) } #[cfg(test)] -mod tests { - use super::{compile_query_intent, extract_memory_facets_core}; - use std::collections::HashSet; - - #[test] - fn personal_switch_emits_clean_update_preference_and_temporal_facets() { - let facets = extract_memory_facets_core( - "Maya switched from coffee to mint tea after the April deploy.", - None, - &[], - ); - - for expected in [ - "type:update", - "type:selection", - "type:preference", - "preference:changed", - "preference_contrast:coffee", - "preference_value:mint", - "preference_value:tea", - "preference_value:mint_tea", - "has:date", - "content_month:04", - "temporal_relation:after", - "temporal_anchor:april_deploy", - "entity:maya", - ] { - assert!(facets.contains(&expected.to_string()), "missing {expected}: {facets:?}"); - } - assert!(!facets.contains(&"entity:april".to_string())); - assert!(!facets.iter().any(|cue| cue.starts_with("person_role_phrase:"))); - } - - #[test] - fn capitalized_month_remains_entity_without_temporal_context() { - let facets = extract_memory_facets_core("April joined Maya on the project.", None, &[]); - assert!(facets.contains(&"entity:april".to_string())); - assert!(facets.contains(&"entity:maya".to_string())); - } - - #[test] - fn structural_role_phrase_survives_without_a_role_dictionary() { - let facets = extract_memory_facets_core("Engineering manager Maya approved it.", None, &[]); - assert!(facets.contains(&"person_role_phrase:engineering_manager".to_string())); - assert!(facets.contains(&"person_ref:named".to_string())); - } - - #[test] - fn non_person_transition_is_not_promoted_to_preference() { - let facets = extract_memory_facets_core( - "The service switched from primary to fallback after the outage.", - None, - &[], - ); - assert!(facets.contains(&"type:update".to_string())); - assert!(!facets.contains(&"type:preference".to_string())); - assert!(!facets.contains(&"preference:changed".to_string())); - } - - #[test] - fn transition_query_reuses_update_and_event_relation_facets() { - let available = HashSet::from([ - "type:update".to_string(), - "type:selection".to_string(), - "temporal_relation:after".to_string(), - "temporal_anchor:april_deploy".to_string(), - ]); - let intent = compile_query_intent( - "What did Maya switch to after the April deploy?", - |cue| available.contains(cue), - ); - - assert!(intent.labels.contains(&"state_transition".to_string())); - assert!(intent.labels.contains(&"temporal_event_relation".to_string())); - for expected in [ - "type:update", - "type:selection", - "temporal_relation:after", - "temporal_anchor:april_deploy", - ] { - assert!( - intent.weighted_cues.iter().any(|(cue, _)| cue == expected), - "missing {expected}: {:?}", - intent.weighted_cues - ); - } - } -} +#[path = "../tests/unit/facets.rs"] +mod tests; diff --git a/src/grounding.rs b/src/grounding.rs index f2df804..5ec0e4b 100644 --- a/src/grounding.rs +++ b/src/grounding.rs @@ -163,3 +163,93 @@ pub fn create_grounding_proof( excluded_top, } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::collections::HashMap; + + fn result(id: MemoryId, content: &str, metadata: HashMap<String, serde_json::Value>) -> RecallResult { + RecallResult { + memory_id: id, + content: content.to_string(), + score: 0.9, + match_integrity: 0.8, + intersection_count: 2, + recency_score: 0.3, + reinforcement_score: 0.4, + salience_score: 0.5, + created_at: 1_700_000_000.0, + metadata, + explain: None, + } + } + + #[test] + fn estimates_tokens_from_character_count() { + assert_eq!(GroundingEngine::estimate_tokens(""), 0); + assert_eq!(GroundingEngine::estimate_tokens("1234"), 1); + assert_eq!(GroundingEngine::estimate_tokens("12345"), 2); + } + + #[test] + fn selects_items_with_metadata_and_formats_context() { + let mut metadata = HashMap::new(); + metadata.insert("source".to_string(), json!("notes.md")); + metadata.insert("timestamp".to_string(), json!("2024-01-01T00:00:00Z")); + let (selected, excluded, block) = GroundingEngine::select_memories( + "query".to_string(), + vec!["query".to_string()], + vec![("query".to_string(), 1.0)], + vec![result(7, "1234", metadata)], + 1, + ); + + assert_eq!(selected.len(), 1); + assert!(excluded.is_empty()); + assert_eq!(selected[0].memory_id, 7); + assert_eq!(selected[0].source, "notes.md"); + assert_eq!(selected[0].timestamp, "2024-01-01T00:00:00Z"); + assert!(block.contains("[VERIFIED CONTEXT]")); + assert!(block.contains("notes.md")); + assert!(block.ends_with("[/VERIFIED CONTEXT]")); + } + + #[test] + fn excludes_top_five_items_that_exceed_budget_and_falls_back_to_timestamp() { + let results = (1..=7) + .map(|id| result(id, "12345", HashMap::new())) + .collect(); + let (selected, excluded, block) = GroundingEngine::select_memories( + "query".to_string(), + Vec::new(), + Vec::new(), + results, + 2, + ); + + assert_eq!(selected.len(), 1); + assert_eq!(excluded.len(), 5); + assert!(excluded[0].reason.contains("Exceeds remaining token budget")); + assert!(selected[0].timestamp.starts_with("2023-11-")); + assert!(!block.is_empty()); + assert!(GroundingEngine::format_context_block(&[]).is_empty()); + } + + #[test] + fn creates_a_serializable_grounding_proof() { + let proof = create_grounding_proof( + "trace-1".to_string(), + "what?".to_string(), + vec!["what".to_string()], + vec![("what".to_string(), 0.75)], + 128, + Vec::new(), + Vec::new(), + ); + assert_eq!(proof.trace_id, "trace-1"); + assert_eq!(proof.token_budget, 128); + assert_eq!(serde_json::to_value(&proof).unwrap()["query_text"], "what?"); + } +} diff --git a/src/intent.rs b/src/intent.rs new file mode 100644 index 0000000..cc564aa --- /dev/null +++ b/src/intent.rs @@ -0,0 +1,711 @@ +//! Frozen-MiniLM intent classification shared by CueKey, ingestion jobs, and +//! the hybrid recall reranker. +//! +//! A tiny offline-trained linear head maps sentence embeddings to intent +//! scores. Runtime classification contains no semantic word or phrase rules. +//! Scores are ranking logits rather than calibrated probabilities; the stored +//! margin and confidence weight make uncertain classifications matter less. + +use crate::semantic::SemanticEncoder; +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::sync::Arc; + +pub const INTENT_TAXONOMY_VERSION: &str = "cuekey-intents-v4"; +pub const INTENT_LABELS: [&str; 8] = [ + "preference", + "decision", + "standing_instruction", + "personal_fact", + "event_or_plan", + "summary_or_timeline", + "chitchat", + "action_or_command", +]; + +const RECALL_LABELS: [&str; 6] = [ + "preference", + "decision", + "standing_instruction", + "personal_fact", + "event_or_plan", + "summary_or_timeline", +]; + +const PROBE_MAGIC: &[u8; 8] = b"CMPINTP1"; +const QINT8_PROBE: &[u8] = + include_bytes!("../assets/all-MiniLM-L3-v2/intent_probe_qint8.head"); +const Q4_PROBE: &[u8] = include_bytes!("../assets/all-MiniLM-L3-v2/intent_probe_q4.head"); + +#[derive(Debug)] +struct IntentProbe { + dimensions: usize, + score_scale: f32, + weights: Vec<f32>, + biases: Vec<f32>, +} + +impl IntentProbe { + fn from_bytes(bytes: &[u8]) -> Result<Self, String> { + const HEADER_BYTES: usize = 20; + if bytes.len() < HEADER_BYTES || &bytes[..PROBE_MAGIC.len()] != PROBE_MAGIC { + return Err("intent probe has an invalid header".to_string()); + } + let dimensions = read_u32(bytes, 8)? as usize; + let class_count = read_u32(bytes, 12)? as usize; + let score_scale = read_f32(bytes, 16)?; + if dimensions == 0 || class_count != INTENT_LABELS.len() { + return Err("intent probe dimensions or class count are invalid".to_string()); + } + if !score_scale.is_finite() || score_scale <= 0.0 { + return Err("intent probe score scale is invalid".to_string()); + } + let weight_count = dimensions + .checked_mul(class_count) + .ok_or_else(|| "intent probe dimensions overflow".to_string())?; + let value_count = weight_count + .checked_add(class_count) + .ok_or_else(|| "intent probe value count overflow".to_string())?; + let expected_bytes = value_count + .checked_mul(std::mem::size_of::<f32>()) + .and_then(|size| size.checked_add(HEADER_BYTES)) + .ok_or_else(|| "intent probe byte count overflow".to_string())?; + if bytes.len() != expected_bytes { + return Err(format!( + "intent probe size mismatch: expected {expected_bytes}, received {}", + bytes.len() + )); + } + let mut values = Vec::with_capacity(value_count); + for offset in (HEADER_BYTES..bytes.len()).step_by(std::mem::size_of::<f32>()) { + let value = read_f32(bytes, offset)?; + if !value.is_finite() { + return Err("intent probe contains a non-finite value".to_string()); + } + values.push(value); + } + let biases = values.split_off(weight_count); + Ok(Self { + dimensions, + score_scale, + weights: values, + biases, + }) + } + + fn scores(&self, embedding: &[f32]) -> Result<Vec<f32>, String> { + if embedding.len() != self.dimensions { + return Err(format!( + "intent probe dimension mismatch: expected {}, received {}", + self.dimensions, + embedding.len() + )); + } + let mut scores = Vec::with_capacity(INTENT_LABELS.len()); + for (class, weights) in self.weights.chunks_exact(self.dimensions).enumerate() { + let score = weights + .iter() + .zip(embedding) + .map(|(weight, value)| weight * value) + .sum::<f32>(); + scores.push((score + self.biases[class]) * self.score_scale); + } + Ok(scores) + } +} + +fn read_u32(bytes: &[u8], offset: usize) -> Result<u32, String> { + let value = bytes + .get(offset..offset + 4) + .ok_or_else(|| "intent probe ended unexpectedly".to_string())?; + Ok(u32::from_le_bytes(value.try_into().map_err(|_| { + "intent probe contains an invalid integer".to_string() + })?)) +} + +fn read_f32(bytes: &[u8], offset: usize) -> Result<f32, String> { + let value = bytes + .get(offset..offset + 4) + .ok_or_else(|| "intent probe ended unexpectedly".to_string())?; + Ok(f32::from_le_bytes(value.try_into().map_err(|_| { + "intent probe contains an invalid float".to_string() + })?)) +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum IntentTarget { + Query, + Memory, +} + +impl Default for IntentTarget { + fn default() -> Self { + Self::Query + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IntentClassification { + pub primary_intent: String, + pub scores: BTreeMap<String, f32>, + pub top_intents: Vec<String>, + pub top_score: f32, + pub margin: f32, + pub confidence_weight: f32, + /// Independent query-side gate for whether CueKey should ask CueMap for + /// recall. This is deliberately separate from `primary_intent`: a + /// historical question can be semantically ambiguous without becoming a + /// no-recall command or social response. + #[serde(default)] + pub recall_eligible: bool, + pub recall_action: String, + /// Whether this text is useful as durable memory evidence. For memory + /// classifications this is the signal used by the hybrid reranker to + /// distinguish ordinary evidence from social/imperative noise. + pub memory_eligible: bool, + pub model_version: String, + pub taxonomy_version: String, +} + +impl IntentClassification { + pub fn is_recall_intent(&self) -> bool { + // The action string keeps old persisted classifications readable while + // the explicit field is populated by the current taxonomy. + self.recall_eligible || self.recall_action == "recall" + } + + pub fn score(&self, label: &str) -> f32 { + self.scores.get(label).copied().unwrap_or(0.0) + } + + /// Convert the ranking scores into a soft distribution for reranking. + /// This is intentionally only a relative softmax, not a calibrated + /// probability estimate. + pub fn soft_distribution(&self, temperature: f32) -> BTreeMap<String, f32> { + let temperature = temperature.max(0.001); + let max_score = self + .scores + .values() + .copied() + .fold(f32::NEG_INFINITY, f32::max); + let mut values = BTreeMap::new(); + let mut total = 0.0f32; + for label in INTENT_LABELS { + let value = ((self.score(label) - max_score) / temperature).exp(); + values.insert(label.to_string(), value); + total += value; + } + if total > 0.0 && total.is_finite() { + for value in values.values_mut() { + *value /= total; + } + } + values + } +} + +pub struct IntentClassifier { + encoder: Arc<dyn SemanticEncoder>, + probe: IntentProbe, + model_version: String, +} + +impl IntentClassifier { + pub fn new( + encoder: Arc<dyn SemanticEncoder>, + model_version: impl Into<String>, + ) -> Result<Self, String> { + let model_version = model_version.into(); + let bytes = match model_version.as_str() { + "bundled-qint8-minilm-l3" => QINT8_PROBE, + "bundled-q4-minilm-l3" => Q4_PROBE, + _ => { + return Err(format!( + "no trained intent probe is available for semantic model {model_version}" + )) + } + }; + let probe = IntentProbe::from_bytes(bytes)?; + if encoder.dimensions() != probe.dimensions { + return Err(format!( + "intent probe dimension mismatch: model expects {}, encoder provides {}", + probe.dimensions, + encoder.dimensions() + )); + } + Ok(Self { + encoder, + probe, + model_version, + }) + } + + pub fn classify(&self, text: &str, target: IntentTarget) -> Result<IntentClassification, String> { + let vector = self.encoder.encode(text)?; + self.classify_vector(text, &vector, target) + } + + pub fn classify_with_embedding( + &self, + text: &str, + embedding: &[f32], + target: IntentTarget, + ) -> Result<IntentClassification, String> { + if embedding.len() != self.encoder.dimensions() { + return Err(format!( + "intent embedding dimension mismatch: expected {}, received {}", + self.encoder.dimensions(), + embedding.len() + )); + } + self.classify_vector(text, embedding, target) + } + + fn classify_vector( + &self, + text: &str, + vector: &[f32], + target: IntentTarget, + ) -> Result<IntentClassification, String> { + let mut normalized_vector = vector.to_vec(); + normalize(&mut normalized_vector)?; + let scores = self.probe.scores(&normalized_vector)?; + let signals = structural_signals(text); + + let mut order = (0..INTENT_LABELS.len()).collect::<Vec<_>>(); + order.sort_unstable_by(|left, right| { + scores[*right] + .partial_cmp(&scores[*left]) + .unwrap_or(Ordering::Equal) + }); + let top = order[0]; + let second = order[1]; + let top_score = scores[top]; + let margin = top_score - scores[second]; + let margin_weight = ((margin - 0.02) / 0.13).clamp(0.0, 1.0); + let score_weight = ((top_score + 0.05) / 0.35).clamp(0.0, 1.0); + let confidence_weight = (margin_weight * 0.75 + score_weight * 0.25).clamp(0.0, 1.0); + let primary_intent = INTENT_LABELS[top].to_string(); + let recall_eligible = recall_eligible(&primary_intent, &signals, margin, target); + let memory_eligible = memory_eligible(&primary_intent); + let recall_action = if recall_eligible { + "recall" + } else { + "no_recall" + }; + let mut score_map = BTreeMap::new(); + for (index, label) in INTENT_LABELS.iter().enumerate() { + score_map.insert((*label).to_string(), scores[index]); + } + + Ok(IntentClassification { + primary_intent, + scores: score_map, + top_intents: order + .iter() + .take(3) + .map(|index| INTENT_LABELS[*index].to_string()) + .collect(), + top_score, + margin, + confidence_weight, + recall_eligible, + recall_action: recall_action.to_string(), + memory_eligible, + model_version: self.model_version.clone(), + taxonomy_version: INTENT_TAXONOMY_VERSION.to_string(), + }) + } + +} + +/// Soft compatibility between a query intent distribution and a memory +/// intent distribution. Exact intent matches are intentionally strongest. +/// Only a small set of semantically adjacent types receive a secondary +/// match; unrelated types remain neutral so intent reranking rewards useful +/// matches instead of inflating the entire candidate slate. The query's +/// primary intent appearing in the memory's top three intents is also an +/// explicit directional signal because this is the useful retrieval-side +/// question: does this memory plausibly belong to the query's intent? +pub fn intent_compatibility( + query: &IntentClassification, + memory: &IntentClassification, +) -> f64 { + let query_distribution = query.soft_distribution(0.08); + let memory_distribution = memory.soft_distribution(0.08); + let mut total = 0.0f32; + for query_label in INTENT_LABELS { + let query_value = query_distribution.get(query_label).copied().unwrap_or(0.0); + for memory_label in INTENT_LABELS { + let memory_value = memory_distribution.get(memory_label).copied().unwrap_or(0.0); + total += query_value * memory_value * compatibility_weight(query_label, memory_label); + } + } + let soft_compatibility = total.clamp(0.0, 1.0); + let top3_compatibility = directional_top3_compatibility(query, memory); + f64::from(soft_compatibility.max(top3_compatibility).clamp(0.0, 1.0)) +} + +fn directional_top3_compatibility( + query: &IntentClassification, + memory: &IntentClassification, +) -> f32 { + match memory + .top_intents + .iter() + .position(|intent| intent == &query.primary_intent) + { + Some(0) => 1.0, + Some(1) => 0.70, + Some(2) => 0.45, + _ => 0.0, + } +} + +fn compatibility_weight(query: &str, memory: &str) -> f32 { + if query == memory { + return 1.0; + } + match (query, memory) { + ("summary_or_timeline", "event_or_plan") + | ("event_or_plan", "summary_or_timeline") => 0.45, + ("summary_or_timeline", "decision") + | ("decision", "summary_or_timeline") => 0.35, + ("summary_or_timeline", "preference") + | ("preference", "summary_or_timeline") + | ("summary_or_timeline", "personal_fact") + | ("personal_fact", "summary_or_timeline") + | ("summary_or_timeline", "standing_instruction") + | ("standing_instruction", "summary_or_timeline") => 0.25, + ("event_or_plan", "decision") + | ("decision", "event_or_plan") + | ("decision", "standing_instruction") + | ("standing_instruction", "decision") => 0.35, + ("preference", "personal_fact") | ("personal_fact", "preference") => 0.40, + _ => 0.0, + } +} + +#[derive(Debug, Clone, Copy)] +struct StructuralSignals { + question_like: bool, + continuation: bool, +} + +const SHORT_MESSAGE_MAX_TOKENS: usize = 12; +const STRUCTURAL_RECALL_FALLBACK_MARGIN: f32 = 0.05; + +fn structural_signals(text: &str) -> StructuralSignals { + let lower = text.trim().to_ascii_lowercase(); + let tokens = lower + .split(|character: char| !character.is_ascii_alphabetic()) + .filter(|token| !token.is_empty()) + .collect::<Vec<_>>(); + let first = tokens.first().copied().unwrap_or_default(); + let question_like = text.contains('?') + || matches!( + first, + "what" + | "when" + | "where" + | "which" + | "who" + | "whom" + | "whose" + | "why" + | "how" + | "am" + | "are" + | "is" + | "was" + | "were" + | "do" + | "does" + | "did" + | "have" + | "has" + | "had" + | "can" + | "could" + | "will" + | "would" + | "should" + | "might" + | "may" + | "must" + ); + let last = tokens.last().copied().unwrap_or_default(); + let short_message = tokens.len() <= SHORT_MESSAGE_MAX_TOKENS; + let continuation = short_message + && (text.contains("...") + || text.contains('…') + || matches!( + last, + "at" | "in" | "on" | "to" | "with" | "for" | "the" | "my" | "our" + )); + + StructuralSignals { + question_like, + continuation, + } +} + +fn recall_eligible( + primary_intent: &str, + signals: &StructuralSignals, + margin: f32, + target: IntentTarget, +) -> bool { + if RECALL_LABELS.contains(&primary_intent) { + return true; + } + target == IntentTarget::Query + && margin < STRUCTURAL_RECALL_FALLBACK_MARGIN + && (signals.question_like || signals.continuation) +} + +fn memory_eligible(primary_intent: &str) -> bool { + RECALL_LABELS.contains(&primary_intent) +} + +fn normalize(vector: &mut [f32]) -> Result<(), String> { + let norm = vector.iter().map(|value| value * value).sum::<f32>().sqrt(); + if !norm.is_finite() || norm <= f32::EPSILON { + return Err("intent vector has zero or invalid magnitude".to_string()); + } + for value in vector { + *value /= norm; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct AxisEncoder; + + impl SemanticEncoder for AxisEncoder { + fn dimensions(&self) -> usize { + INTENT_LABELS.len() + } + + fn encode(&self, _text: &str) -> Result<Vec<f32>, String> { + Err("test encoder requires an explicit embedding".to_string()) + } + } + + fn intent_axis(index: usize) -> Vec<f32> { + let mut vector = vec![0.0; INTENT_LABELS.len()]; + vector[index] = 1.0; + vector + } + + fn test_classifier() -> IntentClassifier { + let mut weights = vec![0.0; INTENT_LABELS.len() * INTENT_LABELS.len()]; + for index in 0..INTENT_LABELS.len() { + weights[index * INTENT_LABELS.len() + index] = 1.0; + } + IntentClassifier { + encoder: Arc::new(AxisEncoder), + probe: IntentProbe { + dimensions: INTENT_LABELS.len(), + score_scale: 1.0, + weights, + biases: vec![0.0; INTENT_LABELS.len()], + }, + model_version: "test-model".to_string(), + } + } + + #[test] + fn bundled_probes_are_well_formed() { + for bytes in [QINT8_PROBE, Q4_PROBE] { + let probe = IntentProbe::from_bytes(bytes).unwrap(); + assert_eq!(probe.dimensions, 384); + assert_eq!(probe.biases.len(), INTENT_LABELS.len()); + assert_eq!( + probe.weights.len(), + probe.dimensions * INTENT_LABELS.len() + ); + } + } + + #[test] + fn classifier_requires_a_probe_for_the_exact_encoder_version() { + let error = IntentClassifier::new(Arc::new(AxisEncoder), "unknown-model") + .err() + .unwrap(); + assert!(error.contains("no trained intent probe")); + } + + #[cfg(feature = "semantic-encoder")] + #[test] + fn bundled_encoder_and_probe_classify_unseen_sentences() { + let config = crate::semantic::SemanticConfig::default().resolved(); + let encoder = crate::semantic::load_configured_encoder(&config) + .unwrap() + .unwrap(); + let classifier = IntentClassifier::new(encoder, config.model_version).unwrap(); + + for (text, expected) in [ + ("Could you lint this crate now?", "action_or_command"), + ("That explanation was lovely, cheers.", "chitchat"), + ("Which option did the team ultimately approve?", "decision"), + ] { + let classification = classifier.classify(text, IntentTarget::Query).unwrap(); + assert_eq!(classification.primary_intent, expected); + } + } + + #[test] + fn surface_vocabulary_cannot_override_the_embedding_category() { + let classifier = test_classifier(); + + for (text, semantic_intent) in [ + ("Run tests and commit the changes", 0), + ("Thanks, got it and understood", 1), + ("We decided and settled on this", 4), + ] { + let classification = classifier + .classify_with_embedding(text, &intent_axis(semantic_intent), IntentTarget::Query) + .unwrap(); + assert_eq!(classification.primary_intent, INTENT_LABELS[semantic_intent]); + assert_eq!(classification.top_score, 1.0); + assert_eq!(classification.margin, 1.0); + } + } + + #[test] + fn structural_shape_only_admits_uncertain_queries() { + let classifier = test_classifier(); + let mut uncertain_no_recall = vec![0.0; INTENT_LABELS.len()]; + uncertain_no_recall[6] = 0.71; + uncertain_no_recall[7] = 0.70; + + let question = classifier + .classify_with_embedding( + "Could this be relevant?", + &uncertain_no_recall, + IntentTarget::Query, + ) + .unwrap(); + assert_eq!(question.primary_intent, "chitchat"); + assert!(question.recall_eligible); + assert!(!question.memory_eligible); + + let statement = classifier + .classify_with_embedding( + "This may be relevant.", + &uncertain_no_recall, + IntentTarget::Query, + ) + .unwrap(); + assert!(!statement.recall_eligible); + + let memory = classifier + .classify_with_embedding( + "Could this be relevant?", + &uncertain_no_recall, + IntentTarget::Memory, + ) + .unwrap(); + assert!(!memory.recall_eligible); + assert!(!memory.memory_eligible); + + let confident_question = classifier + .classify_with_embedding( + "How are you?", + &intent_axis(6), + IntentTarget::Query, + ) + .unwrap(); + assert!(!confident_question.recall_eligible); + } + + #[test] + fn continuation_shape_is_short_and_token_bounded() { + assert!(structural_signals("an unfinished thought...").continuation); + assert!(structural_signals("the answer depends on").continuation); + assert!(!structural_signals("aspirin").continuation); + + let long_fragment = format!("{}...", vec!["word"; SHORT_MESSAGE_MAX_TOKENS + 1].join(" ")); + assert!(!structural_signals(&long_fragment).continuation); + } + + #[test] + fn embedding_path_rejects_incompatible_dimensions() { + let classifier = test_classifier(); + let error = classifier + .classify_with_embedding("query", &[1.0], IntentTarget::Query) + .unwrap_err(); + assert!(error.contains("dimension mismatch")); + } + + fn hard_intent_classification(primary_intent: &str) -> IntentClassification { + let mut scores = BTreeMap::new(); + for label in INTENT_LABELS { + scores.insert(label.to_string(), if label == primary_intent { 1.0 } else { 0.0 }); + } + IntentClassification { + primary_intent: primary_intent.to_string(), + scores, + top_intents: vec![primary_intent.to_string()], + top_score: 1.0, + margin: 1.0, + confidence_weight: 1.0, + recall_eligible: true, + recall_action: "recall".to_string(), + memory_eligible: true, + model_version: "test-model".to_string(), + taxonomy_version: INTENT_TAXONOMY_VERSION.to_string(), + } + } + + #[test] + fn intent_compatibility_prioritizes_exact_matches() { + let query = hard_intent_classification("preference"); + let exact = hard_intent_classification("preference"); + let related = hard_intent_classification("personal_fact"); + let unrelated = hard_intent_classification("event_or_plan"); + + let exact_score = intent_compatibility(&query, &exact); + let related_score = intent_compatibility(&query, &related); + let unrelated_score = intent_compatibility(&query, &unrelated); + + assert!(exact_score > 0.99); + assert!(related_score > 0.30 && related_score < 0.50); + assert!(unrelated_score < 0.05); + assert!(exact_score > related_score); + assert!(related_score > unrelated_score); + } + + #[test] + fn intent_compatibility_uses_directional_memory_top_three() { + let query = hard_intent_classification("preference"); + let mut second_place = hard_intent_classification("event_or_plan"); + second_place.top_intents = vec![ + "event_or_plan".to_string(), + "preference".to_string(), + "decision".to_string(), + ]; + let mut third_place = hard_intent_classification("event_or_plan"); + third_place.top_intents = vec![ + "event_or_plan".to_string(), + "decision".to_string(), + "preference".to_string(), + ]; + let unrelated = hard_intent_classification("event_or_plan"); + + let second_score = intent_compatibility(&query, &second_place); + let third_score = intent_compatibility(&query, &third_place); + let unrelated_score = intent_compatibility(&query, &unrelated); + + assert!((second_score - 0.70).abs() < 0.001); + assert!((third_score - 0.45).abs() < 0.001); + assert!(second_score > third_score); + assert!(third_score > unrelated_score); + } +} diff --git a/src/jobs.rs b/src/jobs.rs index 5f786dd..716847d 100644 --- a/src/jobs.rs +++ b/src/jobs.rs @@ -1,4 +1,5 @@ use crate::config::*; +use crate::intent::IntentTarget; use crate::metrics::MetricsCollector; use crate::multi_tenant::MultiTenantEngine; use crate::projects::ProjectContext; @@ -64,6 +65,7 @@ pub enum Job { file_path: String, structural_cues: Vec<String>, metadata: Option<HashMap<String, serde_json::Value>>, + embedding: Option<Vec<f32>>, category: crate::agent::chunker::ChunkCategory, }, VerifyFile { @@ -88,6 +90,10 @@ pub enum Job { project_id: String, memory_ref: MemoryRef, }, + ClassifyMemory { + project_id: String, + memory_id: MemoryId, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -102,6 +108,12 @@ pub struct JobProgress { pub phase: String, pub writes_completed: usize, pub writes_total: usize, + #[serde(default)] + pub intent_completed: usize, + #[serde(default)] + pub intent_total: usize, + #[serde(default)] + pub intent_failed: usize, } /// Tracks a bulk ingestion session with buffered jobs @@ -110,6 +122,9 @@ pub struct IngestionSession { pub phase: std::sync::atomic::AtomicU8, // 0=Writing, 1=Processing, 2=Done pub writes_completed: std::sync::atomic::AtomicUsize, pub writes_total: std::sync::atomic::AtomicUsize, + pub intent_completed: std::sync::atomic::AtomicUsize, + pub intent_total: std::sync::atomic::AtomicUsize, + pub intent_failed: std::sync::atomic::AtomicUsize, last_write: tokio::sync::Mutex<std::time::Instant>, } @@ -120,6 +135,9 @@ impl IngestionSession { phase: std::sync::atomic::AtomicU8::new(0), writes_completed: std::sync::atomic::AtomicUsize::new(0), writes_total: std::sync::atomic::AtomicUsize::new(0), + intent_completed: std::sync::atomic::AtomicUsize::new(0), + intent_total: std::sync::atomic::AtomicUsize::new(0), + intent_failed: std::sync::atomic::AtomicUsize::new(0), last_write: tokio::sync::Mutex::new(std::time::Instant::now()), } } @@ -133,17 +151,34 @@ impl IngestionSession { } pub fn get_progress(&self) -> JobProgress { - let phase = match self.get_phase() { - IngestionPhase::Writing => "writing", - IngestionPhase::Processing => "processing", - IngestionPhase::Done => "done", + let writes_completed = self + .writes_completed + .load(std::sync::atomic::Ordering::Relaxed); + let writes_total = self.writes_total.load(std::sync::atomic::Ordering::Relaxed); + let intent_completed = self + .intent_completed + .load(std::sync::atomic::Ordering::Relaxed); + let intent_total = self.intent_total.load(std::sync::atomic::Ordering::Relaxed); + let intent_failed = self + .intent_failed + .load(std::sync::atomic::Ordering::Relaxed); + let intent_finished = intent_completed.saturating_add(intent_failed); + let phase = if writes_total == 0 && intent_total == 0 { + "idle" + } else if writes_completed < writes_total { + "writing" + } else if intent_finished < intent_total { + "processing" + } else { + "done" }; JobProgress { phase: phase.to_string(), - writes_completed: self - .writes_completed - .load(std::sync::atomic::Ordering::Relaxed), - writes_total: self.writes_total.load(std::sync::atomic::Ordering::Relaxed), + writes_completed, + writes_total, + intent_completed, + intent_total, + intent_failed, } } @@ -168,6 +203,21 @@ impl IngestionSession { .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } + pub fn expect_intent(&self) { + self.intent_total + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + pub fn intent_complete(&self) { + self.intent_completed + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + pub fn intent_failed(&self) { + self.intent_failed + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + /// Check if we should auto-flush (no writes for 2 seconds) pub async fn should_auto_flush(&self) -> bool { let last = *self.last_write.lock().await; @@ -244,6 +294,9 @@ impl SessionManager { phase: "idle".to_string(), // Default writes_completed: 0, writes_total: 0, + intent_completed: 0, + intent_total: 0, + intent_failed: 0, }; let mut active_count = 0; @@ -253,6 +306,9 @@ impl SessionManager { global.writes_completed += p.writes_completed; global.writes_total += p.writes_total; + global.intent_completed += p.intent_completed; + global.intent_total += p.intent_total; + global.intent_failed += p.intent_failed; if p.phase != "idle" && p.phase != "done" { active_count += 1; @@ -392,6 +448,7 @@ impl JobQueue { let metrics_clone = metrics.clone(); let background_processing = jobs_config.background_processing; let jobs_config_for_worker = jobs_config.clone(); + let tx_worker = tx.clone(); tokio::spawn(async move { while let Some(job) = rx.recv().await { @@ -407,6 +464,8 @@ impl JobQueue { &provider_clone, &metrics_clone, &jobs_config_for_worker, + &session_manager_clone, + &tx_worker, ) .await; } @@ -470,7 +529,23 @@ impl JobQueue { /// Enqueue a job immediately (for non-buffered jobs like Reinforce) pub async fn enqueue(&self, job: Job) { + if !self.background_processing && matches!(job, Job::ClassifyMemory { .. }) { + return; + } + let classify_project_id = if let Job::ClassifyMemory { project_id, .. } = &job { + self.session_manager + .get_or_create(project_id) + .expect_intent(); + Some(project_id.clone()) + } else { + None + }; if let Err(e) = self.sender.send(job).await { + if let Some(project_id) = classify_project_id { + self.session_manager + .get_or_create(&project_id) + .intent_failed(); + } warn!("Failed to enqueue job: {}", e); } } @@ -496,10 +571,15 @@ impl JobQueue { let session = entry.value(); let progress = session.get_progress(); // Count jobs that haven't completed yet - let pending = progress + let pending_writes = progress .writes_total .saturating_sub(progress.writes_completed); - count += pending; + let pending_intents = progress.intent_total.saturating_sub( + progress + .intent_completed + .saturating_add(progress.intent_failed), + ); + count += pending_writes.saturating_add(pending_intents); } count } @@ -587,11 +667,300 @@ fn choose_canonical(a: &str, b: &str) -> (String, String) { } } +#[cfg(test)] +mod tests { + use super::*; + use crate::multi_tenant::MultiTenantEngine; + + #[test] + fn memory_refs_and_canonical_helpers_are_deterministic() { + let id = MemoryRef::Id(42); + let source = MemoryRef::SourceKey("doc#1".to_string()); + assert_eq!(id.to_string(), "42"); + assert_eq!(source.to_string(), "source_key:doc#1"); + assert_eq!( + cue_tokens("User:Name-test").into_iter().collect::<Vec<_>>(), + vec!["user".to_string(), "name".to_string(), "test".to_string()] + ); + assert!(lexical_gate("user:name", "name-alias")); + assert!(lexical_gate("prefix", "prefix-long")); + assert!(!lexical_gate("aa", "bb")); + assert!(is_canonical_format("kind:value")); + assert!(!is_canonical_format("kind:")); + assert_eq!(choose_canonical("alias", "kind:value"), ("kind:value".into(), "alias".into())); + assert_eq!(choose_canonical("z", "a"), ("a".into(), "z".into())); + } + + #[tokio::test] + async fn ingestion_session_tracks_progress_and_flush_state() { + let session = IngestionSession::new("project".to_string()); + assert_eq!(session.get_phase(), IngestionPhase::Writing); + assert_eq!(session.get_progress().phase, "idle"); + assert!(!session.should_auto_flush().await); + + session.expect_write(); + session.expect_intent(); + session.intent_complete(); + session.intent_failed(); + session.write_complete(); + let progress = session.get_progress(); + assert_eq!(progress.phase, "done"); + assert_eq!(progress.writes_completed, 1); + assert_eq!(progress.writes_total, 1); + assert_eq!(progress.intent_completed, 1); + assert_eq!(progress.intent_failed, 1); + + session + .buffer_job(Job::ProposeAliases { + project_id: "project".to_string(), + }) + .await; + assert!(!session.is_stale()); + } + + #[tokio::test] + async fn session_manager_aggregates_projects_and_flushes() { + let dir = tempfile::tempdir().unwrap(); + let provider: Arc<dyn ProjectProvider> = Arc::new(MultiTenantEngine::with_snapshots_dir( + dir.path(), + TuningConfig::default(), + )); + let manager = SessionManager::new(provider, None, JobsConfig::default()); + assert!(manager.get("missing").is_none()); + let first = manager.get_or_create("one"); + assert!(Arc::ptr_eq(&first, &manager.get_or_create("one"))); + first.expect_write(); + let second = manager.get_or_create("two"); + second.expect_write(); + second.write_complete(); + let progress = manager.get_global_progress(); + assert_eq!(progress.writes_total, 2); + assert_eq!(progress.writes_completed, 1); + assert_eq!(progress.phase, "processing (1 projects)"); + + manager.flush_session("one").await; + assert_eq!(first.get_phase(), IngestionPhase::Done); + } + + #[tokio::test] + async fn process_job_handles_ingestion_classification_and_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let mut config = ServerConfig::default(); + config.semantic.encoder_enabled = false; + let engine = Arc::new(MultiTenantEngine::with_config( + config, + dir.path().to_path_buf(), + )); + let provider: Arc<dyn ProjectProvider> = engine.clone(); + let metrics = Arc::new(MetricsCollector::new()); + let metrics_opt = Some(metrics.clone()); + let jobs_config = JobsConfig::default(); + let sessions = Arc::new(SessionManager::new( + provider.clone(), + metrics_opt.clone(), + jobs_config.clone(), + )); + let (sender, _receiver) = mpsc::channel(8); + + let context = engine.get_or_create_project("jobs-project".to_string()).unwrap(); + let memory_id = context.main.add_memory_with_source_key( + "fn main() {}".to_string(), + vec!["path:src/main.rs".to_string(), "source:agent".to_string()], + None, + MainStats::default(), + false, + Some("src/main.rs:1-1".to_string()), + ); + process_job( + Job::ClassifyMemory { + project_id: "jobs-project".to_string(), + memory_id, + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + + let session = sessions.get("jobs-project").unwrap(); + let progress = session.get_progress(); + // The test engine deliberately disables the optional encoder, so the + // worker records the unavailable-classifier failure deterministically. + assert_eq!(progress.intent_failed, 1); + process_job( + Job::VerifyFile { + project_id: "jobs-project".to_string(), + file_path: "src/main.rs".to_string(), + valid_source_keys: Vec::new(), + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + assert!(context.main.get_memory(memory_id).is_none()); + + let replacement = context.main.add_memory_with_source_key( + "delete me".to_string(), + vec!["cleanup".to_string()], + None, + MainStats::default(), + false, + Some("cleanup-key".to_string()), + ); + process_job( + Job::DeleteMemory { + project_id: "jobs-project".to_string(), + memory_ref: MemoryRef::SourceKey("cleanup-key".to_string()), + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + assert!(context.main.get_memory(replacement).is_none()); + + process_job( + Job::DeleteMemory { + project_id: "jobs-project".to_string(), + memory_ref: MemoryRef::SourceKey("missing-key".to_string()), + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + } + + #[tokio::test] + async fn process_job_handles_alias_reinforcement_lexicon_and_heatmap() { + let dir = tempfile::tempdir().unwrap(); + let mut config = ServerConfig::default(); + config.semantic.encoder_enabled = false; + let engine = Arc::new(MultiTenantEngine::with_config( + config, + dir.path().to_path_buf(), + )); + let provider: Arc<dyn ProjectProvider> = engine.clone(); + let metrics = Arc::new(MetricsCollector::new()); + let metrics_opt = Some(metrics); + let jobs_config = JobsConfig::default(); + let sessions = Arc::new(SessionManager::new( + provider.clone(), + metrics_opt.clone(), + jobs_config.clone(), + )); + let (sender, _receiver) = mpsc::channel(8); + let context = engine.get_or_create_project("jobs-analysis".to_string()).unwrap(); + + let mut memory_ids = Vec::new(); + for index in 0..3 { + memory_ids.push(context.main.add_memory( + format!("feature report {index}"), + vec!["feature".to_string(), "feature_flag".to_string()], + None, + MainStats::default(), + false, + )); + } + + process_job( + Job::ProposeAliases { + project_id: "jobs-analysis".to_string(), + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + assert!(context.aliases.total_memories() >= 1); + + process_job( + Job::ReinforceMemories { + project_id: "jobs-analysis".to_string(), + memory_ids: memory_ids.clone(), + cues: vec!["feature".to_string()], + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + + let lexicon_id = context.lexicon.upsert_memory_with_source_key( + "lexicon-key".to_string(), + "feature alias".to_string(), + vec!["feature".to_string()], + None, + Some(crate::structures::LexiconStats::default()), + false, + true, + ); + process_job( + Job::ReinforceLexicon { + project_id: "jobs-analysis".to_string(), + memory_ids: vec![lexicon_id], + cues: vec!["feature".to_string()], + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + + process_job( + Job::UpdateMarketHeatmap { + project_id: "jobs-analysis".to_string(), + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + + let missing_session_before = sessions.get("missing-project"); + assert!(missing_session_before.is_none()); + process_job( + Job::ClassifyMemory { + project_id: "missing-project".to_string(), + memory_id: 42, + }, + &provider, + &metrics_opt, + &jobs_config, + &sessions, + &sender, + ) + .await; + // MultiTenantEngine creates projects on demand, so this path records a + // failed classification for the newly created empty project. + assert_eq!(sessions.get("missing-project").unwrap().get_progress().intent_failed, 1); + } +} + async fn process_job( job: Job, provider: &Arc<dyn ProjectProvider>, metrics: &Option<Arc<MetricsCollector>>, _jobs_config: &JobsConfig, + session_manager: &Arc<SessionManager>, + sender: &mpsc::Sender<Job>, ) { match job { Job::ProposeAliases { project_id } => { @@ -771,6 +1140,7 @@ async fn process_job( file_path, structural_cues, metadata, + embedding, category, } => { if let Some(ctx) = provider.get_project(&project_id) { @@ -780,8 +1150,9 @@ async fn process_job( let file_path_clone = file_path.clone(); let structural_cues_clone = structural_cues.clone(); let metadata_clone = metadata.clone(); + let embedding_clone = embedding.clone(); - tokio::task::spawn_blocking(move || { + let memory_id = tokio::task::spawn_blocking(move || { debug!( "Agent: Fast extraction starting for {} (category: {:?})", source_key_clone, category @@ -809,7 +1180,9 @@ async fn process_job( resolved_cues.push(format!("category:{:?}", category).to_lowercase()); // 3. Upsert memory (Lean cues only) - let memory_id = ctx_clone.main.upsert_memory_with_source_key( + let memory_id = ctx_clone + .main + .upsert_memory_with_source_key_and_options_and_vector( source_key_clone.clone(), content_clone, resolved_cues.clone(), @@ -817,6 +1190,9 @@ async fn process_job( Some(MainStats::default()), false, true, + true, + None, + embedding_clone, ); debug!( @@ -826,10 +1202,49 @@ async fn process_job( category, resolved_cues.len() ); + memory_id }) .await .unwrap(); + if memory_id != crate::structures::INVALID_MEMORY_ID { + session_manager + .get_or_create(&project_id) + .expect_intent(); + let classification_job = Job::ClassifyMemory { + project_id: project_id.clone(), + memory_id, + }; + match sender.try_send(classification_job) { + Ok(()) => {} + Err(tokio::sync::mpsc::error::TrySendError::Full(job)) => { + // This runs inside the queue's only consumer. Awaiting a send + // to the same full bounded queue here would deadlock ingestion. + let follow_up_sender = sender.clone(); + let follow_up_sessions = Arc::clone(session_manager); + let follow_up_project_id = project_id.clone(); + tokio::spawn(async move { + if let Err(error) = follow_up_sender.send(job).await { + follow_up_sessions + .get_or_create(&follow_up_project_id) + .intent_failed(); + warn!( + memory_id, + error = %error, + "Failed to enqueue deferred memory intent classification" + ); + } + }); + } + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + session_manager + .get_or_create(&project_id) + .intent_failed(); + warn!(memory_id, "Failed to enqueue memory intent classification: queue closed"); + } + } + } + // Record ingestion metric if let Some(m) = metrics { m.record_ingestion(); @@ -849,9 +1264,16 @@ async fn process_job( // 3. Delete them let path_cue = format!("path:{}", file_path); - if let Some(ordered_set) = ctx.main.get_cue_index().get(&path_cue) { + // Copy the IDs out before mutating the engine. Holding a + // DashMap guard while deleting memories can deadlock when the + // deletion touches the same shard. + let current_memories = ctx + .main + .get_cue_index() + .get(&path_cue) + .map(|ordered_set| ordered_set.get_recent_owned(None)); + if let Some(current_memories) = current_memories { // Get all memory IDs associated with this file - let current_memories = ordered_set.get_recent_owned(None); let valid_set: HashSet<String> = valid_source_keys.into_iter().collect(); let mut deleted_count = 0; @@ -880,6 +1302,51 @@ async fn process_job( } } } + Job::ClassifyMemory { + project_id, + memory_id, + } => { + let result = if let Some(ctx) = provider.get_project(&project_id) { + let ctx_clone = ctx.clone(); + tokio::task::spawn_blocking(move || { + let memory = ctx_clone + .main + .get_memory(memory_id) + .ok_or_else(|| "memory no longer exists".to_string())?; + let content = ctx_clone.main.read_memory_content(&memory)?; + let classification = if let Some(vector) = memory.semantic_vector.as_ref() { + let embedding = vector.normalized_values(); + ctx_clone.main.classify_intent_with_embedding( + &content, + IntentTarget::Memory, + &embedding, + )? + } else { + ctx_clone.main.classify_intent(&content, IntentTarget::Memory)? + }; + if !ctx_clone + .main + .attach_intent_classification(memory_id, classification) + { + return Err("memory disappeared while attaching intent".to_string()); + } + Ok::<(), String>(()) + }) + .await + .unwrap_or_else(|error| Err(format!("intent worker failed: {error}"))) + } else { + Err("project no longer exists".to_string()) + }; + + let session = session_manager.get_or_create(&project_id); + match result { + Ok(()) => session.intent_complete(), + Err(error) => { + session.intent_failed(); + warn!(project_id, memory_id, error = %error, "Memory intent classification failed"); + } + } + } Job::DeleteMemory { project_id, memory_ref, diff --git a/src/lib.rs b/src/lib.rs index a59c808..243ea6b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,10 +4,10 @@ pub mod auth; pub mod config; pub mod crypto; pub mod cuebridge; -pub mod cuepacks; pub mod engine; pub mod facets; pub mod grounding; +pub mod intent; pub mod jobs; pub mod metrics; pub mod multi_tenant; @@ -17,3 +17,6 @@ pub mod persistence; pub mod projects; pub mod structures; pub mod taxonomy; +pub mod semantic; +#[cfg(feature = "semantic-encoder")] +pub mod semantic_encoder; diff --git a/src/main.rs b/src/main.rs index be229bd..5388e81 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,1173 @@ struct Cli { command: Commands, } +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + async fn mock_http_server(responses: Vec<(u16, String)>) -> String { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + for (status, body) in responses { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let mut request = [0_u8; 8192]; + let _ = stream.read(&mut request).await; + let reason = match status { + 200 => "OK", + 404 => "Not Found", + 500 => "Internal Server Error", + _ => "Response", + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()).await; + } + }); + format!("http://{address}") + } + + fn add_args(url: String) -> AddArgs { + AddArgs { + content: "cli memory".to_string(), + project: Some("cli-project".to_string()), + metadata: Some(r#"{"source":"cli-test"}"#.to_string()), + cues: vec!["cli".to_string()], + disable_temporal_chunking: false, + async_ingest: false, + url, + } + } + + fn recall_args(url: String) -> RecallArgs { + RecallArgs { + query: "cli query".to_string(), + project: Some("cli-project".to_string()), + limit: 5, + cues: vec!["cli".to_string()], + semantic_mode: "lexical".to_string(), + depth: 1, + token_budget: 128, + port: 8080, + no_auto_reinforce: false, + min_intersection: None, + query_time: None, + disable_salience_bias: false, + parent_fusion: "off".to_string(), + parent_fusion_limit: 80, + parent_fusion_min_chunks: 2, + ordered_reconstruction: "off".to_string(), + ordered_reconstruction_limit: 80, + ordered_session_scan_limit: 4096, + ordered_max_sessions: 3, + evidence_coverage: "off".to_string(), + evidence_coverage_limit: 100, + evidence_coverage_session_scan_limit: 4096, + evidence_coverage_max_sessions: 3, + expansion_depth: 1, + enable_alias_expansion: false, + disable_cuebridge_artifacts: false, + cuebridge_gap_limit: 6, + grounded: false, + explain: true, + trace_timing: true, + url, + web: false, + target_url: None, + persist: false, + } + } + + #[test] + fn cli_parses_start_overrides_and_nested_project_commands() { + let cli = Cli::try_parse_from([ + "cuemap", "start", "--port", "9090", "--data-dir", "/tmp/cuemap-data", + "--profile", "benchmark", "--disable-bg-jobs", "--disable-snapshots", "--disk-content", + ]).unwrap(); + match cli.command { + Commands::Start(args) => { + assert_eq!(args.port, Some(9090)); + assert_eq!(args.data_dir.as_deref(), Some("/tmp/cuemap-data")); + assert_eq!(args.profile.as_deref(), Some("benchmark")); + assert!(args.disable_bg_jobs); + assert!(args.disable_snapshots); + assert!(args.disk_content); + } + _ => panic!("expected start command"), + } + + let cli = Cli::try_parse_from([ + "cuemap", "projects", "set-watch-dir", "project", "/tmp/repo", "--url", "http://example.test", + ]).unwrap(); + assert!(matches!(cli.command, Commands::Projects(ProjectArgs { cmd: ProjectCmd::SetWatchDir { .. } }))); + } + + #[test] + fn cli_rejects_invalid_ports_and_unknown_commands() { + assert!(Cli::try_parse_from(["cuemap", "start", "--port", "70000"]).is_err()); + assert!(Cli::try_parse_from(["cuemap", "unknown"]).is_err()); + } + + #[test] + fn startup_overrides_apply_all_cli_options() { + let cli = Cli::try_parse_from([ + "cuemap", + "start", + "--port", + "9091", + "--data-dir", + "/tmp/cuemap-data", + "--assets-dir", + "/tmp/cuemap-assets", + "--snapshot-interval", + "17", + "--agent-dir", + "/tmp/cuemap-watch", + "--agent-throttle", + "250", + "--disable-bg-jobs", + "--disable-snapshots", + "--disk-content", + "--cloud-backup", + "local", + "--cloud-bucket", + "/tmp/cuemap-cloud", + "--cloud-region", + "eu-west-1", + "--cloud-endpoint", + "http://localhost:9000", + "--cloud-prefix", + "release/", + "--cloud-auto-backup", + ]) + .unwrap(); + + let args = match cli.command { + Commands::Start(args) => args, + _ => panic!("expected start command"), + }; + let config = apply_start_overrides(config::ServerConfig::default(), &args); + + assert_eq!(config.server.port, 9091); + assert_eq!(config.server.data_dir, "/tmp/cuemap-data"); + assert_eq!(config.server.assets_dir.as_deref(), Some("/tmp/cuemap-assets")); + assert_eq!(config.persistence.snapshot_interval_seconds, 17); + assert_eq!(config.agent.watch_dir.as_deref(), Some("/tmp/cuemap-watch")); + assert!(config.agent.enabled); + assert_eq!(config.agent.throttle_ms, 250); + assert!(!config.jobs.background_processing); + assert!(!config.persistence.enabled); + assert!(config.server.store_content_on_disk); + assert_eq!(config.persistence.cloud.provider, "local"); + assert_eq!(config.persistence.cloud.bucket, "/tmp/cuemap-cloud"); + assert_eq!(config.persistence.cloud.region, "eu-west-1"); + assert_eq!( + config.persistence.cloud.endpoint.as_deref(), + Some("http://localhost:9000") + ); + assert_eq!(config.persistence.cloud.prefix, "release/"); + assert!(config.persistence.cloud.auto_backup); + } + + #[test] + fn startup_config_loading_applies_profile_and_cli_overrides() { + let root = tempfile::tempdir().unwrap(); + let config_path = root.path().join("missing.toml"); + let cli = Cli::try_parse_from([ + "cuemap", + "start", + "--config", + config_path.to_str().unwrap(), + "--profile", + "benchmark", + "--port", + "9191", + ]) + .unwrap(); + let args = match cli.command { + Commands::Start(args) => args, + _ => panic!("expected start command"), + }; + + let config = load_start_config(&args).unwrap(); + assert_eq!(config.server.port, 9191); + assert!(!config.persistence.enabled); + assert!(!config.jobs.background_processing); + } + + #[test] + fn snapshot_selection_handles_static_auxiliary_main_and_legacy_layouts() { + let root = tempfile::tempdir().unwrap(); + let data_dir = root.path().join("data"); + let configured = data_dir.join("snapshots"); + let legacy = root.path().join("snapshots"); + std::fs::create_dir_all(&configured).unwrap(); + + assert_eq!( + select_snapshots_dir(data_dir.to_str().unwrap(), Some("/tmp/static")), + "/tmp/static" + ); + assert!(!has_main_snapshot(&configured)); + + std::fs::write(configured.join("project_aliases.bin"), b"aliases").unwrap(); + std::fs::write(configured.join("project_lexicon.bin"), b"lexicon").unwrap(); + assert!(!has_main_snapshot(&configured)); + assert_eq!( + select_snapshots_dir(data_dir.to_str().unwrap(), None), + configured.to_string_lossy() + ); + + std::fs::create_dir_all(&legacy).unwrap(); + std::fs::write(legacy.join("project.bin"), b"snapshot").unwrap(); + assert!(has_main_snapshot(&legacy)); + assert_eq!( + select_snapshots_dir(data_dir.to_str().unwrap(), None), + data_dir.join("..").join("snapshots").to_string_lossy() + ); + + std::fs::write(configured.join("project.bin"), b"new snapshot").unwrap(); + assert_eq!( + select_snapshots_dir(data_dir.to_str().unwrap(), None), + configured.to_string_lossy() + ); + assert!(!has_main_snapshot(&root.path().join("missing"))); + } + + #[test] + fn master_key_resolution_obeys_precedence_and_validation() { + let mut security = config::SecurityConfig::default(); + let config_key = "22".repeat(32); + security.master_key = Some(config_key); + + let env_key = "11".repeat(32); + let resolved = resolve_master_key(&security, Some(&env_key), None, None).unwrap(); + assert_eq!(resolved.as_bytes(), &[0x11; 32]); + + // An explicitly supplied but malformed environment key must not silently + // fall back to a lower-precedence config value. + assert!(resolve_master_key(&security, Some("not-hex"), None, None).is_none()); + + let password_key = resolve_master_key( + &config::SecurityConfig::default(), + None, + Some("correct horse battery staple"), + Some(b"test-salt"), + ) + .unwrap(); + assert_eq!(password_key.as_bytes().len(), 32); + + let config_key = resolve_master_key(&security, None, None, None).unwrap(); + assert_eq!(config_key.as_bytes(), &[0x22; 32]); + + security.master_key = Some("short".to_string()); + assert!(resolve_master_key(&security, None, None, None).is_none()); + assert!(resolve_master_key(&config::SecurityConfig::default(), None, None, None).is_none()); + } + + #[test] + fn context_signer_resolution_handles_ed25519_legacy_and_invalid_keys() { + let mut security = config::SecurityConfig::default(); + assert!(resolve_context_signer(&security).is_none()); + + security.secret_key = Some("legacy-secret".to_string()); + assert!(resolve_context_signer(&security).is_some()); + + security.signing_private_key = Some("00".repeat(32)); + assert!(resolve_context_signer(&security).is_some()); + + security.signing_private_key = Some("not-hex".to_string()); + assert!(resolve_context_signer(&security).is_none()); + } + + #[test] + fn kdf_salt_environment_override_is_used_for_startup() { + let previous = std::env::var("CUEMAP_KDF_SALT").ok(); + std::env::set_var("CUEMAP_KDF_SALT", "release-test-salt"); + assert_eq!(get_or_create_salt(), b"release-test-salt"); + + if let Some(value) = previous { + std::env::set_var("CUEMAP_KDF_SALT", value); + } else { + std::env::remove_var("CUEMAP_KDF_SALT"); + } + } + + #[test] + fn local_kdf_salt_loading_handles_existing_short_and_unwritable_files() { + let root = tempfile::tempdir().unwrap(); + let existing = vec![7_u8; 32]; + std::fs::write(root.path().join("salt"), &existing).unwrap(); + assert_eq!(load_or_create_salt(root.path(), None), existing); + + std::fs::write(root.path().join("salt"), b"short").unwrap(); + let regenerated = load_or_create_salt(root.path(), None); + assert_eq!(regenerated.len(), 32); + assert_eq!(std::fs::read(root.path().join("salt")).unwrap(), regenerated); + + let missing_root = tempfile::tempdir().unwrap(); + let generated = load_or_create_salt(missing_root.path(), None); + assert_eq!(generated.len(), 32); + + let file_root = root.path().join("not-a-directory"); + std::fs::write(&file_root, b"file").unwrap(); + assert_eq!(load_or_create_salt(&file_root, None).len(), 32); + assert_eq!(load_or_create_salt(root.path(), Some("override")), b"override"); + } + + #[tokio::test] + async fn detached_readiness_waiter_handles_offsets_success_timeout_and_missing_logs() { + let root = tempfile::tempdir().unwrap(); + let log_path = root.path().join("server.log"); + std::fs::write(&log_path, "old line\nready: Unstable sorting for speed\n").unwrap(); + let start_pos = "old line\n".len() as u64; + assert!(wait_for_readiness( + &log_path, + start_pos, + "Unstable sorting for speed", + Duration::from_millis(50), + ) + .await + .unwrap()); + + let empty_path = root.path().join("empty.log"); + std::fs::write(&empty_path, "").unwrap(); + assert!(!wait_for_readiness( + &empty_path, + 0, + "never appears", + Duration::from_millis(1), + ) + .await + .unwrap()); + assert!(wait_for_readiness( + &root.path().join("missing.log"), + 0, + "ready", + Duration::from_millis(1), + ) + .await + .is_err()); + + let spawned_log = root.path().join("spawned.log"); + let shell_args = vec![ + "-c".to_string(), + "printf 'Unstable sorting for speed\\n'".to_string(), + ]; + assert!(spawn_detached_process( + Path::new("/bin/sh"), + &shell_args, + &spawned_log, + "Unstable sorting for speed", + Duration::from_secs(1), + ) + .await + .unwrap()); + + let timeout_args = vec!["-c".to_string(), "true".to_string()]; + assert!(!spawn_detached_process( + Path::new("/bin/sh"), + &timeout_args, + &root.path().join("spawn-timeout.log"), + "never appears", + Duration::from_millis(1), + ) + .await + .unwrap()); + assert!(spawn_detached_process( + Path::new("/definitely/missing/cuemap-child"), + &[], + &root.path().join("spawn-error.log"), + "ready", + Duration::from_millis(1), + ) + .await + .is_err()); + } + + #[tokio::test] + async fn static_server_startup_builds_and_binds_the_cli_router() { + let root = tempfile::tempdir().unwrap(); + let snapshots = root.path().join("snapshots"); + std::fs::create_dir_all(&snapshots).unwrap(); + let pid_path = root.path().join("server.pid"); + let port = std::net::TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + + let mut server_config = config::ServerConfig::default(); + server_config.server.port = port; + server_config.server.data_dir = root.path().join("data").to_string_lossy().to_string(); + server_config.persistence.enabled = false; + server_config.jobs.background_processing = false; + server_config.semantic.enabled = false; + server_config.semantic.encoder_enabled = false; + server_config.semantic.profile = cuemap::semantic::SemanticProfile::Off; + + let task = tokio::spawn(run_server_with_pid_path( + server_config, + Some(snapshots.to_string_lossy().to_string()), + true, + pid_path.clone(), + )); + + let ready = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if tokio::net::TcpStream::connect(("127.0.0.1", port)).await.is_ok() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .is_ok(); + assert!(ready, "static server did not bind its configured port"); + assert_eq!(std::fs::read_to_string(&pid_path).unwrap(), std::process::id().to_string()); + + task.abort(); + let _ = task.await; + + let live_port = std::net::TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + let live_pid_path = root.path().join("live-server.pid"); + let mut live_config = config::ServerConfig::default(); + live_config.server.port = live_port; + live_config.server.data_dir = root.path().join("live-data").to_string_lossy().to_string(); + live_config.persistence.enabled = false; + live_config.jobs.background_processing = false; + live_config.semantic.enabled = false; + live_config.semantic.encoder_enabled = false; + live_config.semantic.profile = cuemap::semantic::SemanticProfile::Off; + live_config.persistence.cloud.provider = "s3".to_string(); + + let live_task = tokio::spawn(run_server_with_pid_path( + live_config, + None, + true, + live_pid_path, + )); + let live_ready = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if tokio::net::TcpStream::connect(("127.0.0.1", live_port)).await.is_ok() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .is_ok(); + assert!(live_ready, "live server did not bind its configured port"); + live_task.abort(); + let _ = live_task.await; + } + + #[tokio::test] + async fn stop_handler_covers_missing_success_and_failed_pid_paths() { + let root = tempfile::tempdir().unwrap(); + let missing_path = root.path().join("missing.pid"); + handle_stop_at(missing_path).await; + + let mut child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(); + let pid_path = root.path().join("running.pid"); + std::fs::write(&pid_path, child.id().to_string()).unwrap(); + handle_stop_at(pid_path.clone()).await; + assert!(!pid_path.exists()); + let _ = child.wait(); + + let failed_path = root.path().join("failed.pid"); + // Keep the synthetic PID positive when converted to Unix pid_t. + // u32::MAX becomes -1 on Linux, where kill(-1, SIGTERM) signals every + // process the current user is permitted to terminate. + std::fs::write(&failed_path, (i32::MAX as u32).to_string()).unwrap(); + handle_stop_at(failed_path.clone()).await; + assert!(failed_path.exists()); + + #[cfg(unix)] + { + let unsafe_path = root.path().join("unsafe.pid"); + std::fs::write(&unsafe_path, u32::MAX.to_string()).unwrap(); + handle_stop_at(unsafe_path.clone()).await; + assert!(unsafe_path.exists()); + } + } + + #[tokio::test] + async fn cli_http_handlers_cover_success_and_failure_paths() { + let add_url = mock_http_server(vec![(200, r#"{"id":42}"#.to_string())]).await; + handle_add(add_args(add_url)).await; + let add_error_url = mock_http_server(vec![(500, r#"{"error":"rejected"}"#.to_string())]).await; + handle_add(add_args(add_error_url)).await; + + let ingest_url = mock_http_server(vec![(200, r#"{"status":"ingested"}"#.to_string())]).await; + handle_ingest(IngestArgs { + type_: IngestType::Content { + content: "content from cli".to_string(), + project: Some("cli-project".to_string()), + filename: "note.md".to_string(), + source_key: Some("cli-note".to_string()), + metadata: Some(r#"{"source":"test"}"#.to_string()), + structural_cues: vec!["cli".to_string()], + segmenter: "sentence_window".to_string(), + segment_window_size: Some(2), + segment_overlap: Some(1), + segment_min_chunk_chars: Some(16), + segment_max_chunk_chars: Some(128), + url: ingest_url, + }, + }) + .await; + + let file_dir = tempfile::tempdir().unwrap(); + let file_path = file_dir.path().join("cli-note.md"); + std::fs::write(&file_path, "file content from cli").unwrap(); + let file_url = mock_http_server(vec![(200, r#"{"status":"ingested"}"#.to_string())]).await; + handle_ingest(IngestArgs { + type_: IngestType::File { + path: file_path.to_string_lossy().to_string(), + project: Some("cli-project".to_string()), + url: file_url, + }, + }) + .await; + + let file_error_url = mock_http_server(vec![(500, r#"{"error":"file rejected"}"#.to_string())]).await; + handle_ingest(IngestArgs { + type_: IngestType::File { + path: file_path.to_string_lossy().to_string(), + project: Some("cli-project".to_string()), + url: file_error_url, + }, + }) + .await; + handle_ingest(IngestArgs { + type_: IngestType::File { + path: file_dir.path().join("missing.md").to_string_lossy().to_string(), + project: Some("cli-project".to_string()), + url: "http://127.0.0.1:1".to_string(), + }, + }) + .await; + + let url_ingest_server = + mock_http_server(vec![(200, r#"{"status":"started"}"#.to_string())]).await; + handle_ingest(IngestArgs { + type_: IngestType::Url { + url: "https://example.test/docs".to_string(), + project: Some("cli-project".to_string()), + depth: 2, + same_domain_only: true, + server_url: url_ingest_server, + }, + }) + .await; + + let url_ingest_error_server = + mock_http_server(vec![(500, r#"{"error":"crawl rejected"}"#.to_string())]).await; + handle_ingest(IngestArgs { + type_: IngestType::Url { + url: "https://example.test/docs".to_string(), + project: Some("cli-project".to_string()), + depth: 0, + same_domain_only: false, + server_url: url_ingest_error_server, + }, + }) + .await; + + let recall_url = mock_http_server(vec![ + ( + 200, + r#"{"results":[{"score":1.25,"memory_id":42,"content":"cli result"}],"timing":{"total_ms":1.0}}"#.to_string(), + ), + ]) + .await; + handle_recall(recall_args(recall_url)).await; + + let grounded_body = serde_json::json!({ + "verified_context": "grounded context", + "proof": { + "trace_id": "trace", + "query_text": "cli query", + "normalized_query": [], + "expanded_cues": [], + "token_budget": 128, + "selected": [], + "excluded_top": [] + }, + "engine_latency_ms": 1.0, + "signature_alg": "none", + "signature": "", + "public_key": null + }) + .to_string(); + let grounded_url = mock_http_server(vec![(200, grounded_body)]).await; + let mut grounded = recall_args(grounded_url); + grounded.grounded = true; + handle_recall(grounded).await; + + let web_url = mock_http_server(vec![ + ( + 200, + r#"{"results":[{"score":0.8,"intersection":2,"content":"web result"}],"urls":["https://example.test"]}"#.to_string(), + ), + ]) + .await; + let mut web = recall_args(web_url); + web.web = true; + web.target_url = Some("https://example.test/page".to_string()); + web.persist = true; + handle_recall(web).await; + + let recall_error_url = mock_http_server(vec![(500, r#"{"error":"down"}"#.to_string())]).await; + handle_recall(recall_args(recall_error_url)).await; + } + + #[tokio::test] + async fn cli_status_project_alias_and_memory_handlers_use_http_api() { + let status_url = mock_http_server(vec![ + (200, r#"{"total_memories":2}"#.to_string()), + (200, "cuemap_requests_total 2\n".to_string()), + ]) + .await; + handle_status(StatusArgs { + server: false, + jobs: false, + project: Some("cli-project".to_string()), + json: true, + url: status_url, + }) + .await; + + let jobs_url = mock_http_server(vec![(200, r#"{"phase":"done"}"#.to_string())]).await; + handle_status(StatusArgs { + server: false, + jobs: true, + project: Some("cli-project".to_string()), + json: false, + url: jobs_url, + }) + .await; + + let jobs_json_url = mock_http_server(vec![(200, r#"{"phase":"queued"}"#.to_string())]).await; + handle_status(StatusArgs { + server: false, + jobs: true, + project: Some("cli-project".to_string()), + json: true, + url: jobs_json_url, + }) + .await; + + let list_url = mock_http_server(vec![ + (200, r#"[{"project_id":"cli-project","total_memories":2}]"#.to_string()), + ]) + .await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::List { url: list_url }, + }) + .await; + + let create_url = mock_http_server(vec![(200, r#"{"project_id":"new-project"}"#.to_string())]).await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::Create { + name: "new-project".to_string(), + url: create_url, + }, + }) + .await; + + let watch_url = mock_http_server(vec![(200, r#"{"status":"updated"}"#.to_string())]).await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::SetWatchDir { + project: "cli-project".to_string(), + path: "/tmp".to_string(), + url: watch_url, + }, + }) + .await; + + let alias_add_url = mock_http_server(vec![(200, r#"{"id":1}"#.to_string())]).await; + handle_alias(AliasArgs { + text: "rust".to_string(), + project: Some("cli-project".to_string()), + add: Some("rust-language".to_string()), + weight: Some(0.9), + url: alias_add_url, + }) + .await; + + let alias_query_url = mock_http_server(vec![(200, "[]".to_string())]).await; + handle_alias(AliasArgs { + text: "rust".to_string(), + project: Some("cli-project".to_string()), + add: None, + weight: None, + url: alias_query_url, + }) + .await; + + let memory_get_url = mock_http_server(vec![(200, r#"{"content":"memory","created_at":1.0,"cues":["cli"],"stats":{"reinforcement":1}}"#.to_string())]).await; + handle_memories(MemoriesArgs { + id: 42, + reinforce: false, + delete: false, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: memory_get_url, + }) + .await; + + let memory_reinforce_url = mock_http_server(vec![(200, r#"{"status":"reinforced"}"#.to_string())]).await; + handle_memories(MemoriesArgs { + id: 42, + reinforce: true, + delete: false, + cues: vec!["cli".to_string()], + project: Some("cli-project".to_string()), + url: memory_reinforce_url, + }) + .await; + + let memory_delete_url = mock_http_server(vec![(200, r#"{"status":"deleted"}"#.to_string())]).await; + handle_memories(MemoriesArgs { + id: 42, + reinforce: false, + delete: true, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: memory_delete_url, + }) + .await; + + let memory_missing_url = mock_http_server(vec![(404, r#"{"error":"missing"}"#.to_string())]).await; + handle_memories(MemoriesArgs { + id: 999, + reinforce: false, + delete: false, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: memory_missing_url, + }) + .await; + + let delete_missing_url = mock_http_server(vec![(404, "missing".to_string())]).await; + handle_memories(MemoriesArgs { + id: 404, + reinforce: false, + delete: true, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: delete_missing_url, + }) + .await; + } + + #[tokio::test] + async fn cli_logs_handler_covers_missing_head_tail_and_full_modes() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("server.log"); + std::fs::write(&log_path, "first\nsecond\nthird\n").unwrap(); + + handle_logs(LogsArgs { + head: Some(2), + tail: None, + follow: false, + path: Some(log_path.to_string_lossy().to_string()), + }) + .await; + handle_logs(LogsArgs { + head: None, + tail: Some(2), + follow: false, + path: Some(log_path.to_string_lossy().to_string()), + }) + .await; + handle_logs(LogsArgs { + head: None, + tail: None, + follow: false, + path: Some(log_path.to_string_lossy().to_string()), + }) + .await; + + let follow_path = log_path.clone(); + let follow_task = tokio::spawn(async move { + handle_logs(LogsArgs { + head: None, + tail: None, + follow: true, + path: Some(follow_path.to_string_lossy().to_string()), + }) + .await; + }); + tokio::time::sleep(Duration::from_millis(120)).await; + std::fs::OpenOptions::new() + .append(true) + .open(&log_path) + .unwrap() + .write_all(b"followed\n") + .unwrap(); + tokio::time::sleep(Duration::from_millis(150)).await; + follow_task.abort(); + let _ = follow_task.await; + + handle_logs(LogsArgs { + head: None, + tail: None, + follow: false, + path: Some(dir.path().join("missing.log").to_string_lossy().to_string()), + }) + .await; + } + + #[tokio::test] + async fn cli_handlers_cover_error_responses_and_alias_defaults() { + let ingest_error_url = + mock_http_server(vec![(500, r#"{"error":"invalid content"}"#.to_string())]).await; + handle_ingest(IngestArgs { + type_: IngestType::Content { + content: "bad content".to_string(), + project: Some("cli-project".to_string()), + filename: "note.txt".to_string(), + source_key: None, + metadata: None, + structural_cues: Vec::new(), + segmenter: "sentence_window".to_string(), + segment_window_size: None, + segment_overlap: None, + segment_min_chunk_chars: None, + segment_max_chunk_chars: None, + url: ingest_error_url, + }, + }) + .await; + + let status_error_url = mock_http_server(vec![(500, "server error".to_string())]).await; + handle_status(StatusArgs { + server: true, + jobs: false, + project: None, + json: false, + url: status_error_url, + }) + .await; + let jobs_error_url = mock_http_server(vec![(500, "jobs error".to_string())]).await; + handle_status(StatusArgs { + server: false, + jobs: true, + project: None, + json: true, + url: jobs_error_url, + }) + .await; + + let list_error_url = mock_http_server(vec![(500, "list error".to_string())]).await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::List { url: list_error_url }, + }) + .await; + let create_error_url = mock_http_server(vec![(500, "create error".to_string())]).await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::Create { + name: "bad-project".to_string(), + url: create_error_url, + }, + }) + .await; + let watch_error_url = mock_http_server(vec![(500, "watch error".to_string())]).await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::SetWatchDir { + project: "cli-project".to_string(), + path: "/missing".to_string(), + url: watch_error_url, + }, + }) + .await; + + let alias_add_error_url = mock_http_server(vec![(500, "alias error".to_string())]).await; + handle_alias(AliasArgs { + text: "rust".to_string(), + project: Some("cli-project".to_string()), + add: Some("rust-language".to_string()), + weight: None, + url: alias_add_error_url, + }) + .await; + let alias_query_result_url = mock_http_server(vec![ + ( + 200, + r#"[{"id":1,"from":"rust","to":"rust-language","weight":0.9}]"#.to_string(), + ), + ]) + .await; + handle_alias(AliasArgs { + text: "rust".to_string(), + project: Some("cli-project".to_string()), + add: None, + weight: None, + url: alias_query_result_url, + }) + .await; + + let reinforce_missing_url = mock_http_server(vec![(404, "missing".to_string())]).await; + handle_memories(MemoriesArgs { + id: 404, + reinforce: true, + delete: false, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: reinforce_missing_url, + }) + .await; + let delete_error_url = mock_http_server(vec![(500, "delete error".to_string())]).await; + handle_memories(MemoriesArgs { + id: 500, + reinforce: false, + delete: true, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: delete_error_url, + }) + .await; + } + + #[tokio::test] + async fn cli_lexicon_and_project_config_paths_are_covered() { + let lexicon_url = mock_http_server(vec![ + ( + 200, + r#"{"cue":"rust","outgoing":[],"incoming":[]}"#.to_string(), + ), + ]) + .await; + handle_lexicon(LexiconArgs { + cmd: LexiconCmd::Inspect { + cue: "rust".to_string(), + project: Some("cli-project".to_string()), + url: lexicon_url, + }, + }) + .await; + + let lexicon_error_url = mock_http_server(vec![(500, "lexicon unavailable".to_string())]).await; + handle_lexicon(LexiconArgs { + cmd: LexiconCmd::Inspect { + cue: "missing".to_string(), + project: Some("cli-project".to_string()), + url: lexicon_error_url, + }, + }) + .await; + + let config_dir = tempfile::tempdir().unwrap(); + let config_path = config_dir.path().join("config.json"); + assert_eq!(read_default_project(&config_path), None); + write_default_project(&config_path, "cli-project").unwrap(); + assert_eq!(read_default_project(&config_path).as_deref(), Some("cli-project")); + + std::fs::write(&config_path, "not-json").unwrap(); + assert_eq!(read_default_project(&config_path), None); + std::fs::write(&config_path, r#"{"default_project":42}"#).unwrap(); + assert_eq!(read_default_project(&config_path), None); + assert!(write_default_project(config_dir.path(), "bad-path").is_err()); + } + + #[tokio::test] + async fn cli_handlers_cover_connection_failures_and_recall_modes() { + let dead_url = "http://127.0.0.1:1".to_string(); + handle_add(add_args(dead_url.clone())).await; + + handle_ingest(IngestArgs { + type_: IngestType::Content { + content: "connection failure".to_string(), + project: Some("cli-project".to_string()), + filename: "failure.txt".to_string(), + source_key: None, + metadata: None, + structural_cues: Vec::new(), + segmenter: "sentence_window".to_string(), + segment_window_size: None, + segment_overlap: None, + segment_min_chunk_chars: None, + segment_max_chunk_chars: None, + url: dead_url.clone(), + }, + }) + .await; + handle_ingest(IngestArgs { + type_: IngestType::Url { + url: "https://example.test".to_string(), + project: Some("cli-project".to_string()), + depth: 0, + same_domain_only: true, + server_url: dead_url.clone(), + }, + }) + .await; + + for semantic_mode in ["semantic", "hybrid"] { + let mut recall = recall_args(dead_url.clone()); + recall.semantic_mode = semantic_mode.to_string(); + recall.trace_timing = false; + handle_recall(recall).await; + } + + let mut grounded = recall_args(dead_url.clone()); + grounded.grounded = true; + handle_recall(grounded).await; + let grounded_error_url = mock_http_server(vec![(500, "grounded error".to_string())]).await; + let mut grounded_error = recall_args(grounded_error_url); + grounded_error.grounded = true; + handle_recall(grounded_error).await; + + let web_error_url = mock_http_server(vec![(500, "web error".to_string())]).await; + let mut web_error = recall_args(web_error_url); + web_error.web = true; + handle_recall(web_error).await; + let mut web_failure = recall_args(dead_url.clone()); + web_failure.web = true; + handle_recall(web_failure).await; + + let status_url = mock_http_server(vec![(200, r#"{"total_memories":1}"#.to_string())]).await; + handle_status(StatusArgs { + server: true, + jobs: false, + project: Some("cli-project".to_string()), + json: false, + url: status_url, + }) + .await; + handle_status(StatusArgs { + server: true, + jobs: false, + project: Some("cli-project".to_string()), + json: false, + url: dead_url.clone(), + }) + .await; + handle_status(StatusArgs { + server: false, + jobs: true, + project: Some("cli-project".to_string()), + json: false, + url: dead_url.clone(), + }) + .await; + + handle_memories(MemoriesArgs { + id: 1, + reinforce: false, + delete: true, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: dead_url.clone(), + }) + .await; + handle_memories(MemoriesArgs { + id: 1, + reinforce: true, + delete: false, + cues: vec!["cli".to_string()], + project: Some("cli-project".to_string()), + url: dead_url.clone(), + }) + .await; + handle_memories(MemoriesArgs { + id: 1, + reinforce: false, + delete: false, + cues: Vec::new(), + project: Some("cli-project".to_string()), + url: dead_url.clone(), + }) + .await; + + handle_alias(AliasArgs { + text: "rust".to_string(), + project: Some("cli-project".to_string()), + add: Some("language".to_string()), + weight: None, + url: dead_url.clone(), + }) + .await; + handle_alias(AliasArgs { + text: "rust".to_string(), + project: Some("cli-project".to_string()), + add: None, + weight: None, + url: dead_url.clone(), + }) + .await; + + handle_projects(ProjectArgs { + cmd: ProjectCmd::List { + url: dead_url.clone(), + }, + }) + .await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::Create { + name: "dead-project".to_string(), + url: dead_url.clone(), + }, + }) + .await; + handle_projects(ProjectArgs { + cmd: ProjectCmd::SetWatchDir { + project: "cli-project".to_string(), + path: "/tmp".to_string(), + url: dead_url, + }, + }) + .await; + } + + #[test] + fn cli_ingest_and_lexicon_server_url_defaults_parse() { + let cli = Cli::try_parse_from(["cuemap", "ingest", "file", "note.md", "--project", "p"]) + .unwrap(); + match cli.command { + Commands::Ingest(IngestArgs { + type_: IngestType::File { url, .. }, + }) => assert_eq!(url, "http://localhost:8080"), + _ => panic!("expected file ingest command"), + } + + let cli = Cli::try_parse_from(["cuemap", "ingest", "url", "https://example.test"]) + .unwrap(); + match cli.command { + Commands::Ingest(IngestArgs { + type_: IngestType::Url { server_url, .. }, + }) => assert_eq!(server_url, "http://localhost:8080"), + _ => panic!("expected URL ingest command"), + } + + let cli = Cli::try_parse_from(["cuemap", "lexicon", "inspect", "rust"]).unwrap(); + match cli.command { + Commands::Lexicon(LexiconArgs { + cmd: LexiconCmd::Inspect { url, .. }, + }) => assert_eq!(url, "http://localhost:8080"), + _ => panic!("expected lexicon inspect command"), + } + } +} + #[derive(clap::Subcommand, Debug)] enum Commands { /// Start the CueMap server @@ -40,9 +1207,6 @@ enum Commands { /// Manage lexicon entries Lexicon(LexiconArgs), - /// Manage deterministic CuePacks - Cuepack(CuePackArgs), - /// Manage aliases Alias(AliasArgs), @@ -205,12 +1369,6 @@ struct AddArgs { /// Manual cues to associate #[arg(short, long)] cues: Vec<String>, - /// CuePacks to apply while extracting synchronous facets (comma-separated) - #[arg(long, value_delimiter = ',')] - cuepacks: Option<Vec<String>>, - /// Disable bundled default CuePacks for this add request - #[arg(long)] - disable_default_cuepacks: bool, /// Disable temporal chunking for this memory #[arg(long)] disable_temporal_chunking: bool, @@ -271,6 +1429,9 @@ enum IngestType { path: String, #[arg(short, long)] project: Option<String>, + /// Server URL + #[arg(long, default_value = "http://localhost:8080")] + url: String, }, /// Ingest a URL Url { @@ -283,6 +1444,9 @@ enum IngestType { /// Only follow links within the same domain #[arg(long, default_value = "true")] same_domain_only: bool, + /// Server URL + #[arg(long, default_value = "http://localhost:8080")] + server_url: String, }, } @@ -299,6 +1463,9 @@ struct RecallArgs { /// Manual cues to filter by #[arg(short, long)] cues: Vec<String>, + /// Query signal mode: lexical, semantic, or hybrid + #[arg(long, default_value = "hybrid", value_parser = ["lexical", "semantic", "hybrid"])] + semantic_mode: String, /// Multi-hop recall depth #[arg(long, default_value = "1")] depth: usize, @@ -357,13 +1524,6 @@ struct RecallArgs { #[arg(long, default_value_t = 1)] pub expansion_depth: usize, - /// CuePacks to apply (comma-separated, e.g. "memory-general") - #[arg(long, value_delimiter = ',')] - pub cuepacks: Option<Vec<String>>, - /// Disable bundled default CuePacks for this recall request - #[arg(long)] - pub disable_default_cuepacks: bool, - /// Enable alias expansion (default: disabled) #[arg(long)] pub enable_alias_expansion: bool, @@ -412,25 +1572,12 @@ enum LexiconCmd { cue: String, #[arg(short, long)] project: Option<String>, + /// Server URL + #[arg(long, default_value = "http://localhost:8080")] + url: String, }, } -#[derive(Parser, Debug)] -struct CuePackArgs { - #[command(subcommand)] - cmd: CuePackCmd, -} - -#[derive(clap::Subcommand, Debug)] -enum CuePackCmd { - /// List bundled and local CuePacks - List, - /// Inspect a loaded CuePack by name - Inspect { name: String }, - /// Validate a CuePack TOML file - Validate { path: String }, -} - #[derive(Parser, Debug)] struct MemoriesArgs { /// Memory ID @@ -506,6 +1653,171 @@ enum ProjectCmd { }, } +fn apply_start_overrides( + mut server_config: config::ServerConfig, + args: &StartArgs, +) -> config::ServerConfig { + if let Some(port) = args.port { + server_config.server.port = port; + } + if let Some(data_dir) = &args.data_dir { + server_config.server.data_dir = data_dir.clone(); + } + if let Some(assets_dir) = &args.assets_dir { + server_config.server.assets_dir = Some(assets_dir.clone()); + } + if let Some(snapshot_interval) = args.snapshot_interval { + server_config.persistence.snapshot_interval_seconds = snapshot_interval; + } + if let Some(agent_throttle) = args.agent_throttle { + server_config.agent.throttle_ms = agent_throttle; + } + if let Some(agent_dir) = &args.agent_dir { + server_config.agent.watch_dir = Some(agent_dir.clone()); + server_config.agent.enabled = true; + } + + if args.disable_bg_jobs { + server_config.jobs.background_processing = false; + } + if args.disable_snapshots { + server_config.persistence.enabled = false; + } + if args.disk_content { + server_config.server.store_content_on_disk = true; + } + + if let Some(provider) = &args.cloud_backup { + server_config.persistence.cloud.provider = provider.clone(); + } + if let Some(bucket) = &args.cloud_bucket { + server_config.persistence.cloud.bucket = bucket.clone(); + } + if let Some(region) = &args.cloud_region { + server_config.persistence.cloud.region = region.clone(); + } + if let Some(endpoint) = &args.cloud_endpoint { + server_config.persistence.cloud.endpoint = Some(endpoint.clone()); + } + if let Some(prefix) = &args.cloud_prefix { + server_config.persistence.cloud.prefix = prefix.clone(); + } + if args.cloud_auto_backup { + server_config.persistence.cloud.auto_backup = true; + } + + server_config +} + +fn load_start_config(args: &StartArgs) -> Result<config::ServerConfig, String> { + let config_path = args.config.clone().map(std::path::PathBuf::from); + let config = config::ServerConfig::load(config_path, args.profile.clone())?; + Ok(apply_start_overrides(config, args)) +} + +fn has_main_snapshot(dir: &Path) -> bool { + std::fs::read_dir(dir) + .map(|entries| { + entries.flatten().any(|entry| { + entry + .path() + .file_name() + .and_then(|name| name.to_str()) + .map(|name| { + name.ends_with(".bin") + && !name.ends_with("_aliases.bin") + && !name.ends_with("_lexicon.bin") + }) + .unwrap_or(false) + }) + }) + .unwrap_or(false) +} + +fn select_snapshots_dir(data_dir: &str, load_static: Option<&str>) -> String { + if let Some(static_dir) = load_static { + return static_dir.to_string(); + } + + let configured = PathBuf::from(data_dir).join("snapshots"); + let legacy = PathBuf::from(data_dir).join("..").join("snapshots"); + let selected = if !has_main_snapshot(&configured) && has_main_snapshot(&legacy) { + warn!( + configured = %configured.display(), + legacy = %legacy.display(), + "No snapshots found in configured data directory; using legacy snapshot directory" + ); + legacy + } else { + configured + }; + selected.to_string_lossy().to_string() +} + +fn resolve_master_key( + security: &config::SecurityConfig, + env_master_key: Option<&str>, + env_password: Option<&str>, + password_salt: Option<&[u8]>, +) -> Option<Arc<cuemap::crypto::EncryptionKey>> { + if let Some(key_hex) = env_master_key { + match hex::decode(key_hex) { + Ok(bytes) if bytes.len() == 32 => { + info!("Security: Master key loaded from CUEMAP_MASTER_KEY (Hex)"); + Some(Arc::new(cuemap::crypto::EncryptionKey::new(bytes))) + } + _ => { + error!("Security: CUEMAP_MASTER_KEY must be a 32-byte hex string"); + None + } + } + } else if let Some(passphrase) = env_password { + info!("Security: Deriving master key from CUEMAP_MASTER_PASSWORD..."); + let salt = password_salt.unwrap_or_default(); + Some(Arc::new(cuemap::crypto::EncryptionKey::from_passphrase( + passphrase, salt, + ))) + } else if let Some(key_hex) = &security.master_key { + match hex::decode(key_hex) { + Ok(bytes) if bytes.len() == 32 => { + info!("Security: Master key loaded from config file"); + Some(Arc::new(cuemap::crypto::EncryptionKey::new(bytes))) + } + _ => { + error!("Security: master_key in config must be a 32-byte hex string"); + None + } + } + } else { + info!("Security: Encryption-at-rest disabled (no master key configured)"); + None + } +} + +fn resolve_context_signer( + security: &config::SecurityConfig, +) -> Option<Arc<cuemap::crypto::ContextSigner>> { + if let Some(seed_hex) = &security.signing_private_key { + match crypto::ContextSigner::from_ed25519_seed_hex(seed_hex) { + Ok(signer) => { + info!("Immutable RAG: Ed25519 context signing enabled"); + Some(Arc::new(signer)) + } + Err(err) => { + error!("Immutable RAG: invalid Ed25519 signing private key: {}", err); + None + } + } + } else if let Some(secret) = &security.secret_key { + info!("Immutable RAG: legacy HMAC-SHA256 context signing enabled"); + Some(Arc::new(crypto::ContextSigner::from_hmac_secret( + secret.clone().into_bytes(), + ))) + } else { + None + } +} + #[tokio::main] async fn main() { let cli = Cli::parse(); @@ -516,62 +1828,7 @@ async fn main() { handle_start_detached(args).await; } else { // Layering Logic: Config File -> CLI Args - let config_path = args.config.clone().map(std::path::PathBuf::from); - let mut config = config::ServerConfig::load(config_path, args.profile.clone()) - .expect("Failed to load configuration"); - - // Apply CLI Overrides - if let Some(p) = args.port { - config.server.port = p; - } - if let Some(d) = &args.data_dir { - config.server.data_dir = d.clone(); - } - if let Some(a) = &args.assets_dir { - config.server.assets_dir = Some(a.clone()); - } - if let Some(s) = args.snapshot_interval { - config.persistence.snapshot_interval_seconds = s; - } - if let Some(t) = args.agent_throttle { - config.agent.throttle_ms = t; - } - if let Some(w) = &args.agent_dir { - config.agent.watch_dir = Some(w.clone()); - config.agent.enabled = true; - } - - // Boolean flags (only enable restriction/feature if flag is present, or if config says so) - // For "disable" flags: if CLI says disable, force disable. - if args.disable_bg_jobs { - config.jobs.background_processing = false; - } - if args.disable_snapshots { - config.persistence.enabled = false; - } - if args.disk_content { - config.server.store_content_on_disk = true; - } - - // Cloud overrides - if let Some(p) = &args.cloud_backup { - config.persistence.cloud.provider = p.clone(); - } - if let Some(b) = &args.cloud_bucket { - config.persistence.cloud.bucket = b.clone(); - } - if let Some(r) = &args.cloud_region { - config.persistence.cloud.region = r.clone(); - } - if let Some(e) = &args.cloud_endpoint { - config.persistence.cloud.endpoint = Some(e.clone()); - } - if let Some(p) = &args.cloud_prefix { - config.persistence.cloud.prefix = p.clone(); - } - if args.cloud_auto_backup { - config.persistence.cloud.auto_backup = true; - } + let config = load_start_config(&args).expect("Failed to load configuration"); run_server(config, args.load_static, args.child_process).await; } @@ -580,7 +1837,6 @@ async fn main() { Commands::Ingest(args) => handle_ingest(args).await, Commands::Recall(args) => handle_recall(args).await, Commands::Lexicon(args) => handle_lexicon(args).await, - Commands::Cuepack(args) => handle_cuepack(args), Commands::Memories(args) => handle_memories(args).await, Commands::Alias(args) => handle_alias(args).await, Commands::Projects(args) => handle_projects(args).await, @@ -592,6 +1848,16 @@ async fn main() { } async fn run_server(config: config::ServerConfig, load_static: Option<String>, _is_child: bool) { + let pid_path = config::get_base_dir().join("server.pid"); + run_server_with_pid_path(config, load_static, _is_child, pid_path).await; +} + +async fn run_server_with_pid_path( + config: config::ServerConfig, + load_static: Option<String>, + _is_child: bool, + pid_path: PathBuf, +) { // Extract commonly used configs let server_config = &config.server; let auth_config_struct = &config.security; @@ -608,11 +1874,13 @@ async fn run_server(config: config::ServerConfig, load_static: Option<String>, _ // Build layers let stdout_layer = fmt::layer().with_writer(std::io::stdout); - Registry::default().with(filter).with(stdout_layer).init(); + let _ = Registry::default() + .with(filter) + .with(stdout_layer) + .try_init(); // Write PID file for the server let pid = std::process::id(); - let pid_path = config::get_base_dir().join("server.pid"); if let Err(e) = std::fs::write(&pid_path, pid.to_string()) { warn!("Failed to write PID file: {}", e); } @@ -643,106 +1911,43 @@ async fn run_server(config: config::ServerConfig, load_static: Option<String>, _ } } - use cuemap::crypto::EncryptionKey; - // Build the router with appropriate engine state info!("Multi-tenant mode enabled"); - let snapshots_dir = if let Some(ref static_dir) = load_static { - static_dir.clone() - } else { - PathBuf::from(&server_config.data_dir) - .join("snapshots") - .to_string_lossy() - .to_string() - }; + let snapshots_dir = select_snapshots_dir(&server_config.data_dir, load_static.as_deref()); let mut mt_engine = multi_tenant::MultiTenantEngine::with_config( config.clone(), PathBuf::from(&snapshots_dir), ); - let mut cuepack_dirs = config - .cuepacks - .dirs - .iter() - .map(PathBuf::from) - .collect::<Vec<_>>(); - cuepack_dirs.push(config::get_base_dir().join("cuepacks")); - let cuepack_registry = Arc::new(if config.cuepacks.enabled { - cuemap::cuepacks::CuePackRegistry::load(config.cuepacks.default_packs_enabled, &cuepack_dirs) - } else { - cuemap::cuepacks::CuePackRegistry::load(false, &[]) - }); - for error in cuepack_registry.load_errors() { - warn!("CuePack load error: {}", error); - } - info!("Loaded {} CuePacks", cuepack_registry.infos().len()); - // Master Key Discovery Hierarchy - let master_key = if let Ok(key_hex) = std::env::var("CUEMAP_MASTER_KEY") { - // 1. Env Var (Hex) - Highest priority for automation - match hex::decode(&key_hex) { - Ok(bytes) if bytes.len() == 32 => { - info!("Security: Master key loaded from CUEMAP_MASTER_KEY (Hex)"); - Some(Arc::new(EncryptionKey::new(bytes))) - } - _ => { - error!("Security: CUEMAP_MASTER_KEY must be a 32-byte hex string"); - None - } - } - } else if let Ok(pass) = std::env::var("CUEMAP_MASTER_PASSWORD") { - // 2. Env Var (Passphrase) - Secondary automation path - info!("Security: Deriving master key from CUEMAP_MASTER_PASSWORD..."); - let salt = get_or_create_salt(); - Some(Arc::new(EncryptionKey::from_passphrase(&pass, &salt))) - } else if let Some(key_hex) = &auth_config_struct.master_key { - // 3. Config File (Hex) - match hex::decode(key_hex) { - Ok(bytes) if bytes.len() == 32 => { - info!("Security: Master key loaded from config file"); - Some(Arc::new(EncryptionKey::new(bytes))) - } - _ => { - error!("Security: master_key in config must be a 32-byte hex string"); - None - } - } - } else { - info!("Security: Encryption-at-rest disabled (no master key configured)"); - None - }; + let env_master_key = std::env::var("CUEMAP_MASTER_KEY").ok(); + let env_password = std::env::var("CUEMAP_MASTER_PASSWORD").ok(); + let password_salt = env_password.as_ref().map(|_| get_or_create_salt()); + let master_key = resolve_master_key( + auth_config_struct, + env_master_key.as_deref(), + env_password.as_deref(), + password_salt.as_deref(), + ); if let Some(key) = master_key { mt_engine.set_master_key(Some(key)); } - let context_signer = if let Some(seed_hex) = &auth_config_struct.signing_private_key { - match crypto::ContextSigner::from_ed25519_seed_hex(seed_hex) { - Ok(signer) => { - info!("Immutable RAG: Ed25519 context signing enabled"); - Some(Arc::new(signer)) - } - Err(err) => { - error!("Immutable RAG: invalid Ed25519 signing private key: {}", err); - None - } - } - } else if let Some(secret) = &auth_config_struct.secret_key { - info!("Immutable RAG: legacy HMAC-SHA256 context signing enabled"); - Some(Arc::new(crypto::ContextSigner::from_hmac_secret( - secret.clone().into_bytes(), - ))) - } else { - None - }; + let context_signer = resolve_context_signer(auth_config_struct); let mt_engine = Arc::new(mt_engine); // Auto-load all available snapshots info!("Loading snapshots from: {}", snapshots_dir); - let _ = mt_engine.load_all(); // Ignoring errors for brevity + for (project_id, result) in mt_engine.load_all() { + match result { + Ok(()) => info!(project_id = %project_id, "Loaded project snapshot"), + Err(error) => error!(project_id = %project_id, error = %error, "Failed to load project snapshot"), + } + } // Setup shutdown handler if !is_static { @@ -788,8 +1993,9 @@ async fn run_server(config: config::ServerConfig, load_static: Option<String>, _ .join("snapshots") .join(format!("{}_agent_state.json", meta.project_id)), ), - ignored_patterns: Vec::new(), - ignored_extensions: Vec::new(), + included_paths: meta.included_paths, + ignored_patterns: meta.ignored_patterns, + ignored_extensions: meta.ignored_extensions, }; agent_manager .start_agent(&meta.project_id, agent_config) @@ -832,7 +2038,6 @@ async fn run_server(config: config::ServerConfig, load_static: Option<String>, _ cloud_backup, context_signer, agent_manager.clone(), - cuepack_registry, )) .layer(CorsLayer::permissive()); @@ -908,41 +2113,44 @@ async fn setup_multi_tenant_shutdown_handler(mt_engine: Arc<multi_tenant::MultiT // ========== CLI Client Handlers ========== +fn read_default_project(config_path: &Path) -> Option<String> { + let content = std::fs::read_to_string(config_path).ok()?; + let config = serde_json::from_str::<serde_json::Value>(&content).ok()?; + config + .get("default_project") + .and_then(|value| value.as_str().map(str::to_string)) +} + +fn write_default_project(config_path: &Path, project_id: &str) -> Result<(), String> { + let config = serde_json::json!({ + "default_project": project_id + }); + let content = serde_json::to_string_pretty(&config).map_err(|err| err.to_string())?; + std::fs::write(config_path, content).map_err(|err| err.to_string()) +} + fn get_default_project() -> Option<String> { let config_path = config::get_base_dir().join("config.json"); - if let Ok(content) = std::fs::read_to_string(config_path) { - if let Ok(config) = serde_json::from_str::<serde_json::Value>(&content) { - return config - .get("default_project") - .and_then(|v| v.as_str().map(|s| s.to_string())); - } - } - None + read_default_project(&config_path) } fn handle_set_project(project_id: String) { let config_path = config::get_base_dir().join("config.json"); - let config = serde_json::json!({ - "default_project": project_id - }); - if let Ok(content) = serde_json::to_string_pretty(&config) { - if std::fs::write(config_path, content).is_ok() { - println!("✓ Default project set to: {}", project_id); - } else { - eprintln!("✗ Failed to write config file"); - } + match write_default_project(&config_path, &project_id) { + Ok(()) => println!("✓ Default project set to: {}", project_id), + Err(_) => eprintln!("✗ Failed to write config file"), } } -fn get_or_create_salt() -> Vec<u8> { +fn load_or_create_salt(base_dir: &Path, env_override: Option<&str>) -> Vec<u8> { // 1. Check environment variable override (escape hatch / migration) - if let Ok(salt_str) = std::env::var("CUEMAP_KDF_SALT") { + if let Some(salt_str) = env_override { info!("Security: Using KDF salt from environment (CUEMAP_KDF_SALT)"); - return salt_str.into_bytes(); + return salt_str.as_bytes().to_vec(); } // 2. Check local config file - let salt_path = config::get_base_dir().join("salt"); + let salt_path = base_dir.join("salt"); if salt_path.exists() { if let Ok(salt) = std::fs::read(&salt_path) { if salt.len() >= 16 { @@ -970,22 +2178,26 @@ fn get_or_create_salt() -> Vec<u8> { salt } +fn get_or_create_salt() -> Vec<u8> { + let env_override = std::env::var("CUEMAP_KDF_SALT").ok(); + load_or_create_salt(&config::get_base_dir(), env_override.as_deref()) +} + async fn handle_add(args: AddArgs) { let project = args .project .or_else(get_default_project) .expect("Project ID required (use --project or set-project)"); let client = reqwest::Client::new(); - let cuepacks = selected_cuepacks(args.cuepacks, args.disable_default_cuepacks); - let payload = api::AddMemoryRequest { content: args.content, source_key: None, + event_time: None, metadata: args .metadata .map(|m| serde_json::from_str(&m).unwrap_or_default()), + embedding: None, cues: args.cues, - cuepacks, disable_temporal_chunking: args.disable_temporal_chunking, async_ingest: args.async_ingest, minimal_response: false, @@ -1064,13 +2276,13 @@ async fn handle_ingest(args: IngestArgs) { Err(e) => eprintln!("✗ Failed: {}", e), } } - IngestType::File { path, project } => { + IngestType::File { path, project, url } => { let project = project .or_else(get_default_project) .expect("Project ID required"); if let Ok(content) = std::fs::read_to_string(&path) { let res = client - .post("http://localhost:8080/ingest/content") + .post(format!("{}/ingest/content", url)) .header("X-Project-ID", project) .json(&serde_json::json!({ "content": content, "filename": path })) .send() @@ -1089,12 +2301,13 @@ async fn handle_ingest(args: IngestArgs) { project, depth, same_domain_only, + server_url, } => { let project = project .or_else(get_default_project) .expect("Project ID required"); let res = client - .post("http://localhost:8080/ingest/url") + .post(format!("{}/ingest/url", server_url)) .header("X-Project-ID", project) .json(&api::IngestUrlRequest { url, @@ -1118,7 +2331,6 @@ async fn handle_recall(args: RecallArgs) { .or_else(get_default_project) .expect("Project ID required"); let client = reqwest::Client::new(); - let cuepacks = selected_cuepacks(args.cuepacks.clone(), args.disable_default_cuepacks); let parent_fusion = match args.parent_fusion.as_str() { "auto" => api::ParentFusionMode::Auto, "force" => api::ParentFusionMode::Force, @@ -1146,7 +2358,6 @@ async fn handle_recall(args: RecallArgs) { min_intersection: args.min_intersection, disable_alias_expansion: !args.enable_alias_expansion, expansion_depth: args.expansion_depth, - cuepacks: cuepacks.clone(), }; let res = client .post(format!("{}/recall/grounded", args.url)) @@ -1206,6 +2417,12 @@ async fn handle_recall(args: RecallArgs) { let payload = api::RecallRequest { cues: args.cues, query_text: Some(args.query), + query_embedding: None, + semantic_mode: match args.semantic_mode.as_str() { + "lexical" => cuemap::semantic::SemanticRecallMode::Lexical, + "semantic" => cuemap::semantic::SemanticRecallMode::Semantic, + _ => cuemap::semantic::SemanticRecallMode::Hybrid, + }, query_time: args.query_time, limit: args.limit, auto_reinforce: !args.no_auto_reinforce, @@ -1217,7 +2434,6 @@ async fn handle_recall(args: RecallArgs) { disable_alias_expansion: !args.enable_alias_expansion, depth: args.depth, expansion_depth: args.expansion_depth, - cuepacks, parent_fusion, parent_fusion_limit: args.parent_fusion_limit, parent_fusion_min_chunks: args.parent_fusion_min_chunks, @@ -1274,12 +2490,12 @@ async fn handle_lexicon(args: LexiconArgs) { let client = reqwest::Client::new(); match args.cmd { - LexiconCmd::Inspect { cue, project } => { + LexiconCmd::Inspect { cue, project, url } => { let project_id = project .or_else(get_default_project) .expect("Project ID required"); let res = client - .get(format!("http://localhost:8080/lexicon/inspect/{}", cue)) + .get(format!("{}/lexicon/inspect/{}", url, cue)) .header("X-Project-ID", project_id) .send() .await; @@ -1295,78 +2511,6 @@ async fn handle_lexicon(args: LexiconArgs) { } } -fn selected_cuepacks(cuepacks: Option<Vec<String>>, disable_defaults: bool) -> Option<Vec<String>> { - if disable_defaults { - Some(vec!["off".to_string()]) - } else { - cuepacks - } -} - -fn local_cuepack_registry() -> cuemap::cuepacks::CuePackRegistry { - cuemap::cuepacks::CuePackRegistry::load_from_default_locations(true) -} - -fn handle_cuepack(args: CuePackArgs) { - match args.cmd { - CuePackCmd::List => { - let registry = local_cuepack_registry(); - println!("\n--- CUEPACKS ---"); - for info in registry.infos() { - let default_marker = if info.enabled_by_default { - "default" - } else { - "opt-in" - }; - println!( - "- {} {} [{}] memory_rules={} query_rules={} source={}", - info.name, - info.version, - default_marker, - info.memory_rules, - info.query_rules, - info.source - ); - } - for error in registry.load_errors() { - eprintln!("! {}", error); - } - } - CuePackCmd::Inspect { name } => { - let registry = local_cuepack_registry(); - let Some(info) = registry - .infos() - .into_iter() - .find(|info| info.name == name) - else { - eprintln!("✗ CuePack not found: {}", name); - return; - }; - println!("\n--- CUEPACK: {} ---", info.name); - println!("Version: {}", info.version); - println!("Default: {}", info.enabled_by_default); - println!("Source: {}", info.source); - println!("Memory rules: {}", info.memory_rules); - println!("Query rules: {}", info.query_rules); - if let Some(description) = info.description { - println!("{}", description); - } - } - CuePackCmd::Validate { path } => { - match cuemap::cuepacks::CuePackRegistry::validate_file(Path::new(&path)) { - Ok(info) => { - println!("✓ CuePack is valid: {} {}", info.name, info.version); - println!( - "memory_rules={} query_rules={}", - info.memory_rules, info.query_rules - ); - } - Err(err) => eprintln!("✗ {}", err), - } - } - } -} - async fn handle_memories(args: MemoriesArgs) { let project = args .project @@ -1685,6 +2829,65 @@ async fn handle_logs(args: LogsArgs) { } } +async fn wait_for_readiness( + log_path: &Path, + start_pos: u64, + sentinel: &str, + timeout: Duration, +) -> std::io::Result<bool> { + let mut file = File::open(log_path)?; + let _ = file.seek(SeekFrom::Start(start_pos)); + let mut reader = BufReader::new(file); + let start_time = std::time::Instant::now(); + let mut line = String::new(); + + while start_time.elapsed() < timeout { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Ok(_) => { + print!("{}", line); + if line.contains(sentinel) { + return Ok(true); + } + } + Err(_) => return Ok(false), + } + } + + Ok(false) +} + +async fn spawn_detached_process( + exe: &Path, + child_args: &[String], + log_path: &Path, + sentinel: &str, + timeout: Duration, +) -> std::io::Result<bool> { + let log_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path)?; + let stdout_file = log_file.try_clone()?; + let stderr_file = log_file.try_clone()?; + let start_pos = log_file.metadata().map(|metadata| metadata.len()).unwrap_or(0); + + let child = std::process::Command::new(exe) + .args(child_args) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::from(stdout_file)) + .stderr(std::process::Stdio::from(stderr_file)) + .spawn()?; + + println!("✓ Background server spawning (PID: {})...", child.id()); + println!("✓ Waiting for readiness sentinel in {}...", log_path.display()); + tokio::time::sleep(Duration::from_millis(100)).await; + wait_for_readiness(log_path, start_pos, sentinel, timeout).await +} + async fn handle_start_detached(args: StartArgs) { let mut child_args: Vec<String> = std::env::args() .filter(|a| a != "--detach" && a != "-d") @@ -1701,87 +2904,42 @@ async fn handle_start_detached(args: StartArgs) { path.to_string_lossy().to_string() }); - // Open log file for redirection - let log_file = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&log_path) - .expect("Failed to open log file"); - - // Clone file handles for stdout and stderr - let stdout_file = log_file - .try_clone() - .expect("Failed to clone log file handle"); - let stderr_file = log_file - .try_clone() - .expect("Failed to clone log file handle"); - - // Capture current size to start reading from - let start_pos = log_file.metadata().map(|m| m.len()).unwrap_or(0); - - // Spawn the child - let _child = std::process::Command::new(&exe) - .args(&child_args[1..]) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::from(stdout_file)) - .stderr(std::process::Stdio::from(stderr_file)) - .spawn() - .expect("Failed to spawn background server"); - - println!("✓ Background server spawning (PID: {})...", _child.id()); - println!("✓ Waiting for readiness sentinel in {}...", log_path); - // Readiness sentinel we are looking for: "Unstable sorting for speed" let sentinel = "Unstable sorting for speed"; - let start_time = std::time::Instant::now(); let timeout = Duration::from_secs(30); - // Wait for logs to appear - tokio::time::sleep(Duration::from_millis(100)).await; - - if let Ok(mut file) = File::open(&log_path) { - let _ = file.seek(SeekFrom::Start(start_pos)); // Start tailing from the spawn time - let mut reader = BufReader::new(file); - let mut line = String::new(); - let mut found = false; - - while start_time.elapsed() < timeout { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => { - tokio::time::sleep(Duration::from_millis(100)).await; - } - Ok(_) => { - print!("{}", line); - if line.contains(sentinel) { - found = true; - break; - } - } - Err(_) => break, - } - } - - if found { + match spawn_detached_process( + &exe, + &child_args[1..], + Path::new(&log_path), + sentinel, + timeout, + ) + .await + { + Ok(true) => { println!("\n✓ CueMap server is now running in the background."); println!(" - View logs: cuemap logs --follow"); println!(" - Stop server: cuemap stop"); - } else { + } + Ok(false) => { eprintln!( "\n✗ Timeout waiting for server readiness. Check logs at: {}", log_path ); } - } else { - eprintln!( + Err(_) => eprintln!( "\n✗ Could not open log file to verify startup: {}", log_path - ); + ), } } async fn handle_stop(_args: StopArgs) { - let pid_path = config::get_base_dir().join("server.pid"); + handle_stop_at(config::get_base_dir().join("server.pid")).await; +} + +async fn handle_stop_at(pid_path: PathBuf) { if !pid_path.exists() { eprintln!("✗ No server.pid found. Server might not be running or wasn't started with this version."); return; @@ -1793,6 +2951,10 @@ async fn handle_stop(_args: StopArgs) { #[cfg(unix)] { use std::process::Command; + if pid <= 1 || pid > i32::MAX as u32 { + eprintln!("✗ Refusing to signal invalid server PID {}.", pid); + return; + } let res = Command::new("kill") .arg("-15") // SIGTERM .arg(pid.to_string()) diff --git a/src/metrics.rs b/src/metrics.rs index 20be5b0..144e93d 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -124,53 +124,5 @@ pub fn get_memory_usage_bytes() -> u64 { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_ingestion_counter() { - let metrics = MetricsCollector::new(); - assert_eq!(metrics.ingestion_count.load(Ordering::Relaxed), 0); - - metrics.record_ingestion(); - metrics.record_ingestion(); - - assert_eq!(metrics.ingestion_count.load(Ordering::Relaxed), 2); - } - - #[test] - fn test_recall_counter_and_latency() { - let metrics = MetricsCollector::new(); - - metrics.record_recall(1.0); - metrics.record_recall(2.0); - metrics.record_recall(10.0); - - assert_eq!(metrics.recall_count.load(Ordering::Relaxed), 3); - - // With only 3 samples, P99 should be the max - let p99 = metrics.get_p99_latency(); - assert!((p99 - 10.0).abs() < 0.01); - } - - #[test] - fn test_avg_latency() { - let metrics = MetricsCollector::new(); - - metrics.record_recall(1.0); - metrics.record_recall(2.0); - metrics.record_recall(3.0); - - let avg = metrics.get_avg_latency(); - assert!((avg - 2.0).abs() < 0.01); - } - - #[test] - fn test_empty_latencies() { - let metrics = MetricsCollector::new(); - - assert_eq!(metrics.get_p99_latency(), 0.0); - assert_eq!(metrics.get_avg_latency(), 0.0); - assert_eq!(metrics.get_sample_count(), 0); - } -} +#[path = "../tests/unit/metrics.rs"] +mod tests; diff --git a/src/multi_tenant.rs b/src/multi_tenant.rs index db4b7b0..cc193ac 100644 --- a/src/multi_tenant.rs +++ b/src/multi_tenant.rs @@ -6,6 +6,7 @@ use crate::engine::CueMapEngine; use crate::normalization::NormalizationConfig; use crate::persistence::PersistenceManager; use crate::projects::ProjectContext; +use crate::semantic::SemanticEncoder; use crate::structures::{LexiconStats, MainStats}; use crate::taxonomy::Taxonomy; use ahash::RandomState; @@ -15,7 +16,7 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, OnceLock, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; pub type ProjectId = String; @@ -35,6 +36,12 @@ pub struct ProjectMeta { pub created_at: u64, pub watch_dir: Option<String>, pub agent_enabled: bool, + #[serde(default)] + pub included_paths: Vec<String>, + #[serde(default)] + pub ignored_patterns: Vec<String>, + #[serde(default)] + pub ignored_extensions: Vec<String>, } impl ProjectMeta { @@ -47,6 +54,9 @@ impl ProjectMeta { .as_secs(), watch_dir: None, agent_enabled: false, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), } } } @@ -58,6 +68,7 @@ pub struct MultiTenantEngine { master_key: Option<Arc<EncryptionKey>>, tuning: Arc<TuningConfig>, config: crate::config::ServerConfig, + semantic_encoder: Arc<OnceLock<Result<Option<Arc<dyn SemanticEncoder>>, String>>>, } impl MultiTenantEngine { @@ -83,24 +94,43 @@ impl MultiTenantEngine { master_key: None, tuning: Arc::new(tuning), config: crate::config::ServerConfig::default(), + semantic_encoder: Arc::new(OnceLock::new()), } } pub fn with_config( - config: crate::config::ServerConfig, + mut config: crate::config::ServerConfig, snapshots_dir: PathBuf, ) -> Self { if let Err(e) = fs::create_dir_all(&snapshots_dir) { eprintln!("Warning: Failed to create snapshots directory: {}", e); } - Self { + config.semantic = config.semantic.resolved(); + + let engine = Self { projects: Arc::new(DashMap::with_hasher(RandomState::new())), snapshots_dir, master_key: None, tuning: Arc::new(config.tuning.clone()), config, + semantic_encoder: Arc::new(OnceLock::new()), + }; + if engine.config.semantic.encoder_enabled { + if let Err(error) = engine.configured_semantic_encoder() { + tracing::warn!( + error = %error, + "Semantic encoder unavailable; continuing without automatic text embeddings" + ); + } } + engine + } + + fn configured_semantic_encoder(&self) -> Result<Option<Arc<dyn SemanticEncoder>>, String> { + self.semantic_encoder + .get_or_init(|| crate::semantic::load_configured_encoder(&self.config.semantic)) + .clone() } pub fn set_master_key(&mut self, key: Option<Arc<EncryptionKey>>) { @@ -116,12 +146,24 @@ impl MultiTenantEngine { Ok(ctx.clone()) } else { // Create new project with default config - let mut ctx_obj = ProjectContext::new( + let semantic_encoder = match self.configured_semantic_encoder() { + Ok(encoder) => encoder, + Err(error) => { + tracing::warn!( + project_id = %project_id, + error = %error, + "Semantic encoder unavailable; continuing without automatic text embeddings" + ); + None + } + }; + let mut ctx_obj = ProjectContext::new_with_encoder( NormalizationConfig::default(), Taxonomy::default(), self.tuning.clone(), self.config.clone(), project_id.clone(), + semantic_encoder, ); // Set master key on engines @@ -259,6 +301,16 @@ impl MultiTenantEngine { ); main_engine.set_master_key(self.master_key.clone()); main_engine.set_tuning_config(self.tuning.as_ref().clone()); + match self.configured_semantic_encoder() { + Ok(semantic_encoder) => main_engine.set_semantic_encoder(semantic_encoder), + Err(error) => { + tracing::warn!( + project_id = %project_id, + error = %error, + "Semantic encoder unavailable for loaded project; continuing without automatic text embeddings" + ); + } + } // Load aliases engine (optional - may not exist for older snapshots) let mut aliases_engine = if aliases_path.exists() { @@ -273,6 +325,7 @@ impl MultiTenantEngine { tracing::debug!("Loaded aliases for project '{}'", project_id); let mut local_config = self.config.clone(); local_config.server.store_content_on_disk = false; + local_config.semantic = crate::semantic::SemanticConfig::default(); let engine = CueMapEngine::from_state( memories, source_key_to_id, @@ -308,6 +361,7 @@ impl MultiTenantEngine { tracing::debug!("Loaded lexicon for project '{}'", project_id); let mut local_config = self.config.clone(); local_config.server.store_content_on_disk = false; + local_config.semantic = crate::semantic::SemanticConfig::default(); let engine = CueMapEngine::from_state( memories, source_key_to_id, @@ -456,6 +510,30 @@ impl MultiTenantEngine { Ok(()) } + /// Persist the complete repository ingestion scope for a project. + pub fn set_project_watch_config( + &self, + project_id: &str, + watch_dir: String, + included_paths: Vec<String>, + ignored_patterns: Vec<String>, + ignored_extensions: Vec<String>, + ) -> Result<ProjectMeta, String> { + let path = Path::new(&watch_dir); + if !path.is_dir() { + return Err(format!("Directory '{}' does not exist", watch_dir)); + } + + let mut meta = self.load_project_meta(&project_id.to_string())?; + meta.watch_dir = Some(watch_dir); + meta.agent_enabled = true; + meta.included_paths = included_paths; + meta.ignored_patterns = ignored_patterns; + meta.ignored_extensions = ignored_extensions; + self.save_project_meta(&meta)?; + Ok(meta) + } + pub fn get_global_stats(&self) -> HashMap<String, serde_json::Value> { let projects = self.list_projects(); diff --git a/src/nl.rs b/src/nl.rs index 758afb6..44b43fd 100644 --- a/src/nl.rs +++ b/src/nl.rs @@ -987,18 +987,5 @@ pub fn tokenize_to_cues_with_lang(text: &str, lang: Language) -> Vec<String> { } #[cfg(test)] -mod tests { - use super::tokenize_to_cues; - - #[test] - fn temporal_connector_breaks_phrase_without_removing_token() { - let cues = tokenize_to_cues( - "Maya switched from coffee to mint tea after the April deploy.", - ); - - assert!(cues.contains(&"after".to_string())); - assert!(cues.contains(&"mint_tea".to_string())); - assert!(cues.contains(&"april_deploy".to_string())); - assert!(!cues.contains(&"mint_tea_after".to_string())); - } -} +#[path = "../tests/unit/nl.rs"] +mod tests; diff --git a/src/persistence.rs b/src/persistence.rs index 9ffae23..d0b732d 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -51,7 +51,59 @@ struct PersistedState<T> { cue_global_counts: Option<HashMap<String, u64>>, } -const PERSISTENCE_VERSION: u32 = 2; +// Version 2 snapshots used bincode. Version 3 uses zstd-compressed JSON so +// serde_json::Value metadata can be restored reliably (bincode cannot +// deserialize dynamic JSON values) without making snapshots unnecessarily +// large. Keep accepting v2 for snapshots that do not contain such metadata. +const PERSISTENCE_VERSION: u32 = 3; +const LEGACY_PERSISTENCE_VERSION: u32 = 2; + +fn serialize_state<T>(state: &PersistedState<T>) -> Result<Vec<u8>, std::io::Error> +where + T: Serialize, +{ + let json = serde_json::to_vec(state).map_err(|error| { + std::io::Error::new(std::io::ErrorKind::InvalidData, error) + })?; + zstd::stream::encode_all(std::io::Cursor::new(json), 3) +} + +fn deserialize_state<T>(data: &[u8]) -> Result<PersistedState<T>, std::io::Error> +where + T: for<'de> Deserialize<'de>, +{ + let decoded = if crate::crypto::is_compressed(data) { + zstd::stream::decode_all(std::io::Cursor::new(data))? + } else { + data.to_vec() + }; + + match serde_json::from_slice(&decoded) { + Ok(state) => Ok(state), + Err(json_error) => bincode::deserialize(&decoded).map_err(|bincode_error| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "failed to decode snapshot as JSON ({json_error}) or legacy bincode ({bincode_error})" + ), + ) + }), + } +} + +fn check_snapshot_version(version: u32) -> Result<(), std::io::Error> { + if version == PERSISTENCE_VERSION || version == LEGACY_PERSISTENCE_VERSION { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "Unsupported snapshot version {} (expected {} or {}). Reingest required.", + version, PERSISTENCE_VERSION, LEGACY_PERSISTENCE_VERSION + ), + )) + } +} pub struct PersistenceManager { data_dir: PathBuf, @@ -143,8 +195,9 @@ impl PersistenceManager { cue_global_counts: global_counts_map, }; - // Serialize to bincode - let data = bincode::serialize(&state)?; + // JSON is intentionally used for new snapshots, then compressed to + // keep the on-disk representation close to the old bincode size. + let data = serialize_state(&state)?; // Write to temp file first (atomic operation) let temp_path = path.with_extension("bin.tmp"); @@ -197,14 +250,8 @@ impl PersistenceManager { info!("Loading state from {:?}", path); let data = fs::read(path)?; - let state: PersistedState<T> = bincode::deserialize(&data)?; - if state.version != PERSISTENCE_VERSION { - return Err(format!( - "Unsupported snapshot version {} (expected {}). Reingest required.", - state.version, PERSISTENCE_VERSION - ) - .into()); - } + let state: PersistedState<T> = deserialize_state(&data)?; + check_snapshot_version(state.version)?; info!( "Loaded {} memories and {} cues from snapshot (version: {}, saved: {})", @@ -329,14 +376,8 @@ impl PersistenceManager { info!("Loading state from {:?}", snapshot_path); let data = fs::read(&snapshot_path)?; - let state: PersistedState<T> = bincode::deserialize(&data)?; - if state.version != PERSISTENCE_VERSION { - return Err(format!( - "Unsupported snapshot version {} (expected {}). Reingest required.", - state.version, PERSISTENCE_VERSION - ) - .into()); - } + let state: PersistedState<T> = deserialize_state(&data)?; + check_snapshot_version(state.version)?; info!( "Loaded {} memories and {} cues from snapshot (version: {}, saved: {})", @@ -440,8 +481,9 @@ impl PersistenceManager { cue_global_counts: global_counts_map, }; - // Serialize to bincode - let data = bincode::serialize(&state)?; + // Keep the on-disk format compressed JSON so arbitrary metadata values + // round-trip without sacrificing snapshot size. + let data = serialize_state(&state)?; // Write to temp file first (atomic operation) let temp_path = self.temp_snapshot_path(); @@ -925,84 +967,5 @@ impl CloudBackupManager { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_config_from_args_s3() { - let config = CloudBackupConfig::from_args( - Some("s3"), - Some("my-bucket"), - Some("us-west-2"), - None, - "cuemap/", - true, - ) - .unwrap(); - - assert!(config.enabled); - assert!(config.auto_backup); - assert_eq!(config.prefix, "cuemap/"); - - match config.provider { - Some(CloudProvider::S3 { - bucket, - region, - endpoint, - }) => { - assert_eq!(bucket, "my-bucket"); - assert_eq!(region, "us-west-2"); - assert!(endpoint.is_none()); - } - _ => panic!("Expected S3 provider"), - } - } - - #[test] - fn test_config_from_args_s3_with_endpoint() { - let config = CloudBackupConfig::from_args( - Some("s3"), - Some("my-bucket"), - Some("us-east-1"), - Some("http://localhost:9000"), - "backups/", - false, - ) - .unwrap(); - - match config.provider { - Some(CloudProvider::S3 { endpoint, .. }) => { - assert_eq!(endpoint, Some("http://localhost:9000".to_string())); - } - _ => panic!("Expected S3 provider"), - } - } - - #[test] - fn test_config_from_args_gcs() { - let config = - CloudBackupConfig::from_args(Some("gcs"), Some("gcs-bucket"), None, None, "", false) - .unwrap(); - - match config.provider { - Some(CloudProvider::GCS { bucket }) => { - assert_eq!(bucket, "gcs-bucket"); - } - _ => panic!("Expected GCS provider"), - } - } - - #[test] - fn test_config_from_args_missing_bucket() { - let result = CloudBackupConfig::from_args(Some("s3"), None, None, None, "", false); - assert!(result.is_err()); - } - - #[test] - fn test_config_disabled_by_default() { - let config = CloudBackupConfig::from_args(None, None, None, None, "", false).unwrap(); - - assert!(!config.enabled); - assert!(config.provider.is_none()); - } -} +#[path = "../tests/unit/persistence.rs"] +mod tests; diff --git a/src/projects.rs b/src/projects.rs index c94e05f..752fe06 100644 --- a/src/projects.rs +++ b/src/projects.rs @@ -2,6 +2,7 @@ use crate::config::TuningConfig; use crate::cuebridge::{CueBridgeArtifactSummary, CueBridgeArtifacts, CueBridgeAliasExpansion}; use crate::engine::CueMapEngine; use crate::normalization::NormalizationConfig; +use crate::semantic::SemanticEncoder; use crate::structures::{LexiconStats, MainStats, MemoryId}; use crate::taxonomy::Taxonomy; use ahash::RandomState; @@ -50,18 +51,48 @@ impl ProjectContext { tuning: Arc<TuningConfig>, config: crate::config::ServerConfig, project_id: String, + ) -> Self { + let mut context = Self::new_with_encoder( + normalization, + taxonomy, + tuning, + config, + project_id, + None, + ); + if let Err(error) = context.main.configure_semantic_encoder() { + tracing::warn!( + project_id = %context.main.project_id, + error = %error, + "Semantic encoder unavailable; continuing without automatic text embeddings" + ); + } + context + } + + pub fn new_with_encoder( + normalization: NormalizationConfig, + taxonomy: Taxonomy, + tuning: Arc<TuningConfig>, + config: crate::config::ServerConfig, + project_id: String, + semantic_encoder: Option<Arc<dyn SemanticEncoder>>, ) -> Self { let mut main = CueMapEngine::with_tuning(tuning.as_ref().clone()); main.config = config.clone(); main.project_id = project_id.clone(); + main.set_semantic_config(config.semantic.clone()); + main.set_semantic_encoder(semantic_encoder); let mut aliases = CueMapEngine::with_tuning(tuning.as_ref().clone()); aliases.config = config.clone(); + aliases.config.semantic = crate::semantic::SemanticConfig::default(); aliases.config.server.store_content_on_disk = false; // Disable for aliases (tiny memories, arbitrary IDs) aliases.project_id = project_id.clone(); let mut lexicon = CueMapEngine::with_tuning(tuning.as_ref().clone()); lexicon.config = config.clone(); + lexicon.config.semantic = crate::semantic::SemanticConfig::default(); lexicon.config.server.store_content_on_disk = false; // Disable for lexicon (tiny memories, arbitrary IDs) lexicon.project_id = project_id.clone(); diff --git a/src/semantic.rs b/src/semantic.rs new file mode 100644 index 0000000..c5c2b83 --- /dev/null +++ b/src/semantic.rs @@ -0,0 +1,838 @@ +//! Semantic retrieval primitives. +//! +//! This module intentionally contains no language ontology and no text +//! classification rules. It operates on vectors supplied by an embedding +//! provider. The default build bundles a local MiniLM-L3 encoder, while +//! callers can disable automatic encoding or compile without the encoder. + +use half::f16; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +const DEFAULT_RANDOM_SEED: u64 = 0x4355_454d_4150_7637; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SemanticProfile { + Off, + Edge, + Balanced, + Quality, +} + +impl Default for SemanticProfile { + fn default() -> Self { + Self::Quality + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SemanticStorage { + Auto, + F32, + F16, + Int8, +} + +impl Default for SemanticStorage { + fn default() -> Self { + Self::Auto + } +} + +impl SemanticStorage { + pub fn byte_width(self) -> usize { + match self { + Self::Auto | Self::F32 => 4, + Self::F16 => 2, + Self::Int8 => 1, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SemanticIndexMode { + Auto, + Exact, + Ann, +} + +/// Selects which query signal is allowed to drive recall. +/// +/// `Hybrid` keeps lexical candidate discovery bounded by the requested limit +/// and uses the local embedding only to rerank those candidates. `Semantic` +/// accepts query text so a configured local encoder can embed it, then uses +/// semantic candidate discovery without lexical query cues. +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SemanticRecallMode { + Lexical, + Semantic, + Hybrid, +} + +impl Default for SemanticRecallMode { + fn default() -> Self { + Self::Hybrid + } +} + +/// Local text encoder interface. Implementations must be deterministic for a +/// fixed model asset and must not perform network I/O. +pub trait SemanticEncoder: Send + Sync { + fn dimensions(&self) -> usize; + fn encode(&self, text: &str) -> Result<Vec<f32>, String>; +} + +impl Default for SemanticIndexMode { + fn default() -> Self { + Self::Auto + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] +pub struct SemanticConfig { + /// Selects a coherent device-oriented set of semantic defaults. + pub profile: SemanticProfile, + /// Enables vector indexing and vector candidate discovery. + pub enabled: bool, + /// Zero means infer the dimension from the first vector. + pub dimensions: usize, + /// Optional identifier for the local embedding provider that produced the + /// vectors. CueMap does not load or call the provider. + pub model_id: String, + /// Optional provider/model compatibility marker. + pub model_version: String, + /// Enables local text-to-vector inference inside the Rust process. + pub encoder_enabled: bool, + /// Local ONNX model path. Empty selects the bundled MiniLM asset when the + /// semantic encoder feature is present. No runtime download is attempted. + pub model_path: String, + /// Local Hugging Face tokenizer JSON path. Empty selects the bundled + /// MiniLM tokenizer when the semantic encoder feature is present. + pub tokenizer_path: String, + /// Maximum tokenizer sequence length. Both bundled L3 variants use 128 + /// word pieces by default. + pub max_tokens: usize, + /// ONNX Runtime intra-op threads. Zero uses the runtime default. + pub encoder_threads: usize, + /// Enables the CoreML execution provider on Apple targets when the + /// bundled ONNX Runtime was built with CoreML support. Non-Apple targets + /// ignore this setting. + pub coreml_enabled: bool, + /// Representation used for vectors persisted with memories. + pub storage: SemanticStorage, + /// Selects exact candidate discovery, ANN candidate discovery, or an + /// automatic choice based on index size. + pub index: SemanticIndexMode, + /// Approximate memory budget for compact vectors and ANN bookkeeping. + /// Zero means no budget is imposed. + pub max_memory_mb: usize, + /// Number of independent random-projection hash tables. + pub ann_tables: usize, + /// Number of hyperplanes per table. Must be <= 63. + pub ann_bits: usize, + /// Number of one-bit neighboring buckets to probe in addition to the + /// exact bucket for each table. + pub ann_probes: usize, + /// Maximum number of vector candidates passed into lexical/rerank merge. + pub candidate_limit: usize, + /// For small indexes, exact cosine scan avoids poor recall while the + /// index is still warming up. Larger indexes use the ANN buckets. + pub exact_fallback_max: usize, + /// Multiplier applied to cosine similarity before merge/reranking. + pub semantic_score_multiplier: f64, + /// Weight of normalized semantic similarity in bounded hybrid reranking. + /// Zero preserves lexical ordering; one uses semantic ordering among the + /// existing lexical candidates. This does not affect semantic candidate + /// discovery mode. + pub semantic_rerank_weight: f64, + /// Maximum lexical candidate window passed into bounded hybrid semantic + /// reranking. The final request limit is applied after this window is + /// reranked. Zero uses the default window. + pub semantic_rerank_candidate_limit: usize, + /// Number of query text embeddings retained in the bounded in-memory + /// cache. Zero disables query embedding caching. + pub query_embedding_cache_capacity: usize, + /// Enables confidence-weighted intent compatibility during hybrid + /// reranking. + pub intent_rerank_enabled: bool, + /// Maximum fraction of the lexical score range contributed by a matching + /// intent. Unrelated intent pairs contribute no positive compatibility. + pub intent_rerank_weight: f64, + /// Maximum fraction of the lexical score range used to penalize a + /// confidently non-memory candidate. + pub intent_no_recall_penalty: f64, + /// Absolute cap on the score delta contributed by intent reranking. This + /// keeps even a strong exact-intent match from overwhelming lexical and + /// semantic evidence. + pub intent_rerank_max_delta: f64, + /// Enables the optional linear reranker. + pub reranker_enabled: bool, + /// Bias for the tiny binary-shippable linear reranker. + pub reranker_bias: f32, + /// Feature weights for the tiny linear reranker. Empty means neutral. + pub reranker_weights: Vec<f32>, + /// Scales the reranker contribution before adding it to the base score. + pub reranker_scale: f64, +} + +impl Default for SemanticConfig { + fn default() -> Self { + let encoder_enabled = cfg!(feature = "semantic-encoder"); + Self { + profile: SemanticProfile::Quality, + enabled: encoder_enabled, + dimensions: 0, + model_id: "all-MiniLM-L3-v2".to_string(), + model_version: "bundled-qint8-minilm-l3".to_string(), + encoder_enabled, + model_path: String::new(), + tokenizer_path: String::new(), + max_tokens: 128, + encoder_threads: 0, + coreml_enabled: true, + storage: SemanticStorage::Auto, + index: SemanticIndexMode::Auto, + max_memory_mb: 0, + ann_tables: 0, + ann_bits: 0, + ann_probes: 0, + candidate_limit: 0, + exact_fallback_max: 0, + semantic_score_multiplier: 100.0, + semantic_rerank_weight: 0.60, + semantic_rerank_candidate_limit: 200, + query_embedding_cache_capacity: 256, + intent_rerank_enabled: true, + intent_rerank_weight: 0.65, + intent_no_recall_penalty: 0.20, + intent_rerank_max_delta: 64.0, + reranker_enabled: false, + reranker_bias: 0.0, + reranker_weights: Vec::new(), + reranker_scale: 1.0, + } + } +} + +impl SemanticConfig { + /// Resolves a profile and fills zero/auto fields with stable defaults. + /// Explicit non-zero values remain authoritative, which lets a caller + /// tune dimensions, ANN fanout, and memory limits independently. + pub fn resolved(&self) -> Self { + let (profile_dimensions, profile_storage, profile_max_memory_mb, profile_tables, + profile_bits, profile_probes, profile_candidates, profile_exact) = match self.profile { + SemanticProfile::Off => (0, SemanticStorage::F32, 0, 4, 12, 2, 256, 4096), + // Both bundled MiniLM variants emit 384-dimensional vectors. Edge + // saves memory with the q4 L3 model and compact storage rather + // than projecting the model output to an incompatible dimension. + SemanticProfile::Edge => (384, SemanticStorage::Int8, 32, 4, 10, 2, 128, 4096), + SemanticProfile::Balanced => { + (384, SemanticStorage::Int8, 64, 4, 12, 2, 256, 4096) + } + SemanticProfile::Quality => { + (384, SemanticStorage::Int8, 256, 6, 14, 3, 512, 4096) + } + }; + + let mut resolved = self.clone(); + if resolved.profile == SemanticProfile::Edge + && resolved.model_path.trim().is_empty() + && resolved.model_id == "all-MiniLM-L3-v2" + && resolved.model_version == "bundled-qint8-minilm-l3" + { + resolved.model_version = "bundled-q4-minilm-l3".to_string(); + } + if resolved.profile == SemanticProfile::Edge + && (resolved.max_tokens == 0 || resolved.max_tokens == 256) + { + resolved.max_tokens = 128; + } + if self.profile != SemanticProfile::Off || self.encoder_enabled { + resolved.enabled = true; + } + if resolved.dimensions == 0 { + resolved.dimensions = profile_dimensions; + } + if resolved.storage == SemanticStorage::Auto { + resolved.storage = profile_storage; + } + if resolved.max_memory_mb == 0 { + resolved.max_memory_mb = profile_max_memory_mb; + } + if resolved.ann_tables == 0 { + resolved.ann_tables = profile_tables; + } + if resolved.ann_bits == 0 { + resolved.ann_bits = profile_bits; + } + if resolved.ann_probes == 0 { + resolved.ann_probes = profile_probes; + } + if resolved.candidate_limit == 0 { + resolved.candidate_limit = profile_candidates; + } + if resolved.exact_fallback_max == 0 { + resolved.exact_fallback_max = profile_exact; + } + if resolved.max_tokens == 0 { + resolved.max_tokens = if resolved.profile == SemanticProfile::Edge { + 128 + } else { + 256 + }; + } + resolved.ann_tables = resolved.ann_tables.max(1); + resolved.ann_bits = resolved.ann_bits.clamp(1, 63); + resolved.candidate_limit = resolved.candidate_limit.max(1); + resolved.semantic_score_multiplier = if resolved.semantic_score_multiplier.is_finite() { + resolved.semantic_score_multiplier + } else { + 100.0 + }; + resolved.semantic_rerank_weight = if resolved.semantic_rerank_weight.is_finite() { + resolved.semantic_rerank_weight.clamp(0.0, 1.0) + } else { + 0.60 + }; + if resolved.semantic_rerank_candidate_limit == 0 { + resolved.semantic_rerank_candidate_limit = 200; + } + resolved.intent_rerank_weight = if resolved.intent_rerank_weight.is_finite() { + resolved.intent_rerank_weight.clamp(0.0, 1.0) + } else { + 0.65 + }; + resolved.intent_no_recall_penalty = if resolved.intent_no_recall_penalty.is_finite() { + resolved.intent_no_recall_penalty.clamp(0.0, 1.0) + } else { + 0.20 + }; + resolved.intent_rerank_max_delta = if resolved.intent_rerank_max_delta.is_finite() { + resolved.intent_rerank_max_delta.max(0.0) + } else { + 64.0 + }; + resolved + } + + pub fn estimated_vector_bytes(&self) -> usize { + self.resolved().storage.byte_width() + } + + /// Conservative resident-memory estimate for compact vectors, signatures, + /// bucket IDs, and projection planes. It is intentionally a budget guard, + /// not a byte-perfect allocator report. + pub fn estimated_memory_bytes(&self, memory_count: usize) -> usize { + let config = self.resolved(); + estimate_memory_bytes(&config, config.dimensions, memory_count) + } + + pub fn estimated_memory_bytes_for_dimensions( + &self, + dimensions: usize, + memory_count: usize, + ) -> usize { + let config = self.resolved(); + estimate_memory_bytes(&config, dimensions, memory_count) + } + + pub fn within_memory_budget_for_dimensions( + &self, + dimensions: usize, + memory_count: usize, + ) -> bool { + let config = self.resolved(); + config.max_memory_mb == 0 + || estimate_memory_bytes(&config, dimensions, memory_count) + <= config.max_memory_mb.saturating_mul(1024 * 1024) + } + + pub fn within_memory_budget(&self, memory_count: usize) -> bool { + let config = self.resolved(); + config.max_memory_mb == 0 + || estimate_memory_bytes(&config, config.dimensions, memory_count) + <= config.max_memory_mb.saturating_mul(1024 * 1024) + } +} + +pub fn load_configured_encoder( + config: &SemanticConfig, +) -> Result<Option<Arc<dyn SemanticEncoder>>, String> { + let config = config.resolved(); + if !config.encoder_enabled { + return Ok(None); + } + + #[cfg(feature = "semantic-encoder")] + { + let encoder = crate::semantic_encoder::OnnxSemanticEncoder::from_config(&config)?; + return Ok(Some(Arc::new(encoder))); + } + + #[cfg(not(feature = "semantic-encoder"))] + { + Err("semantic encoder is configured but this binary was built without the 'semantic-encoder' feature".to_string()) + } +} + +fn estimate_memory_bytes( + config: &SemanticConfig, + dimensions: usize, + memory_count: usize, +) -> usize { + if memory_count == 0 || dimensions == 0 || !config.enabled { + return 0; + } + let vector_bytes = dimensions.saturating_mul(config.storage.byte_width()); + let per_memory_index_bytes = config + .ann_tables + .saturating_mul(std::mem::size_of::<u64>() + std::mem::size_of::<u32>()) + .saturating_add(32); + let plane_bytes = config + .ann_tables + .saturating_mul(config.ann_bits) + .saturating_mul(dimensions) + .saturating_mul(std::mem::size_of::<f32>()); + memory_count + .saturating_mul(vector_bytes.saturating_add(per_memory_index_bytes)) + .saturating_add(plane_bytes) +} + +/// Compact persisted representation of a normalized embedding. The public +/// ingest API still accepts `Vec<f32>`; this is the representation retained by +/// memories and used to rebuild the ANN buckets after a snapshot restore. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum StoredSemanticVector { + F32(Vec<f32>), + F16(Vec<u16>), + Int8 { values: Vec<i8>, scale: f32 }, +} + +impl StoredSemanticVector { + pub fn from_f32(vector: &[f32], storage: SemanticStorage) -> Result<Self, String> { + let normalized = normalize_vector(vector)?; + let storage = if storage == SemanticStorage::Auto { + SemanticStorage::F32 + } else { + storage + }; + Ok(match storage { + SemanticStorage::F32 => Self::F32(normalized), + SemanticStorage::F16 => Self::F16( + normalized + .into_iter() + .map(|value| f16::from_f32(value).to_bits()) + .collect(), + ), + SemanticStorage::Int8 => { + let scale = 1.0 / 127.0; + Self::Int8 { + values: normalized + .into_iter() + .map(|value| (value / scale).round().clamp(-127.0, 127.0) as i8) + .collect(), + scale, + } + } + SemanticStorage::Auto => unreachable!("auto storage is resolved above"), + }) + } + + pub fn dimensions(&self) -> usize { + match self { + Self::F32(values) => values.len(), + Self::F16(values) => values.len(), + Self::Int8 { values, .. } => values.len(), + } + } + + pub fn storage(&self) -> SemanticStorage { + match self { + Self::F32(_) => SemanticStorage::F32, + Self::F16(_) => SemanticStorage::F16, + Self::Int8 { .. } => SemanticStorage::Int8, + } + } + + pub fn estimated_bytes(&self) -> usize { + match self { + Self::F32(values) => values.len() * std::mem::size_of::<f32>(), + Self::F16(values) => values.len() * std::mem::size_of::<u16>(), + Self::Int8 { values, .. } => values.len() * std::mem::size_of::<i8>() + size_of_f32(), + } + } + + pub fn normalized_values(&self) -> Vec<f32> { + match self { + Self::F32(values) => values.clone(), + Self::F16(values) => values + .iter() + .map(|value| f16::from_bits(*value).to_f32()) + .collect(), + Self::Int8 { values, scale } => values + .iter() + .map(|value| *value as f32 * *scale) + .collect(), + } + } + + pub fn normalized_query(query: &[f32]) -> Result<Vec<f32>, String> { + normalize_vector(query) + } + + pub fn cosine_similarity(&self, query: &[f32]) -> Result<f32, String> { + let query = Self::normalized_query(query)?; + self.cosine_similarity_normalized(&query) + } + + pub fn cosine_similarity_normalized(&self, query: &[f32]) -> Result<f32, String> { + if query.len() != self.dimensions() { + return Err(format!( + "semantic query dimension mismatch: expected {}, received {}", + self.dimensions(), + query.len() + )); + } + Ok(match self { + Self::F32(values) => dot(values, query), + Self::F16(values) => values + .iter() + .zip(query) + .map(|(value, query_value)| f16::from_bits(*value).to_f32() * query_value) + .sum(), + Self::Int8 { values, scale } => values + .iter() + .zip(query) + .map(|(value, query_value)| *value as f32 * *scale * query_value) + .sum(), + }) + } +} + +fn size_of_f32() -> usize { + std::mem::size_of::<f32>() +} + +#[derive(Clone, Debug)] +struct ProjectionTable { + hyperplanes: Vec<Vec<f32>>, + buckets: HashMap<u64, Vec<u32>>, +} + +/// In-memory approximate nearest-neighbor index based on random-hyperplane +/// locality-sensitive hashing. It stores only IDs, signatures, and buckets; +/// the compact vector remains owned by the corresponding memory. +#[derive(Clone, Debug)] +pub struct SemanticIndex { + config: SemanticConfig, + dimensions: Option<usize>, + tables: Vec<ProjectionTable>, + indexed_ids: HashSet<u32>, + signatures: HashMap<u32, Vec<u64>>, +} + +impl SemanticIndex { + pub fn new(config: SemanticConfig) -> Self { + Self { + config: config.resolved(), + dimensions: None, + tables: Vec::new(), + indexed_ids: HashSet::new(), + signatures: HashMap::new(), + } + } + + pub fn config(&self) -> &SemanticConfig { + &self.config + } + + pub fn len(&self) -> usize { + self.indexed_ids.len() + } + + pub fn is_empty(&self) -> bool { + self.indexed_ids.is_empty() + } + + pub fn dimensions(&self) -> Option<usize> { + self.dimensions + } + + pub fn clear(&mut self) { + self.dimensions = None; + self.tables.clear(); + self.indexed_ids.clear(); + self.signatures.clear(); + } + + pub fn rebuild<I>(&mut self, vectors: I) + where + I: IntoIterator<Item = (u32, StoredSemanticVector)>, + { + self.clear(); + for (memory_id, vector) in vectors { + if let Err(error) = self.insert(memory_id, &vector) { + tracing::debug!(memory_id, error = %error, "Skipping semantic vector during index rebuild"); + } + } + } + + pub fn insert( + &mut self, + memory_id: u32, + vector: &StoredSemanticVector, + ) -> Result<(), String> { + if !self.config.enabled { + return Ok(()); + } + let normalized = vector.normalized_values(); + let dimensions = normalized.len(); + if dimensions == 0 { + return Err("semantic vectors cannot be empty".to_string()); + } + if let Some(expected) = self.dimensions { + if expected != dimensions { + return Err(format!( + "semantic vector dimension mismatch: expected {}, received {}", + expected, dimensions + )); + } + } else { + let configured = self.config.dimensions; + if configured != 0 && configured != dimensions { + return Err(format!( + "semantic vector dimension mismatch: configured {}, received {}", + configured, dimensions + )); + } + } + + let projected_count = if self.indexed_ids.contains(&memory_id) { + self.indexed_ids.len() + } else { + self.indexed_ids.len().saturating_add(1) + }; + if !self + .config + .within_memory_budget_for_dimensions(dimensions, projected_count) + { + return Err("semantic index memory budget exceeded".to_string()); + } + + if self.dimensions.is_none() { + self.initialize_tables(dimensions); + } + + self.remove(memory_id); + let signatures = self + .tables + .iter_mut() + .map(|table| { + let signature = signature(&table.hyperplanes, &normalized); + table.buckets.entry(signature).or_default().push(memory_id); + signature + }) + .collect::<Vec<_>>(); + self.indexed_ids.insert(memory_id); + self.signatures.insert(memory_id, signatures); + Ok(()) + } + + pub fn remove(&mut self, memory_id: u32) -> bool { + let existed = self.indexed_ids.remove(&memory_id); + if !existed { + return false; + } + if let Some(signatures) = self.signatures.remove(&memory_id) { + for (table, table_signature) in self.tables.iter_mut().zip(signatures) { + if let Some(ids) = table.buckets.get_mut(&table_signature) { + ids.retain(|id| *id != memory_id); + if ids.is_empty() { + table.buckets.remove(&table_signature); + } + } + } + } + true + } + + /// Returns IDs that should be scored against the compact vectors stored by + /// the engine. Exact mode is still bounded by `limit`; ANN mode is bounded + /// after bucket union so query work remains predictable. + pub fn query_candidate_ids(&self, query: &[f32], limit: usize) -> Result<Vec<u32>, String> { + if !self.config.enabled || limit == 0 || self.indexed_ids.is_empty() { + return Ok(Vec::new()); + } + let query = normalize_vector(query)?; + if self.dimensions != Some(query.len()) { + return Err(format!( + "semantic query dimension mismatch: expected {:?}, received {}", + self.dimensions, + query.len() + )); + } + + let exact = self.config.index == SemanticIndexMode::Exact + || (self.config.index == SemanticIndexMode::Auto + && self.indexed_ids.len() <= self.config.exact_fallback_max); + let mut candidate_ids = HashSet::new(); + if exact { + candidate_ids.extend(self.indexed_ids.iter().copied()); + } else { + for table in &self.tables { + let exact_signature = signature(&table.hyperplanes, &query); + for bucket_key in probe_keys( + exact_signature, + self.config.ann_bits, + self.config.ann_probes, + ) { + if let Some(ids) = table.buckets.get(&bucket_key) { + candidate_ids.extend(ids.iter().copied()); + } + } + } + // Empty/very sparse ANN buckets should not turn semantic recall + // into a silent hard miss. Bound the emergency scan and leave the + // normal large-index path approximate. + if candidate_ids.is_empty() { + candidate_ids.extend(self.indexed_ids.iter().copied().take(limit)); + } + } + + let mut candidate_ids = candidate_ids.into_iter().collect::<Vec<_>>(); + candidate_ids.sort_unstable(); + candidate_ids.truncate(limit.min(self.config.candidate_limit.max(1))); + Ok(candidate_ids) + } + + fn initialize_tables(&mut self, dimensions: usize) { + self.dimensions = Some(dimensions); + let table_count = self.config.ann_tables.max(1); + let bit_count = self.config.ann_bits.clamp(1, 63); + let mut rng = SplitMix64::new(DEFAULT_RANDOM_SEED ^ dimensions as u64); + self.tables = (0..table_count) + .map(|_| { + let hyperplanes = (0..bit_count) + .map(|_| { + let mut plane = (0..dimensions) + .map(|_| rng.next_f32() * 2.0 - 1.0) + .collect::<Vec<_>>(); + let norm = plane.iter().map(|value| value * value).sum::<f32>().sqrt(); + if norm > f32::EPSILON { + for value in &mut plane { + *value /= norm; + } + } + plane + }) + .collect(); + ProjectionTable { + hyperplanes, + buckets: HashMap::new(), + } + }) + .collect(); + } +} + +fn normalize_vector(vector: &[f32]) -> Result<Vec<f32>, String> { + if vector.is_empty() { + return Err("semantic vectors cannot be empty".to_string()); + } + if vector.iter().any(|value| !value.is_finite()) { + return Err("semantic vectors must contain only finite values".to_string()); + } + let norm = vector.iter().map(|value| value * value).sum::<f32>().sqrt(); + if norm <= f32::EPSILON { + return Err("semantic vectors cannot have zero magnitude".to_string()); + } + Ok(vector.iter().map(|value| value / norm).collect()) +} + +fn dot(left: &[f32], right: &[f32]) -> f32 { + left.iter().zip(right).map(|(a, b)| a * b).sum() +} + +fn signature(hyperplanes: &[Vec<f32>], vector: &[f32]) -> u64 { + hyperplanes + .iter() + .enumerate() + .fold(0u64, |value, (bit, plane)| { + if dot(plane, vector) >= 0.0 { + value | (1u64 << bit) + } else { + value + } + }) +} + +fn probe_keys(signature: u64, bits: usize, probes: usize) -> Vec<u64> { + let bits = bits.min(63); + let mut keys = Vec::with_capacity(probes.saturating_add(1).min(bits + 1)); + keys.push(signature); + for bit in 0..bits { + if keys.len() >= probes.saturating_add(1) { + break; + } + keys.push(signature ^ (1u64 << bit)); + } + keys +} + +#[derive(Clone, Debug)] +struct SplitMix64(u64); + +impl SplitMix64 { + fn new(seed: u64) -> Self { + Self(seed) + } + + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + fn next_f32(&mut self) -> f32 { + (self.next_u64() as f64 / u64::MAX as f64) as f32 + } +} + +/// A tiny linear reranker model. Its weights are deliberately data, not +/// ontology rules, so a trained model can be shipped as a handful of floats. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LinearReranker { + pub bias: f32, + pub weights: Vec<f32>, +} + +impl LinearReranker { + pub fn from_config(config: &SemanticConfig) -> Self { + Self { + bias: config.reranker_bias, + weights: config.reranker_weights.clone(), + } + } + + pub fn score(&self, features: &[f32]) -> f32 { + self.bias + + self + .weights + .iter() + .zip(features) + .map(|(weight, feature)| weight * feature) + .sum::<f32>() + } +} + +#[cfg(test)] +#[path = "../tests/unit/semantic.rs"] +mod tests; diff --git a/src/semantic_encoder.rs b/src/semantic_encoder.rs new file mode 100644 index 0000000..7b2189a --- /dev/null +++ b/src/semantic_encoder.rs @@ -0,0 +1,207 @@ +//! Local ONNX encoder for the bundled qint8 MiniLM-L3 default and q4 +//! MiniLM-L3 edge asset. +//! +//! This module is compiled with the default `semantic-encoder` feature. It +//! never downloads models: empty paths select the assets embedded in the +//! binary, while non-empty paths provide an explicit local override. + +use crate::semantic::{SemanticConfig, SemanticEncoder}; +use ort::{session::Session, value::Tensor}; +use std::sync::Mutex; +use tokenizers::{ + tokenizer::TruncationDirection, + Tokenizer, TruncationParams, TruncationStrategy, +}; + +const DEFAULT_DIMENSIONS: usize = 384; +const EMBEDDED_L3_MODEL: &[u8] = + include_bytes!("../assets/all-MiniLM-L3-v2/model_qint8_arm64.onnx"); +const EMBEDDED_L3_Q4_MODEL: &[u8] = + include_bytes!("../assets/all-MiniLM-L3-v2/model_int4.onnx"); +const EMBEDDED_L3_TOKENIZER: &[u8] = + include_bytes!("../assets/all-MiniLM-L3-v2/tokenizer.json"); + +pub struct OnnxSemanticEncoder { + session: Mutex<Session>, + tokenizer: Tokenizer, + dimensions: usize, + max_tokens: usize, +} + +impl OnnxSemanticEncoder { + pub fn from_config(config: &SemanticConfig) -> Result<Self, String> { + let use_edge_model = config.model_path.trim().is_empty() + && config.model_version == "bundled-q4-minilm-l3"; + let mut tokenizer = if config.tokenizer_path.trim().is_empty() { + Tokenizer::from_bytes(EMBEDDED_L3_TOKENIZER) + } else { + Tokenizer::from_file(&config.tokenizer_path) + } + .map_err(|error| format!("failed to load semantic tokenizer: {error}"))?; + tokenizer + .with_truncation(Some(TruncationParams { + max_length: config.max_tokens.max(1), + strategy: TruncationStrategy::LongestFirst, + stride: 0, + direction: TruncationDirection::Right, + })) + .map_err(|error| format!("failed to configure semantic tokenizer truncation: {error}"))?; + let mut builder = Session::builder() + .map_err(|error| format!("failed to create ONNX session builder: {error}"))?; + if config.encoder_threads > 0 { + builder = builder + .with_intra_threads(config.encoder_threads) + .map_err(|error| format!("failed to configure ONNX threads: {error}"))?; + } + + #[cfg(any(target_os = "ios", target_os = "macos"))] + if config.coreml_enabled { + builder = builder + .with_execution_providers([ort::ep::CoreML::default() + .with_compute_units(ort::ep::coreml::ComputeUnits::CPUAndNeuralEngine) + .build()]) + .map_err(|error| format!("failed to configure CoreML execution provider: {error}"))?; + } + + let session = if config.model_path.trim().is_empty() { + builder + .commit_from_memory(if use_edge_model { + EMBEDDED_L3_Q4_MODEL + } else { + EMBEDDED_L3_MODEL + }) + .map_err(|error| format!("failed to load bundled semantic ONNX model: {error}"))? + } else { + builder + .commit_from_file(&config.model_path) + .map_err(|error| format!("failed to load semantic ONNX model: {error}"))? + }; + + Ok(Self { + session: Mutex::new(session), + tokenizer, + dimensions: if config.dimensions == 0 { + DEFAULT_DIMENSIONS + } else { + config.dimensions + }, + max_tokens: config.max_tokens.max(1), + }) + } +} + +impl SemanticEncoder for OnnxSemanticEncoder { + fn dimensions(&self) -> usize { + self.dimensions + } + + fn encode(&self, text: &str) -> Result<Vec<f32>, String> { + let mut encoding = self + .tokenizer + .encode(text, true) + .map_err(|error| format!("semantic tokenization failed: {error}"))?; + if encoding.len() > self.max_tokens { + encoding.truncate(self.max_tokens, 0, TruncationDirection::Right); + } + if encoding.is_empty() { + return Err("semantic tokenizer produced an empty sequence".to_string()); + } + + let sequence_length = encoding.len(); + let input_ids = encoding + .get_ids() + .iter() + .map(|value| *value as i64) + .collect::<Vec<_>>(); + let attention_mask = encoding + .get_attention_mask() + .iter() + .map(|value| *value as i64) + .collect::<Vec<_>>(); + let token_type_ids = encoding + .get_type_ids() + .iter() + .map(|value| *value as i64) + .collect::<Vec<_>>(); + + let input_ids = Tensor::from_array(([1usize, sequence_length], input_ids)) + .map_err(|error| format!("failed to construct input_ids tensor: {error}"))?; + let attention_mask = Tensor::from_array(([1usize, sequence_length], attention_mask)) + .map_err(|error| format!("failed to construct attention_mask tensor: {error}"))?; + let token_type_ids = Tensor::from_array(([1usize, sequence_length], token_type_ids)) + .map_err(|error| format!("failed to construct token_type_ids tensor: {error}"))?; + + let mut session = self + .session + .lock() + .map_err(|_| "semantic ONNX session lock was poisoned".to_string())?; + let outputs = session + .run(ort::inputs! { + "input_ids" => input_ids, + "attention_mask" => attention_mask, + "token_type_ids" => token_type_ids, + }) + .map_err(|error| format!("semantic ONNX inference failed: {error}"))?; + let (shape, hidden) = outputs[0] + .try_extract_tensor::<f32>() + .map_err(|error| format!("semantic ONNX output was not f32: {error}"))?; + + if shape.len() != 3 + || shape[0] != 1 + || shape[1] != sequence_length as i64 + || shape[2] != self.dimensions as i64 + { + return Err(format!( + "semantic ONNX output shape is incompatible: expected [1, {}, {}], received {:?}", + sequence_length, self.dimensions, shape + )); + } + + let expected_values = sequence_length.saturating_mul(self.dimensions); + if hidden.len() < expected_values { + return Err(format!( + "semantic ONNX output is too small: expected at least {}, received {}", + expected_values, + hidden.len() + )); + } + + // Sentence-Transformers' mean pooling: average token embeddings using + // the attention mask, then normalize for cosine retrieval. + let mut pooled = vec![0.0f32; self.dimensions]; + let mut token_count = 0.0f32; + for token_index in 0..sequence_length { + let mask = encoding.get_attention_mask()[token_index]; + if mask == 0 { + continue; + } + token_count += 1.0; + let offset = token_index * self.dimensions; + for dimension in 0..self.dimensions { + pooled[dimension] += hidden[offset + dimension]; + } + } + if token_count <= 0.0 { + return Err("semantic attention mask contains no active tokens".to_string()); + } + for value in &mut pooled { + *value /= token_count; + } + let norm = pooled + .iter() + .map(|value| value * value) + .sum::<f32>() + .sqrt(); + if norm <= f32::EPSILON || !norm.is_finite() { + return Err("semantic encoder produced a zero or invalid vector".to_string()); + } + for value in &mut pooled { + *value /= norm; + } + Ok(pooled) + } +} + +#[cfg(test)] +#[path = "../tests/unit/semantic_encoder.rs"] +mod tests; diff --git a/src/structures.rs b/src/structures.rs index c553575..470cab3 100644 --- a/src/structures.rs +++ b/src/structures.rs @@ -4,6 +4,8 @@ use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; use crate::crypto::{self, EncryptionKey}; +use crate::intent::IntentClassification; +use crate::semantic::StoredSemanticVector; use ahash::RandomState; // ============================================================================= @@ -178,6 +180,15 @@ pub struct Memory<T> { pub disk_backed: bool, #[serde(default)] pub scoring_features: MemoryScoringFeatures, + /// Optional externally precomputed semantic representation. It is kept + /// alongside the memory so the in-memory ANN index can be rebuilt after a + /// snapshot restore. + #[serde(default)] + pub semantic_vector: Option<StoredSemanticVector>, + /// Optional versioned intent classification used by CueKey gating and + /// confidence-weighted hybrid reranking. + #[serde(default)] + pub intent_classification: Option<IntentClassification>, /// Type-specific stats payload pub stats: T, } @@ -203,6 +214,8 @@ impl<T: Default> Memory<T> { metadata: metadata.unwrap_or_default(), disk_backed: false, scoring_features: MemoryScoringFeatures::default(), + semantic_vector: None, + intent_classification: None, stats: T::default(), } } diff --git a/tests/agent/chunker/mod.rs b/tests/agent/chunker/mod.rs index 269872f..45c2404 100644 --- a/tests/agent/chunker/mod.rs +++ b/tests/agent/chunker/mod.rs @@ -1,4 +1,4 @@ -use cuemap::agent::chunker::{Chunker, SegmenterConfig}; +use cuemap::agent::chunker::{ChunkCategory, Chunker, SegmenterConfig}; use std::path::PathBuf; #[test] @@ -142,3 +142,193 @@ fn logical_block_chunking_splits_oversized_blocks_with_coarse_windows() { .any(|chunk| chunk.structural_cues.iter().any(|cue| cue == "type:logical_block_split"))); assert!(chunks.iter().all(|chunk| chunk.content.len() <= 100)); } + +#[test] +fn logical_block_code_uses_treesitter_cues() { + let content = "```python\nprint('keep code together')\n```"; + let config = SegmenterConfig { + window_size: 1, + overlap: 0, + min_chunk_chars: 20, + max_chunk_chars: 4000, + }; + + let chunks = Chunker::chunk_text_logical_blocks(content, &config); + + assert_eq!(chunks.len(), 1); + assert_eq!(chunks[0].category, ChunkCategory::Code); + assert!(chunks[0] + .structural_cues + .iter() + .any(|cue| cue == "lang:python")); + assert!(chunks[0] + .structural_cues + .iter() + .any(|cue| cue == "type:call")); + assert!(chunks[0] + .structural_cues + .iter() + .any(|cue| cue == "name:print")); +} + +#[test] +fn chunk_file_dispatches_supported_formats_and_attaches_parent_links() { + let cases = [ + ("module.py", "def greet(name):\n return name"), + ("module.rs", "fn greet() { println!(\"hi\"); }"), + ("module.ts", "function greet(name: string) { return name; }"), + ("module.js", "function greet(name) { return name; }"), + ("module.go", "package main\nfunc main() {}"), + ("page.html", "<main><h1>Title</h1><p>Body text.</p></main>"), + ("page.css", ".card { color: red; }"), + ("module.php", "<?php function greet() { return true; } ?>"), + ("Module.java", "public class Module { void run() {} }"), + ("notes.md", "# Heading\nA paragraph."), + ("rows.csv", "name,email\nAlice,alice@example.com"), + ("data.json", "{\"name\":\"Alice\"}"), + ("data.yaml", "name: Alice"), + ("data.xml", "<root id=\"r1\"><child/></root>"), + ("notes.txt", "A plain text note."), + ]; + + for (filename, content) in cases { + let chunks = Chunker::chunk_file(std::path::Path::new(filename), content); + assert!(!chunks.is_empty(), "expected chunks for {filename}"); + assert!(chunks + .iter() + .all(|chunk| chunk.structural_cues.iter().any(|cue| cue.starts_with("parent:")))); + assert!(chunks + .iter() + .all(|chunk| chunk.structural_cues.iter().any(|cue| cue.starts_with("chunk_idx:")))); + } + + assert_eq!( + Chunker::get_category_for_file(std::path::Path::new("unknown.bin")), + ChunkCategory::Prose + ); + assert_eq!( + Chunker::get_category_for_file(std::path::Path::new("api.json")), + ChunkCategory::Structured + ); +} + +#[test] +fn structured_chunkers_cover_arrays_api_specs_and_fallbacks() { + let array = Chunker::chunk_json("[1, {\"name\":\"Alice\"}]"); + assert_eq!(array.len(), 2); + assert!(array.iter().all(|chunk| chunk.context.starts_with("json_index:"))); + + let api_json = Chunker::chunk_json( + r#"{"swagger":"2.0","info":{"title":"Demo API"},"paths":{"/health":{"get":{"summary":"Health","operationId":"health","tags":["ops"]},"parameters":{}}}}"#, + ); + assert_eq!(api_json.len(), 1); + assert!(api_json[0].structural_cues.iter().any(|cue| cue == "method:GET")); + assert!(api_json[0].structural_cues.iter().any(|cue| cue == "tag:ops")); + + let api_yaml = Chunker::chunk_yaml( + "swagger: 2.0\ninfo:\n title: Demo\npaths:\n /health:\n get:\n summary: Health\n operationId: health\n tags: [ops]\n", + ); + assert_eq!(api_yaml.len(), 1); + assert!(api_yaml[0].structural_cues.iter().any(|cue| cue == "method:GET")); + + let xml = Chunker::chunk_file( + std::path::Path::new("data.xml"), + "<root id=\"r1\"><child name=\"x\"/></root>", + ); + assert!(xml[0].structural_cues.iter().any(|cue| cue == "id:r1")); + assert_eq!(Chunker::chunk_json("not json")[0].context, "text:full"); + assert_eq!(Chunker::chunk_yaml("not: [valid")[0].context, "text:full"); +} + +#[test] +fn social_exports_and_binary_fallbacks_are_routed_deterministically() { + let whatsapp = Chunker::chunk_file( + std::path::Path::new("whatsapp.txt"), + "[1/2/24, 10:00] Alice: Hello there\n[1/2/24, 10:01] Alice: image omitted\n[1/2/24, 10:02] Bob: Great news", + ); + assert_eq!(whatsapp.len(), 2); + assert!(whatsapp[0].structural_cues.iter().any(|cue| cue == "platform:whatsapp")); + + let instagram = Chunker::chunk_file( + std::path::Path::new("instagram.json"), + r#"[{"sender_name":"Alice","timestamp_ms":1700000000000,"content":"Hello","share":{"link":"https://example.com/post"}},{"sender_name":"Bob","timestamp_ms":0,"content":"Liked a message"}]"#, + ); + assert_eq!(instagram.len(), 1); + assert!(instagram[0].structural_cues.iter().any(|cue| cue == "has:shared_link")); + + let chrome = Chunker::chunk_file( + std::path::Path::new("chrome_history.json"), + r#"{"Browser History":[{"title":"CueMap","url":"https://cuemap.dev","time_usec":1700000000000000},{"title":"CueMap","url":"https://cuemap.dev","time_usec":1700000000000000}]}"#, + ); + assert_eq!(chrome.len(), 1); + assert!(chrome[0].structural_cues.iter().any(|cue| cue == "platform:chrome")); + + let youtube = Chunker::chunk_file( + std::path::Path::new("youtube-watch-history.html"), + r#"Watched <a href="https://www.youtube.com/watch?v=abc">Rust release</a> Jan 2, 2024 Searched for <a href="https://www.youtube.com/results">coverage</a> Jan 3, 2024"#, + ); + assert_eq!(youtube.len(), 2); + assert!(youtube.iter().all(|chunk| chunk.category == ChunkCategory::Conversation)); + + assert!(Chunker::chunk_binary_file(std::path::Path::new("missing.pdf")).is_empty()); + assert!(Chunker::chunk_binary_file(std::path::Path::new("missing.docx")).is_empty()); +} + +#[test] +fn text_window_and_structural_cue_helpers_cover_edge_cases() { + let config = SegmenterConfig { + window_size: 2, + overlap: 5, + min_chunk_chars: 1, + max_chunk_chars: 25, + }; + let chunks = Chunker::chunk_text_with_config( + "One sentence is here. Two sentence follows. Three sentence follows.", + &config, + ); + assert!(chunks.len() >= 2); + assert!(chunks.iter().all(|chunk| chunk.content.len() <= 25)); + assert!(Chunker::chunk_text_with_config("", &config).is_empty()); + + let mut chunks = vec![cuemap::agent::chunker::Chunk { + content: "x".to_string(), + start_line: 1, + end_line: 1, + context: "test".to_string(), + structural_cues: vec!["parent:old".to_string(), "chunk_idx:9".to_string()], + category: ChunkCategory::Prose, + }]; + Chunker::attach_parent_links(&mut chunks, "seed"); + Chunker::inherit_structural_cues( + &mut chunks, + &["source:test".to_string(), "source:test".to_string()], + ); + assert!(chunks[0].structural_cues.iter().any(|cue| cue == "source:test")); + assert_eq!( + chunks[0] + .structural_cues + .iter() + .filter(|cue| *cue == "source:test") + .count(), + 1 + ); + Chunker::inherit_structural_cues(&mut chunks, &[]); +} + +#[test] +fn article_content_links_are_scoped_and_resolved() { + let html = scraper::Html::parse_document( + r##"<html><body><nav><a href="/nav">Nav</a></nav><main><h1>Title</h1><p>Long enough article content for extraction.</p><a href="/docs">Docs</a><a href="https://example.com/full">Full</a><a href="#skip">Skip</a></main><footer><a href="/footer">Footer</a></footer></body></html>"##, + ); + let links = Chunker::extract_content_links( + &html, + &url::Url::parse("https://cuemap.dev/base/").unwrap(), + ); + assert_eq!( + links, + vec![ + "https://cuemap.dev/docs".to_string(), + "https://example.com/full".to_string(), + ] + ); +} diff --git a/tests/cuepacks/mod.rs b/tests/cuepacks/mod.rs deleted file mode 100644 index 54297c7..0000000 --- a/tests/cuepacks/mod.rs +++ /dev/null @@ -1,364 +0,0 @@ -use cuemap::cuepacks::{default_registry, CuePackRegistry}; -use cuemap::facets::{ - compile_query_intent_with_cuepacks, extract_memory_facets, extract_memory_facets_core, - extract_memory_facets_with_cuepacks, -}; -use std::fs; - -#[test] -fn bundled_memory_general_pack_loads_by_default() { - let infos = default_registry().infos(); - let pack = infos - .iter() - .find(|info| info.name == "memory-general") - .expect("memory-general CuePack should be bundled"); - - assert!(pack.enabled_by_default); - assert!(pack.memory_rules > 0); - assert!(pack.query_rules > 0); -} - -#[test] -fn core_extractor_stays_structural_while_default_pack_restores_domain_facets() { - let content = "I downloaded Google Maps for directions to the airport station by train."; - - let core = extract_memory_facets_core(content, None, &[]); - assert!(!core.contains(&"type:navigation".to_string())); - assert!(!core.contains(&"travel:route".to_string())); - assert!(!core.contains(&"travel:transit".to_string())); - - let default = extract_memory_facets(content, None, &[]); - assert!(default.contains(&"type:navigation".to_string())); - assert!(default.contains(&"travel:route".to_string())); - assert!(default.contains(&"travel:transit".to_string())); -} - -#[test] -fn cuepack_selection_can_disable_default_memory_facets() { - let content = "I started using a music streaming service for playlists."; - let off = vec!["off".to_string()]; - let facets = - extract_memory_facets_with_cuepacks(content, None, &[], default_registry(), Some(&off)); - - assert!(!facets.contains(&"media:music_streaming".to_string())); - assert!(!facets.contains(&"media:streaming".to_string())); -} - -#[test] -fn bundled_cuepack_extracts_standing_instruction_facets_and_trigger_cues() { - let content = - "Always provide fallback strategies when I ask about error handling in API services."; - let facets = extract_memory_facets(content, None, &[]); - - assert!(facets.contains(&"type:standing_instruction".to_string())); - assert!(facets.contains(&"instruction:conditional".to_string())); - assert!(facets.contains(&"instruction:always".to_string())); - assert!(facets.contains(&"instruction_trigger:api".to_string())); - assert!(facets.contains(&"instruction_action:fallback".to_string())); - - let off = vec!["off".to_string()]; - let core_only = - extract_memory_facets_with_cuepacks(content, None, &[], default_registry(), Some(&off)); - assert!(!core_only.contains(&"type:standing_instruction".to_string())); - assert!(!core_only.contains(&"instruction_trigger:api".to_string())); -} - -#[test] -fn bundled_cuepack_labels_advice_queries_as_instruction_applicable() { - let intent = compile_query_intent_with_cuepacks( - "What are some ways I can manage problems that come up when my API calls fail?", - None, - |_| false, - default_registry(), - None, - ); - - assert!(intent - .labels - .contains(&"instruction_applicable".to_string())); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.instruction_applicable")); -} - -#[test] -fn bundled_cuepack_labels_when_and_if_queries_as_instruction_applicable() { - for query in [ - "When building an application that talks to an API, what should I watch for?", - "If I'm creating a page layout, how should I structure the markup?", - ] { - let intent = - compile_query_intent_with_cuepacks(query, None, |_| false, default_registry(), None); - - assert!( - intent - .labels - .contains(&"instruction_applicable".to_string()), - "query should be instruction-applicable: {query}" - ); - assert!( - intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.instruction_applicable"), - "query should be labeled by memory-general instruction rule: {query}" - ); - } -} - -#[test] -fn bundled_cuepack_extracts_preference_facets_and_value_cues() { - let content = "I prefer geometric vector methods over purely trigonometric formulas for clarity, so can you explain how to use vector algebra to calculate geodesic length between two points on a sphere?"; - let facets = extract_memory_facets(content, None, &[]); - - assert!(facets.contains(&"type:preference".to_string())); - assert!(facets.contains(&"preference:explicit".to_string())); - assert!(facets.contains(&"preference_value:geometric".to_string())); - assert!(facets.contains(&"preference_value:vector".to_string())); - assert!(facets.contains(&"preference_contrast:trigonometric".to_string())); - - let off = vec!["off".to_string()]; - let core_only = - extract_memory_facets_with_cuepacks(content, None, &[], default_registry(), Some(&off)); - assert!(core_only.contains(&"type:preference".to_string())); - assert!(!core_only.contains(&"preference:explicit".to_string())); - assert!(!core_only.contains(&"preference_value:vector".to_string())); -} - -#[test] -fn bundled_cuepack_labels_task_queries_as_preference_applicable() { - let intent = compile_query_intent_with_cuepacks( - "Can you show me how to find the shortest path between two points on a sphere?", - None, - |_| false, - default_registry(), - None, - ); - - assert!(intent - .labels - .contains(&"preference_applicable".to_string())); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.preference_applicable")); -} - -#[test] -fn bundled_cuepack_labels_near_future_task_queries_as_preference_applicable() { - let intent = compile_query_intent_with_cuepacks( - "I'm about to start editing a long draft; what steps would you suggest?", - None, - |_| false, - default_registry(), - None, - ); - - assert!(intent - .labels - .contains(&"preference_applicable".to_string())); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.preference_applicable")); -} - -#[test] -fn bundled_cuepack_labels_summary_queries_as_multi_evidence() { - let intent = compile_query_intent_with_cuepacks( - "Can you provide a detailed summary of everything we covered about deployment planning?", - None, - |_| false, - default_registry(), - None, - ); - - assert!(intent - .labels - .contains(&"multi_evidence_summary".to_string())); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.multi_evidence_summary")); -} - -#[test] -fn bundled_cuepack_labels_overview_queries_as_multi_evidence() { - let intent = compile_query_intent_with_cuepacks( - "Can you give me a comprehensive overview of the key details from the project?", - None, - |_| false, - default_registry(), - None, - ); - - assert!(intent - .labels - .contains(&"multi_evidence_summary".to_string())); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.multi_evidence_summary")); -} - -#[test] -fn bundled_cuepack_labels_collection_queries_as_multi_evidence_collection() { - let intent = compile_query_intent_with_cuepacks( - "What activities has Melanie done with her family?", - None, - |cue| cue == "has:list", - default_registry(), - None, - ); - - assert!(intent - .labels - .contains(&"multi_evidence_collection".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "has:list")); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:memory.multi_evidence_collection")); -} - -#[test] -fn bundled_cuepack_extracts_inline_enumeration_as_list_evidence() { - let facets = extract_memory_facets( - "I like eating greens such as lettuce, spinach, and arugula.", - None, - &[], - ); - - assert!(facets.contains(&"has:list".to_string())); -} - -#[test] -fn cuepack_query_rules_emit_available_weighted_cues_with_provenance() { - let available = |cue: &str| { - matches!( - cue, - "type:navigation" | "travel:route" | "travel:transit" | "media:streaming" - ) - }; - let intent = compile_query_intent_with_cuepacks( - "What transit app did I use to get around?", - None, - available, - default_registry(), - None, - ); - - assert!(intent.labels.contains(&"navigation".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:navigation")); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:navigation.query")); -} - -#[test] -fn bundled_cuepack_maps_aquarium_queries_to_tank_without_core_rule() { - let available = |cue: &str| cue == "tank"; - let intent = compile_query_intent_with_cuepacks( - "How many fish are there in total in both of my aquariums?", - None, - available, - default_registry(), - None, - ); - - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "tank" && *weight > 3.0)); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:aquarium.tank_alias")); -} - -#[test] -fn core_query_intent_does_not_hardcode_aquarium_alias() { - let intent = cuemap::facets::compile_query_intent( - "How many fish are there in total in both of my aquariums?", - |cue| cue == "tank", - ); - - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "tank")); -} - -#[test] -fn bundled_cuepack_maps_room_furniture_queries_to_available_furniture_items() { - let available = |cue: &str| cue == "dresser"; - let intent = compile_query_intent_with_cuepacks( - "Any tips for rearranging the furniture in my bedroom?", - None, - available, - default_registry(), - None, - ); - - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "dresser" && *weight > 4.0)); - assert!(intent - .cuepack_rules - .iter() - .any(|rule| rule == "memory-general:home.bedroom_furniture_alias")); -} - -#[test] -fn core_query_intent_does_not_hardcode_room_furniture_aliases() { - let intent = cuemap::facets::compile_query_intent( - "Any tips for rearranging the furniture in my bedroom?", - |cue| cue == "dresser", - ); - - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "dresser")); -} - -#[test] -fn custom_cuepack_toml_validates_and_overrides_loaded_pack_by_name() { - let dir = tempfile::tempdir().expect("tempdir"); - let path = dir.path().join("memory-general.toml"); - fs::write( - &path, - r#" -name = "memory-general" -version = "9.9.9" -description = "test override" -enabled_by_default = true - -[[memory_rules]] -id = "custom" -contains_any = ["cardiology"] -emits = ["domain:cardiology"] -"#, - ) - .expect("write cuepack"); - - let info = CuePackRegistry::validate_file(&path).expect("valid cuepack"); - assert_eq!(info.name, "memory-general"); - assert_eq!(info.version, "9.9.9"); - - let registry = CuePackRegistry::load(true, &[dir.path().to_path_buf()]); - let facets = registry.extract_memory_facets("I track cardiology notes.", None); - assert!(facets.facets.contains(&"domain:cardiology".to_string())); - assert!(facets - .matched_rules - .contains(&"memory-general:custom".to_string())); -} diff --git a/tests/engine/mod.rs b/tests/engine/mod.rs index 8a3a8a6..841aff8 100644 --- a/tests/engine/mod.rs +++ b/tests/engine/mod.rs @@ -1,7 +1,273 @@ use cuemap::engine::CueMapEngine; +use cuemap::semantic::{SemanticConfig, SemanticEncoder, SemanticStorage, StoredSemanticVector}; use cuemap::structures::{MainStats, INVALID_MEMORY_ID}; use serde_json::json; use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +struct CountingEncoder { + calls: Arc<AtomicUsize>, +} + +impl SemanticEncoder for CountingEncoder { + fn dimensions(&self) -> usize { + 3 + } + + fn encode(&self, text: &str) -> Result<Vec<f32>, String> { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok(vec![text.len() as f32, 1.0, 0.0]) + } +} + +#[test] +fn test_opt_in_semantic_recall_uses_vector_candidates() { + let mut config = SemanticConfig::default(); + config.enabled = true; + config.dimensions = 3; + config.storage = SemanticStorage::Int8; + config.exact_fallback_max = 32; + + let mut engine = CueMapEngine::<MainStats>::new(); + engine.set_semantic_config(config); + let first = engine.add_memory_with_event_time_and_vector( + "A note about a quiet seaside walk".to_string(), + vec!["seaside".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![1.0, 0.0, 0.0]), + ); + let second = engine.add_memory_with_event_time_and_vector( + "A note about a busy train station".to_string(), + vec!["station".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![0.0, 1.0, 0.0]), + ); + + let query = [0.98, 0.02, 0.0]; + let results = engine.recall_weighted_with_query_embedding( + Vec::new(), + 1, + false, + None, + 1, + false, + true, + None, + None, + Some(&query), + ); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory_id, first); + assert_ne!(results[0].memory_id, second); + assert_eq!(engine.semantic_index_stats().1, 2); + assert!(matches!( + engine.get_memory(first).unwrap().semantic_vector, + Some(StoredSemanticVector::Int8 { .. }) + )); +} + +#[test] +fn test_hybrid_semantic_rerank_does_not_add_candidates() { + let mut config = SemanticConfig::default(); + config.enabled = true; + config.dimensions = 3; + config.storage = SemanticStorage::F32; + + let mut engine = CueMapEngine::<MainStats>::new(); + engine.set_semantic_config(config); + let lexical_candidate = engine.add_memory_with_event_time_and_vector( + "A lexical memory with a weak semantic match".to_string(), + vec!["lexical".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![0.0, 1.0, 0.0]), + ); + let semantic_only_candidate = engine.add_memory_with_event_time_and_vector( + "A semantic-only memory with a strong semantic match".to_string(), + vec!["other".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![1.0, 0.0, 0.0]), + ); + + let results = engine.recall_weighted_with_query_embedding_rerank_only( + vec![("lexical".to_string(), 1.0)], + 1, + false, + None, + 1, + false, + true, + None, + None, + Some(&[1.0, 0.0, 0.0]), + ); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory_id, lexical_candidate); + assert_ne!(results[0].memory_id, semantic_only_candidate); + assert!(results[0].explain.is_none()); +} + +#[test] +fn test_hybrid_semantic_rerank_fuses_normalized_scores() { + let mut config = SemanticConfig::default(); + config.enabled = true; + config.dimensions = 3; + config.storage = SemanticStorage::F32; + config.semantic_rerank_weight = 1.0; + + let mut engine = CueMapEngine::new(); + engine.set_semantic_config(config); + let lexical_strong = engine.add_memory_with_event_time_and_vector( + "A lexical memory with an extra exact cue".to_string(), + vec!["lexical".to_string(), "rare".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![0.0, 1.0, 0.0]), + ); + let semantic_strong = engine.add_memory_with_event_time_and_vector( + "A semantically aligned lexical memory".to_string(), + vec!["lexical".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![1.0, 0.0, 0.0]), + ); + + let results = engine.recall_weighted_with_query_embedding_rerank_only( + vec![("lexical".to_string(), 1.0), ("rare".to_string(), 1.0)], + 2, + false, + None, + 1, + false, + true, + None, + None, + Some(&[1.0, 0.0, 0.0]), + ); + + assert_eq!(results.len(), 2); + assert_eq!(results[0].memory_id, semantic_strong); + assert_eq!(results[1].memory_id, lexical_strong); + assert!(results[0].score > results[1].score); +} + +#[test] +fn test_hybrid_semantic_rerank_window_is_applied_before_semantic_scoring() { + let mut config = SemanticConfig::default(); + config.enabled = true; + config.dimensions = 3; + config.storage = SemanticStorage::F32; + config.semantic_rerank_weight = 1.0; + config.semantic_rerank_candidate_limit = 1; + + let mut engine = CueMapEngine::new(); + engine.set_semantic_config(config); + let lexical_strong = engine.add_memory_with_event_time_and_vector( + "A lexical memory with an extra exact cue".to_string(), + vec!["lexical".to_string(), "rare".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![0.0, 1.0, 0.0]), + ); + let semantic_strong = engine.add_memory_with_event_time_and_vector( + "A semantically aligned lexical memory".to_string(), + vec!["lexical".to_string()], + None, + MainStats::default(), + true, + None, + Some(vec![1.0, 0.0, 0.0]), + ); + + let (results, timing) = engine.recall_weighted_with_query_embedding_rerank_only_with_timing( + vec![("lexical".to_string(), 1.0), ("rare".to_string(), 1.0)], + 1, + false, + None, + 1, + false, + true, + None, + None, + Some(&[1.0, 0.0, 0.0]), + ); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory_id, lexical_strong); + assert_ne!(results[0].memory_id, semantic_strong); + assert_eq!(timing.semantic_rerank_candidate_limit, 1); + assert_eq!(timing.semantic_rerank_candidate_count, 1); +} + +#[test] +fn test_query_embedding_cache_reuses_and_bounds_encoded_queries() { + let calls = Arc::new(AtomicUsize::new(0)); + let mut config = SemanticConfig::default(); + config.enabled = true; + config.encoder_enabled = true; + config.dimensions = 3; + config.query_embedding_cache_capacity = 2; + + let mut engine = CueMapEngine::<MainStats>::new(); + engine.set_semantic_config(config); + engine.set_semantic_encoder(Some(Arc::new(CountingEncoder { + calls: Arc::clone(&calls), + }))); + + assert!(engine.encode_semantic_text("same query").is_some()); + assert!(engine.encode_semantic_text("same query").is_some()); + assert_eq!(calls.load(Ordering::Relaxed), 1); + + assert!(engine.encode_semantic_text("second query").is_some()); + assert!(engine.encode_semantic_text("third query").is_some()); + assert!(engine.encode_semantic_text("same query").is_some()); + assert_eq!(calls.load(Ordering::Relaxed), 4); +} + +#[test] +fn test_query_embedding_cache_is_cleared_when_encoder_changes() { + let first_calls = Arc::new(AtomicUsize::new(0)); + let second_calls = Arc::new(AtomicUsize::new(0)); + let mut config = SemanticConfig::default(); + config.enabled = true; + config.encoder_enabled = true; + config.dimensions = 3; + + let mut engine = CueMapEngine::<MainStats>::new(); + engine.set_semantic_config(config); + engine.set_semantic_encoder(Some(Arc::new(CountingEncoder { + calls: Arc::clone(&first_calls), + }))); + assert!(engine.encode_semantic_text("same query").is_some()); + assert!(engine.encode_semantic_text("same query").is_some()); + assert_eq!(first_calls.load(Ordering::Relaxed), 1); + + engine.set_semantic_encoder(Some(Arc::new(CountingEncoder { + calls: Arc::clone(&second_calls), + }))); + assert!(engine.encode_semantic_text("same query").is_some()); + assert_eq!(second_calls.load(Ordering::Relaxed), 1); +} #[test] fn test_memory_cues_storage() { @@ -74,6 +340,45 @@ fn test_source_key_upsert_reuses_numeric_id() { assert_eq!(results[0].memory_id, first); } +#[test] +fn test_historical_source_upsert_preserves_event_time_and_is_idempotent() { + let engine = CueMapEngine::new(); + let event_time = 1_704_067_200.25; + let first = engine.upsert_memory_with_source_key_and_options( + "openclaw:agent-1:session-1:message-1".to_string(), + "The deployment moved to Friday".to_string(), + vec!["deployment".to_string()], + None, + Some(MainStats::default()), + false, + true, + false, + Some(event_time), + ); + assert!(engine.reinforce_memory(first, vec!["deployment".to_string()])); + let second = engine.upsert_memory_with_source_key_and_options( + "openclaw:agent-1:session-1:message-1".to_string(), + "The deployment moved to next Friday".to_string(), + vec!["deployment".to_string()], + None, + None, + false, + true, + false, + Some(event_time + 60.0), + ); + + assert_eq!(first, second); + assert_eq!(engine.total_memories(), 1); + let memory = engine.get_memory(first).unwrap(); + assert_eq!(memory.created_at, event_time + 60.0); + assert_eq!(memory.stats.reinforcement_count, 1); + assert_eq!( + memory.source_key.as_deref(), + Some("openclaw:agent-1:session-1:message-1") + ); +} + #[test] fn test_source_order_index_tracks_add_upsert_and_delete() { let engine = CueMapEngine::new(); diff --git a/tests/facets/mod.rs b/tests/facets/mod.rs index 513dc46..1053056 100644 --- a/tests/facets/mod.rs +++ b/tests/facets/mod.rs @@ -1,5145 +1,630 @@ -use cuemap::engine::CueMapEngine; -use cuemap::facets::{ - compile_query_intent, compile_query_intent_with_reference_time, extract_memory_facets, -}; -use cuemap::nl::tokenize_to_cues; -use cuemap::structures::MainStats; +use cuemap::facets::{compile_query_plan, extract_memory_facets}; use serde_json::json; -use std::collections::{HashMap, HashSet}; - -fn compile_weighted_query(engine: &CueMapEngine<MainStats>, query: &str) -> Vec<(String, f64)> { - compile_weighted_query_at(engine, query, None) -} - -fn compile_weighted_query_at( - engine: &CueMapEngine<MainStats>, - query: &str, - reference_time: Option<&str>, -) -> Vec<(String, f64)> { - let mut weighted_cues: Vec<(String, f64)> = tokenize_to_cues(query) - .into_iter() - .map(|cue| (cue, 1.0)) - .collect(); - let total_memories = engine.total_memories().max(1); - let intent = compile_query_intent_with_reference_time(query, reference_time, |cue| { - let df = engine.get_cue_frequency(cue); - df > 0 && (df <= 16 || df * 5 <= total_memories) - }); - - for (cue, multiplier) in &intent.cue_weight_adjustments { - if let Some((_, weight)) = weighted_cues - .iter_mut() - .find(|(existing, _)| existing == cue) - { - *weight *= *multiplier; - } - } - for (cue, weight) in &intent.weighted_cues { - if let Some((_, existing_weight)) = weighted_cues - .iter_mut() - .find(|(existing, _)| existing == cue) - { - if *existing_weight < *weight { - *existing_weight = *weight; - } - } else { - weighted_cues.push((cue.clone(), *weight)); - } - } - - weighted_cues -} +use std::collections::HashMap; #[test] -fn extracts_general_source_and_evidence_facets() { +fn extracts_structural_evidence_and_source_cues() { let mut metadata = HashMap::new(); - metadata.insert("role".to_string(), json!("Doctor")); - metadata.insert("channel".to_string(), json!("support inbox")); - metadata.insert("source_session_id".to_string(), json!("Case-123")); + metadata.insert("source_role".to_string(), json!("user")); + metadata.insert("source_date".to_string(), json!("2023-04-21")); let facets = extract_memory_facets( - "Doctor: I currently take 20 mg daily for 2 weeks and paid $15 last week.", + "The meeting is next Friday at 7:30 PM and costs $20.", Some(&metadata), &[], ); - assert!(facets.contains(&"source_role:doctor".to_string())); - assert!(facets.contains(&"source_channel:support_inbox".to_string())); - assert!(facets.contains(&"source_session:case_123".to_string())); - assert!(facets.contains(&"has:number".to_string())); - assert!(facets.contains(&"has:money".to_string())); - assert!(facets.contains(&"has:duration".to_string())); - assert!(facets.contains(&"temporal:current".to_string())); - assert!(facets.contains(&"temporal:last_week".to_string())); + for expected in [ + "source_role:user", + "source_time:dated", + "source_date:2023_04_21", + "source_week:2023_w16", + "has:number", + "has:money", + "has:time", + "time_of_day:evening", + "temporal:relative", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } } #[test] -fn extracts_source_date_facets_from_metadata_timestamps() { - let mut metadata = HashMap::new(); - metadata.insert( - "source_date".to_string(), - json!("2023/04/21 (Fri) 00:30"), - ); +fn keeps_surface_entities_without_classifying_them() { + let facets = extract_memory_facets("Maya reviewed \"Project Atlas\" on Tuesday.", None, &[]); - let facets = extract_memory_facets( - "user: I just planted 12 new tomato saplings today.", - Some(&metadata), - &[], - ); - - assert!(facets.contains(&"source_time:dated".to_string())); - assert!(facets.contains(&"source_date:2023_04_21".to_string())); - assert!(facets.contains(&"source_week:2023_w16".to_string())); - assert!(facets.contains(&"source_month:2023_04".to_string())); - assert!(facets.contains(&"source_year:2023".to_string())); + assert!(facets.iter().any(|facet| facet == "entity:maya")); + assert!(facets.iter().any(|facet| facet == "entity:project_atlas")); + assert!(facets.iter().any(|facet| facet == "has:date")); + assert!(!facets.iter().any(|facet| facet.starts_with("type:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("preference:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("purchase:"))); } #[test] -fn extracts_time_of_day_facets_from_words_and_clock_times() { - let evening = extract_memory_facets( - "User: I prefer winding down by 9:30 pm during the later part of the day.", +fn semantic_language_does_not_create_ontology_facets() { + let facets = extract_memory_facets( + "I prefer tea, bought a new mug, and always want recommendations.", None, &[], ); - assert!(evening.contains(&"has:time".to_string())); - assert!(evening.contains(&"time_of_day:evening".to_string())); - - let morning = extract_memory_facets( - "User: I usually exercise at 7:15 am before work.", - None, - &[], - ); - assert!(morning.contains(&"time_of_day:morning".to_string())); + assert!(!facets.iter().any(|facet| facet.starts_with("type:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("preference:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("instruction:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("purchase:"))); } #[test] -fn extracts_content_month_facets_from_month_names_and_short_dates() { - let named = extract_memory_facets( - "User: I went to the opening night on 15th February.", - None, - &[], - ); - let numeric = extract_memory_facets( - "User: I took my niece to the Natural History Museum on 2/8.", +fn structural_evidence_detects_surface_formats_without_topic_classification() { + let facets = extract_memory_facets( + "Email me at kaan@example.com, open https://cuemap.dev/docs, and edit src/facets.rs.\n\n```rust\nlet answer = 42;\n```\nThe label is \"semantic\".", None, &[], ); - assert!(named.contains(&"has:date".to_string())); - assert!(named.contains(&"content_month:02".to_string())); - assert!(numeric.contains(&"has:date".to_string())); - assert!(numeric.contains(&"content_month:02".to_string())); + for expected in [ + "has:email", + "has:url", + "has:file_path", + "has:code", + "has:quote", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } + assert!(!facets.iter().any(|facet| facet.starts_with("preference:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("purchase:"))); } #[test] -fn extracts_frequency_facets_from_cadence_expressions() { - let facets = extract_memory_facets( - "User: I've been doing yoga twice a week, usually after work.", - None, - &[], - ); - - assert!(facets.contains(&"has:frequency".to_string())); - assert!(facets.contains(&"schedule:frequency".to_string())); - assert!(facets.contains(&"frequency_unit:week".to_string())); - assert!(facets.contains(&"schedule:weekly".to_string())); +fn url_facet_detects_http_urls_and_ignores_incomplete_schemes() { + let facets = extract_memory_facets("Read https://cuemap.dev/docs#facets.", None, &[]); + assert!(facets.iter().any(|facet| facet == "has:url")); - let hourly = extract_memory_facets( - "User: I usually work 40 hours per week during peak campaign seasons.", + let local_endpoints = extract_memory_facets( + "Use http://localhost:3000 or http://127.0.0.1:8080; the UI is at localhost:5173.", None, &[], ); - assert!(hourly.contains(&"has:frequency".to_string())); - assert!(hourly.contains(&"frequency_unit:week".to_string())); + assert!(local_endpoints.iter().any(|facet| facet == "has:url")); + + let incomplete = extract_memory_facets("The value is https://.", None, &[]); + assert!(!incomplete.iter().any(|facet| facet == "has:url")); } #[test] -fn compiles_time_of_day_query_intent_from_raw_query_text() { - let intent = compile_query_intent("Can you suggest activities for the evening?", |cue| { - matches!(cue, "time_of_day:evening" | "has:time") - }); +fn email_facet_detects_address_shapes() { + let facets = extract_memory_facets("Contact kaan+cuemap@example.co.uk.", None, &[]); + assert!(facets.iter().any(|facet| facet == "has:email")); - assert!(intent.labels.contains(&"time_of_day".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "time_of_day:evening")); + let incomplete = extract_memory_facets("The address is kaan@example.", None, &[]); + assert!(!incomplete.iter().any(|facet| facet == "has:email")); } #[test] -fn extracts_type_and_entity_facets_without_benchmark_roles() { - let facets = extract_memory_facets( - "Chef: I prefer Sony A7R IV photos, dislike cinnamon, and bought \"Peak Design Bag\".", - None, - &[], - ); +fn quote_facet_detects_bounded_quoted_spans() { + let facets = extract_memory_facets("The exact label is \"semantic rerank\".", None, &[]); + assert!(facets.iter().any(|facet| facet == "has:quote")); - assert!(facets.contains(&"source_role:chef".to_string())); - assert!(facets.contains(&"type:preference".to_string())); - assert!(facets.contains(&"type:dislike".to_string())); - assert!(facets.contains(&"type:ownership".to_string())); - assert!(facets.contains(&"entity:sony_a7r_iv".to_string())); - assert!(facets.contains(&"entity:peak_design_bag".to_string())); + let incomplete = extract_memory_facets("The apostrophe is in don't.", None, &[]); + assert!(!incomplete.iter().any(|facet| facet == "has:quote")); } #[test] -fn extracts_purchase_consideration_from_first_person_planning_language() { - let upgrade = extract_memory_facets( - "User: I'm considering upgrading from a Fender Stratocaster to a Gibson Les Paul.", - None, - &[], - ); +fn code_facet_detects_inline_and_fenced_code() { + let inline = extract_memory_facets("Call `compile_query_plan` here.", None, &[]); + assert!(inline.iter().any(|facet| facet == "has:code")); - assert!(upgrade.contains(&"type:purchase_consideration".to_string())); + let fenced = extract_memory_facets("```rust\nlet answer = 42;\n```", None, &[]); + assert!(fenced.iter().any(|facet| facet == "has:code")); - let comparison_question = extract_memory_facets( - "User: What are the differences between open D tuning and standard tuning?", - None, - &[], - ); - assert!(!comparison_question.contains(&"type:purchase_consideration".to_string())); + let truncated = extract_memory_facets("The chunk ends at:\n```rust\nlet answer = 42;", None, &[]); + assert!(truncated.iter().any(|facet| facet == "has:code")); + + let prose = extract_memory_facets("Use ordinary prose without delimiters.", None, &[]); + assert!(!prose.iter().any(|facet| facet == "has:code")); } #[test] -fn extracts_competition_event_facets_from_first_person_sports_events() { - let soccer = extract_memory_facets( - "User: I participate in the company's annual charity soccer tournament today.", - None, - &[], - ); - assert!(soccer.contains(&"type:competition_event".to_string())); - assert!(soccer.contains(&"activity_domain:sport".to_string())); - assert!(soccer.contains(&"type:activity".to_string())); - assert!(soccer.contains(&"type:event".to_string())); +fn file_path_facet_detects_paths_and_excludes_urls() { + let relative = extract_memory_facets("Edit src/facets.rs.", None, &[]); + assert!(relative.iter().any(|facet| facet == "has:file_path")); - let future_soccer = extract_memory_facets( - "User: I will participate in the company's annual charity soccer tournament today.", - None, - &[], - ); - assert!(future_soccer.contains(&"type:competition_event".to_string())); - assert!(future_soccer.contains(&"activity_domain:sport".to_string())); - assert!(future_soccer.contains(&"type:activity".to_string())); - assert!(future_soccer.contains(&"type:event".to_string())); - - let planning = extract_memory_facets( - "User: I might watch a soccer tournament this weekend.", - None, - &[], - ); - assert!(!planning.contains(&"type:competition_event".to_string())); -} + let absolute = extract_memory_facets("The file is /tmp/cuemap/state.json.", None, &[]); + assert!(absolute.iter().any(|facet| facet == "has:file_path")); -#[test] -fn extracts_activity_event_from_first_person_did_named_event() { - let charity_walk = extract_memory_facets( - "User: I just did the \"Walk for Hunger\" charity event today with my colleagues from work.", - None, - &[], - ); + for file_name in ["Cargo.toml", "facets.rs", ".env", "model.safetensors"] { + let facets = extract_memory_facets(file_name, None, &[]); + assert!(facets.iter().any(|facet| facet == "has:file_name"), "missing file name cue for {file_name}: {facets:?}"); + assert!(!facets.iter().any(|facet| facet == "has:file_path"), "bare file name became a path: {file_name}: {facets:?}"); + } - assert!(charity_walk.contains(&"type:activity".to_string())); - assert!(charity_walk.contains(&"type:event".to_string())); - assert!(charity_walk.contains(&"event_domain:charity".to_string())); + for path in ["/usr/bin", "../config", "src/components"] { + let facets = extract_memory_facets(path, None, &[]); + assert!(facets.iter().any(|facet| facet == "has:file_path"), "missing path cue for {path}: {facets:?}"); + } - let donation = extract_memory_facets( - "User: I donated to charity online today after reading about the campaign.", - None, - &[], - ); - assert!(!donation.contains(&"event_domain:charity".to_string())); + let url = extract_memory_facets("Open https://cuemap.dev/docs/facets.rs.", None, &[]); + assert!(!url.iter().any(|facet| facet == "has:file_path")); } #[test] -fn extracts_wake_time_routine_from_timed_wake_language() { - let timed = extract_memory_facets( - "User: I like to wake up at 7:30 am on Saturdays and fit in coffee beforehand.", - None, - &[], - ); +fn clock_facets_reject_impossible_meridiem_hours_and_accept_24_hour_times() { + let impossible = extract_memory_facets("The event is at 17 pm, not 23 am.", None, &[]); + assert!(!impossible.iter().any(|facet| facet == "has:time")); + assert!(!impossible.iter().any(|facet| facet.starts_with("time_of_day:"))); - assert!(timed.contains(&"routine:wake_time".to_string())); - assert!(timed.contains(&"type:routine".to_string())); - - let untimed = extract_memory_facets( - "User: Do you have any advice for waking up earlier without feeling tired?", - None, - &[], - ); - assert!(!untimed.contains(&"routine:wake_time".to_string())); + let valid = extract_memory_facets("The event is at 17:00.", None, &[]); + assert!(valid.iter().any(|facet| facet == "has:time")); + assert!(valid.iter().any(|facet| facet == "time_of_day:evening")); } #[test] -fn extracts_bed_time_routine_from_timed_bed_language() { - let timed = extract_memory_facets( - "User: I didn't get to bed until 2 AM last Wednesday.", +fn may_is_only_a_month_in_temporal_context() { + let modal = extract_memory_facets( + "May I ask whether this may improve recall?", None, &[], ); + assert!( + !modal.iter().any(|facet| facet == "has:date"), + "modal facets: {modal:?}" + ); + assert!( + !modal.iter().any(|facet| facet == "content_month:05"), + "modal facets: {modal:?}" + ); - assert!(timed.contains(&"routine:bed_time".to_string())); - assert!(timed.contains(&"type:routine".to_string())); - assert!(timed.contains(&"has:time".to_string())); - - let untimed = extract_memory_facets( - "User: I bought a new bed frame last weekend.", + let date = extract_memory_facets( + "The deployment was in May 2024 and the trip followed on May 3.", None, &[], ); - assert!(!untimed.contains(&"routine:bed_time".to_string())); + assert!(date.iter().any(|facet| facet == "has:date")); + assert!(date.iter().any(|facet| facet == "content_month:05")); } #[test] -fn extracts_iteration_facets_from_revised_or_followup_outputs() { - let revised = extract_memory_facets( - "Assistant: Sure, here's a more romantic and heart-felt song for you.", - None, - &[], - ); - assert!(revised.contains(&"type:iteration".to_string())); +fn ambiguous_textual_months_require_calendar_context() { + for prose in [ + "We march forward.", + "March toward the entrance.", + "An august presence filled the room.", + "August joined the team.", + "April joined the call.", + "June joined the call.", + ] { + let facets = extract_memory_facets(prose, None, &[]); + assert!(!facets.iter().any(|facet| facet == "has:date"), "false date for {prose:?}: {facets:?}"); + assert!(!facets.iter().any(|facet| facet.starts_with("content_month:")), "false month for {prose:?}: {facets:?}"); + } - let first = extract_memory_facets( - "Assistant: Here's a sad song with notes for you.", - None, - &[], - ); - assert!(!first.contains(&"type:iteration".to_string())); + for dated in [ + "in March 2024", + "march 3", + "August 2024", + "April 2024", + "It is June.", + ] { + let facets = extract_memory_facets(dated, None, &[]); + assert!(facets.iter().any(|facet| facet == "has:date"), "missing date for {dated:?}: {facets:?}"); + assert!(facets.iter().any(|facet| facet.starts_with("content_month:")), "missing month for {dated:?}: {facets:?}"); + } - let genre = extract_memory_facets( - "User: I'm really into indie and alternative rock right now.", - None, - &[], - ); - assert!(!genre.contains(&"type:iteration".to_string())); + for modal in ["It may in fact work.", "Version 5 may fail."] { + let facets = extract_memory_facets(modal, None, &[]); + assert!(!facets.iter().any(|facet| facet == "has:date"), "false date for {modal:?}: {facets:?}"); + assert!(!facets.iter().any(|facet| facet == "content_month:05"), "false May for {modal:?}: {facets:?}"); + } } #[test] -fn extracts_named_entity_attribute_relations() { - let facets = extract_memory_facets( - "User: I need a collar that would suit a Golden Retriever like Max.", - None, - &[], - ); +fn query_shape_matching_uses_tokens_not_substrings() { + let accidental = compile_query_plan("What is the smallest callback value?", |cue| { + cue == "has:list" + }); + assert!(!accidental + .labels + .iter() + .any(|label| label == "multi_evidence_collection")); - assert!(facets.contains(&"type:entity_attribute".to_string())); - assert!(facets.contains(&"attribute:class_relation".to_string())); - assert!(facets.contains(&"entity:golden_retriever".to_string())); - assert!(facets.contains(&"entity:max".to_string())); + let explicit = compile_query_plan("List all callback values", |cue| cue == "has:list"); + assert!(explicit + .labels + .iter() + .any(|label| label == "multi_evidence_collection")); } #[test] -fn extracts_preferred_attribute_relations_and_canonicalizes_favourite() { - let facets = extract_memory_facets( - "User: Nike has been my favourite brand so far for running shoes.", - None, - &[], - ); - let cues = tokenize_to_cues("Nike has been my favourite brand so far for running shoes."); +fn query_plan_only_emits_structural_or_query_shape_signals() { + let intent = compile_query_plan("Summarize the events from yesterday", |cue| { + cue == "has:list" || cue.starts_with("temporal:") + }); - assert!(facets.contains(&"type:preference".to_string())); - assert!(facets.contains(&"type:entity_attribute".to_string())); - assert!(facets.contains(&"attribute:class_relation".to_string())); - assert!(cues.contains(&"favorite".to_string())); - assert!(!cues.contains(&"favourite".to_string())); + assert!(intent.labels.iter().any(|label| label == "multi_evidence_summary")); + assert!(intent.labels.iter().any(|label| label == "temporal_yesterday")); + assert!(!intent.labels.iter().any(|label| label.contains("preference"))); + assert!(!intent.labels.iter().any(|label| label.contains("purchase"))); } #[test] -fn extracts_expertise_facets_from_professional_field_language() { +fn assistant_authored_first_person_text_keeps_assistant_source_role() { + let mut metadata = HashMap::new(); + metadata.insert("source_role".to_string(), json!("assistant")); + let facets = extract_memory_facets( - "User: Can you give me an overview of recent advancements in this field? Skip the basics as I am working in the field.", - None, + "I recommend using Flask routes for this project.", + Some(&metadata), &[], ); - assert!(facets.contains(&"type:expertise".to_string())); - assert!(facets.contains(&"type:interest".to_string())); + assert!(facets.iter().any(|facet| facet == "source_role:assistant")); + assert!(!facets.iter().any(|facet| facet == "source_role:user")); - let service_query_facets = extract_memory_facets( - "User: Can you suggest contractors or services that specialize in backyard landscaping?", + let prefixed_facets = extract_memory_facets( + "assistant: I recommend using Flask routes for this project.", None, &[], ); - assert!(!service_query_facets.contains(&"type:expertise".to_string())); + assert!(prefixed_facets + .iter() + .any(|facet| facet == "source_role:assistant")); + assert!(!prefixed_facets + .iter() + .any(|facet| facet == "source_role:user")); } #[test] -fn extracts_inspiration_source_facets_from_explicit_source_language() { - let facets = extract_memory_facets( - "User: I have been getting inspiration from social media and recently started a 30-day painting challenge.", - None, - &[], - ); +fn query_perspective_is_grammar_only_and_supports_first_person_variants() { + for query in [ + "Have I worked with Flask routes?", + "Did I build the project?", + "What do I need to do?", + "Didn't I mention the project?", + ] { + let intent = compile_query_plan(query, |cue| cue == "source_role:user"); + + assert!( + intent + .labels + .iter() + .any(|label| label == "query_perspective_first_person"), + "missing first-person perspective for {query:?}: {intent:?}" + ); + assert!(intent.labels.iter().any(|label| label == "source_user")); + assert!(intent + .weighted_cues + .iter() + .any(|(cue, weight)| cue == "source_role:user" && *weight == 2.0)); + } +} - assert!(facets.contains(&"type:inspiration_source".to_string())); - assert!(facets.contains(&"type:interest".to_string())); +#[test] +fn query_perspective_keeps_second_and_third_person_separate_from_source_role() { + let second_person = compile_query_plan("What did you recommend?", |cue| { + cue == "source_role:assistant" + }); + assert!(second_person + .labels + .iter() + .any(|label| label == "query_perspective_second_person")); + assert!(second_person + .labels + .iter() + .any(|label| label == "source_assistant")); + assert!(second_person + .weighted_cues + .iter() + .any(|(cue, weight)| cue == "source_role:assistant" && *weight == 2.0)); - let generic_request = extract_memory_facets( - "User: Do you have any ideas for how I can find new inspiration for my paintings?", - None, - &[], - ); - assert!(!generic_request.contains(&"type:inspiration_source".to_string())); + let third_person = compile_query_plan("What did they recommend?", |cue| { + cue == "source_role:user" || cue == "source_role:assistant" + }); + assert!(third_person + .labels + .iter() + .any(|label| label == "query_perspective_third_person")); + assert!(!third_person.labels.iter().any(|label| { + label == "source_user" || label == "source_assistant" + })); + assert!(!third_person + .weighted_cues + .iter() + .any(|(cue, _)| cue == "source_role:user" || cue == "source_role:assistant")); } #[test] -fn extracts_decision_selection_facets_from_confirmation_language() { - let facets = extract_memory_facets( - "User: Fissionator is a really cool one, especially as a final name for the enemy.", - None, - &[], +fn request_wrappers_use_embedded_perspective_instead_of_outer_you() { + let event_ordering = compile_query_plan( + "Can you list the order in which I brought up different aspects?", + |cue| cue == "source_role:user" || cue == "source_role:assistant", ); + assert!(event_ordering + .labels + .iter() + .any(|label| label == "query_perspective_first_person")); + assert!(event_ordering.labels.iter().any(|label| label == "source_user")); + assert!(!event_ordering + .labels + .iter() + .any(|label| label == "source_assistant")); + assert!(event_ordering + .weighted_cues + .iter() + .any(|(cue, weight)| cue == "source_role:user" && *weight == 2.0)); - assert!(facets.contains(&"type:decision".to_string())); - assert!(facets.contains(&"type:selection".to_string())); - assert!(facets.contains(&"type:naming".to_string())); + let assistant_target = compile_query_plan("Can you tell me what you recommended?", |cue| { + cue == "source_role:assistant" + }); + assert!(assistant_target + .labels + .iter() + .any(|label| label == "query_perspective_second_person")); + assert!(assistant_target + .labels + .iter() + .any(|label| label == "source_assistant")); - let undecided_options = extract_memory_facets( - "Assistant: Here are some possible names: Radik, Irradon, Nucleus, and Fissionator.", - None, - &[], - ); - assert!(!undecided_options.contains(&"type:decision".to_string())); - assert!(!undecided_options.contains(&"type:selection".to_string())); + let plain_request = compile_query_plan("Can you recommend a dessert?", |cue| { + cue == "source_role:user" || cue == "source_role:assistant" + }); + assert!(!plain_request.labels.iter().any(|label| { + label == "query_perspective_first_person" + || label == "query_perspective_second_person" + || label == "query_perspective_third_person" + || label == "source_user" + || label == "source_assistant" + })); } #[test] -fn extracts_activity_event_facets_from_first_person_completed_actions() { - let activity = extract_memory_facets( - "User: I just planted 12 new tomato saplings today.", - None, - &[], - ); - assert!(activity.contains(&"type:activity".to_string())); - assert!(activity.contains(&"type:event".to_string())); +fn embedded_question_perspective_beats_outer_wrapper_and_conflicts_are_unset() { + let user_target = compile_query_plan("Do you remember what I bought?", |cue| { + cue == "source_role:user" || cue == "source_role:assistant" + }); + assert!(user_target + .labels + .iter() + .any(|label| label == "query_perspective_first_person")); + assert!(user_target.labels.iter().any(|label| label == "source_user")); + assert!(!user_target + .labels + .iter() + .any(|label| label == "source_assistant")); - let planning = extract_memory_facets( - "User: I am thinking about planting tomatoes next month.", - None, - &[], - ); - assert!(!planning.contains(&"type:activity".to_string())); - assert!(!planning.contains(&"type:event".to_string())); + let assistant_target = compile_query_plan("Do you remember what you recommended?", |cue| { + cue == "source_role:user" || cue == "source_role:assistant" + }); + assert!(assistant_target + .labels + .iter() + .any(|label| label == "query_perspective_second_person")); + assert!(assistant_target + .labels + .iter() + .any(|label| label == "source_assistant")); - let intention = extract_memory_facets( - "User: I wanted to create the trip plan along with dinner suggestions in the same summary.", - None, - &[], + let conflicting = compile_query_plan( + "Do you remember what I bought and what you recommended?", + |cue| cue == "source_role:user" || cue == "source_role:assistant", ); - assert!(!intention.contains(&"type:activity".to_string())); - assert!(!intention.contains(&"type:event".to_string())); + assert!(!conflicting.labels.iter().any(|label| { + label == "source_user" || label == "source_assistant" + })); + assert!(!conflicting.weighted_cues.iter().any(|(cue, _)| { + cue == "source_role:user" || cue == "source_role:assistant" + })); } #[test] -fn extracts_activity_event_facets_from_experience_phrases() { - let got_back = extract_memory_facets( - "User: I just got back from my friend's wedding last weekend.", - None, - &[], - ); - let been_to = extract_memory_facets( - "User: I've been to a few galleries recently.", +fn non_wrapper_embedded_perspective_disagreement_is_unweighted() { + for query in [ + "Who did I meet when you visited?", + "What did I say about what she bought?", + ] { + let plan = compile_query_plan(query, |cue| { + cue == "source_role:user" || cue == "source_role:assistant" + }); + assert!(!plan.labels.iter().any(|label| { + label == "source_user" || label == "source_assistant" + }), "unexpected source preference for {query:?}: {plan:?}"); + assert!(!plan.weighted_cues.iter().any(|(cue, _)| { + cue == "source_role:user" || cue == "source_role:assistant" + }), "unexpected source cue for {query:?}: {plan:?}"); + } + + let matching = compile_query_plan("What did I say about what I bought?", |cue| { + cue == "source_role:user" + }); + assert!(matching.labels.iter().any(|label| label == "source_user")); +} + +#[test] +fn extracts_quantities_percentages_ranges_and_comparisons() { + let facets = extract_memory_facets( + "Latency was 5 ms, the payload was 512 MB, accuracy was 20%, and the safe range was between 4 and 6 kg.", None, &[], ); - assert!(got_back.contains(&"type:activity".to_string())); - assert!(got_back.contains(&"type:event".to_string())); - assert!(been_to.contains(&"type:activity".to_string())); - assert!(been_to.contains(&"type:event".to_string())); + for expected in [ + "has:measurement", + "quantity_unit:ms", + "measurement:5_ms", + "quantity_unit:mb", + "measurement:512_mb", + "has:percentage", + "percentage:20", + "has:numeric_range", + "has:comparator", + "comparison:between", + "range_min:4", + "range_max:6", + "quantity_unit:kg", + "range:4_6_kg", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } + + let comparison = extract_memory_facets("Keep the response under 5 ms, or at least 90% accurate.", None, &[]); + assert!(comparison.iter().any(|facet| facet == "comparison:less_than")); + assert!(comparison.iter().any(|facet| facet == "comparison:greater_than")); + assert!(comparison.iter().any(|facet| facet == "has:comparator")); } #[test] -fn extracts_religious_activity_facets_from_religious_service_events() { - let service = extract_memory_facets( - "User: I attended the Maundy Thursday service at the Episcopal Church.", +fn extracts_technical_identifiers_without_reusing_agent_namespaces() { + let facets = extract_memory_facets( + "UUID 550e8400-e29b-41d4-a716-446655440000, version v0.7.2, PR #142, GH-143, commit a91f72c, endpoint 127.0.0.1:8080, cuemap.dev, CUEMAP_INDEX_PATH, @kaan, and #retrieval.", None, &[], ); - assert!(service.contains(&"topic:religion".to_string())); - assert!(service.contains(&"activity_domain:religion".to_string())); - assert!(service.contains(&"type:activity".to_string())); - assert!(service.contains(&"type:event".to_string())); - let customer_service = extract_memory_facets( - "User: I called customer service about my subscription renewal.", - None, - &[], - ); - assert!(!customer_service.contains(&"activity_domain:religion".to_string())); + for expected in [ + "has:uuid", + "uuid:550e8400_e29b_41d4_a716_446655440000", + "has:semver", + "version:0_7_2", + "has:issue_reference", + "issue:142", + "issue:143", + "has:commit_hash", + "commit:a91f72c", + "has:ip_address", + "ip:127_0_0_1", + "has:port", + "port:8080", + "has:domain", + "domain_name:cuemap_dev", + "has:environment_variable", + "env:cuemap_index_path", + "has:user_mention", + "mention:kaan", + "has:hashtag", + "hashtag:retrieval", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } + assert!(!facets.iter().any(|facet| facet.starts_with("lang:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("type:"))); +} - let topic_discussion = extract_memory_facets( - "User: How does the Tripitaka influence Theravada Buddhist worship and practice?", +#[test] +fn extracts_file_names_extensions_and_directory_segments() { + let facets = extract_memory_facets( + "Edit src/facets.rs, then update Cargo.toml, .env, and model.safetensors.", None, &[], ); - assert!(topic_discussion.contains(&"topic:religion".to_string())); - assert!(!topic_discussion.contains(&"activity_domain:religion".to_string())); + + for expected in [ + "has:file_name", + "has:file_path", + "has:directory_path", + "path_segment:src", + "file_name:facets_rs", + "file_extension:rs", + "file_name:cargo_toml", + "file_extension:toml", + "file_name:env", + "file_name:model_safetensors", + "file_extension:safetensors", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } + assert!(!facets.iter().any(|facet| facet.starts_with("file:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("path:"))); } #[test] -fn extracts_media_streaming_usage_facets_from_watch_history() { - let long_term_services = extract_memory_facets( - "User: I've been using Netflix, Hulu, and Amazon Prime for the past 6 months while looking for new shows to watch.", - None, - &[], - ); - assert!(long_term_services.contains(&"media:watching".to_string())); - assert!(long_term_services.contains(&"media:streaming".to_string())); - assert!(long_term_services.contains(&"type:usage".to_string())); +fn extracts_document_and_code_structure_markers() { + let content = r#"{"name":"cuemap"} +name: cuemap +enabled: true +<root><item>value</item></root> +a,b +c,d - let free_trial = extract_memory_facets( - "User: I saw a documentary on Disney+ during my free trial last month.", - None, - &[], - ); - assert!(free_trial.contains(&"media:watching".to_string())); - assert!(free_trial.contains(&"media:streaming".to_string())); - assert!(free_trial.contains(&"type:usage".to_string())); +| Name | Value | +| --- | --- | +| mode | fast | - let comedy_special = extract_memory_facets( - "User: Can you recommend some stand-up comedy specials on Netflix with strong storytelling?", - None, - &[], - ); - assert!(comedy_special.contains(&"media:watching".to_string())); +Traceback (most recent call last): + at src/main.rs:42 - let music = extract_memory_facets( - "User: I've been listening to their songs a lot on Spotify lately.", - None, - &[], - ); - assert!(music.contains(&"media:music".to_string())); - assert!(music.contains(&"media:music_streaming".to_string())); - assert!(music.contains(&"media:streaming".to_string())); - assert!(music.contains(&"type:usage".to_string())); +diff --git a/a.rs b/a.rs +@@ -1 +1 @@ - let live_show = extract_memory_facets( - "User: Have they been playing any new songs or focusing on their older material?", - None, - &[], - ); - assert!(!live_show.contains(&"media:music_streaming".to_string())); +## Install +- [x] Build the engine +> Preserve this note. +[documentation](https://cuemap.dev/docs) +```rust +let answer = 42; +```"#; + let facets = extract_memory_facets(content, None, &[]); - let writing = extract_memory_facets( - "User: I want to practice using vivid language in my essays.", - None, - &[], - ); - assert!(!writing.contains(&"media:streaming".to_string())); - assert!(!writing.contains(&"type:usage".to_string())); + for expected in [ + "has:json", + "has:key_value_pairs", + "has:yaml", + "has:xml", + "has:csv", + "has:markdown_table", + "has:stack_trace", + "has:diff", + "has:heading", + "heading_level:2", + "has:checklist", + "has:block_quote", + "has:markdown_link", + "code_language:rust", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } } #[test] -fn extracts_current_book_reading_facets_from_first_person_state() { - let current = extract_memory_facets( - "User: I'm currently devouring \"The Seven Husbands of Evelyn Hugo\" and it's hard to put down.", +fn extracts_negation_contrast_correction_and_supersession() { + let facets = extract_memory_facets( + "I never chose the old option, but actually changed my mind and used to prefer it instead of the new one.", None, &[], ); - assert!(current.contains(&"reading:current".to_string())); - assert!(current.contains(&"media:book_reading".to_string())); - assert!(current.contains(&"media:book".to_string())); - assert!(current.contains(&"temporal:current".to_string())); - let old = extract_memory_facets( - "User: We're going to discuss \"The Last House Guest\", which I've already read and enjoyed.", - None, - &[], - ); - assert!(!old.contains(&"reading:current".to_string())); + for expected in [ + "has:negation", + "has:contrast", + "has:correction", + "has:supersession", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } } #[test] -fn extracts_transport_event_facets_from_real_travel_phrases() { - let bus = extract_memory_facets( - "User: I just got back from a bus ride to attend a friend's wedding today.", - None, - &[], - ); - assert!(bus.contains(&"transport_mode:bus".to_string())); - assert!(bus.contains(&"transport_event:bus".to_string())); - assert!(bus.contains(&"type:activity".to_string())); - - let train = extract_memory_facets( - "User: I took a train ride to visit my family today.", - None, - &[], - ); - assert!(train.contains(&"transport_mode:train".to_string())); - assert!(train.contains(&"transport_event:train".to_string())); +fn extracts_emoji_without_script_noise() { + let facets = extract_memory_facets("Hello мир 世界 مرحبا नमस्ते 👋", None, &[]); - let general = extract_memory_facets( - "User: I've been taking more trains and buses instead of driving.", - None, - &[], - ); - assert!(general.contains(&"transport_mode:train".to_string())); - assert!(general.contains(&"transport_mode:bus".to_string())); - assert!(!general.contains(&"transport_event:train".to_string())); - assert!(!general.contains(&"transport_event:bus".to_string())); + assert!(facets.iter().any(|facet| facet == "has:emoji"), "missing emoji facet: {facets:?}"); + assert!(facets.iter().all(|facet| !facet.starts_with("script:") && facet != "has:script"), "unexpected script facets: {facets:?}"); } #[test] -fn extracts_milestone_facets_from_real_milestone_language() { - let first_client = extract_memory_facets( - "User: I just signed a contract with my first client today.", - None, - &[], - ); - assert!(first_client.contains(&"type:activity".to_string())); - assert!(first_client.contains(&"type:event".to_string())); - assert!(first_client.contains(&"type:milestone".to_string())); - - let contract_advice = extract_memory_facets( - "Assistant: A well-drafted contract should include payment terms and scope of work.", - None, - &[], - ); - assert!(!contract_advice.contains(&"type:milestone".to_string())); -} - -#[test] -fn extracts_ownership_from_first_person_acquisition_language() { - let acquired = extract_memory_facets( - "User: I just got a smoker today and I'm excited to experiment with it.", - None, - &[], - ); - assert!(acquired.contains(&"type:ownership".to_string())); - assert!(acquired.contains(&"type:activity".to_string())); - assert!(acquired.contains(&"type:event".to_string())); - assert!(acquired.contains(&"purchase:acquired".to_string())); - - let recipient_acquired = extract_memory_facets( - "User: For my sister's birthday, I got her a yellow dress and a pair of earrings to match.", - None, - &[], - ); - assert!(recipient_acquired.contains(&"type:ownership".to_string())); - assert!(recipient_acquired.contains(&"purchase:acquired".to_string())); - - let sourced = extract_memory_facets( - "User: I'm happy with my new tennis racket, which I got from a sports store downtown.", - None, - &[], - ); - assert!(sourced.contains(&"type:ownership".to_string())); - assert!(sourced.contains(&"purchase:source".to_string())); - - let source_statement = extract_memory_facets( - "User: The new bookshelf is from IKEA, and I'm really happy with it.", - None, - &[], - ); - assert!(source_statement.contains(&"type:ownership".to_string())); - assert!(source_statement.contains(&"purchase:source".to_string())); - - let got_to = extract_memory_facets( - "User: I got to see a great concert last night.", - None, - &[], - ); - assert!(!got_to.contains(&"type:ownership".to_string())); -} - -#[test] -fn extracts_ownership_from_possession_use_and_sale_language() { - let long_term = extract_memory_facets( - "User: I've had my acoustic guitar, a Yamaha FG800, for about 8 years.", - None, - &[], - ); - assert!(long_term.contains(&"type:ownership".to_string())); - assert!(long_term.contains(&"inventory_object:acoustic".to_string())); - assert!(long_term.contains(&"inventory_object:guitar".to_string())); - - let active_use = extract_memory_facets( - "User: I've been playing my black Fender Stratocaster electric guitar a lot lately.", - None, - &[], - ); - assert!(active_use.contains(&"type:ownership".to_string())); - assert!(active_use.contains(&"inventory_object:fender".to_string())); - assert!(active_use.contains(&"inventory_object:guitar".to_string())); - - let sale = extract_memory_facets( - "User: I'm thinking of selling my old drum set, a 5-piece Pearl Export.", - None, - &[], - ); - assert!(sale.contains(&"type:ownership".to_string())); - assert!(sale.contains(&"inventory_object:drum".to_string())); - assert!(sale.contains(&"inventory_object:set".to_string())); - - let appositive = extract_memory_facets( - "User: I need to service my Korg B1, which I've had for about 3 years.", - None, - &[], - ); - assert!(appositive.contains(&"type:ownership".to_string())); - assert!(appositive.contains(&"inventory_object:korg".to_string())); - assert!(appositive.contains(&"inventory_object:b1".to_string())); -} - -#[test] -fn auxiliary_have_and_got_do_not_create_ownership_facets() { - let done = extract_memory_facets( - "User: I think I've got everything I need. Thanks for the help!", - None, - &[], - ); - assert!(!done.contains(&"type:ownership".to_string())); - assert!(!done.iter().any(|facet| facet.starts_with("inventory_object:"))); - - let heard = extract_memory_facets( - "User: I have heard that Disney has faced criticism for its changes.", - None, - &[], - ); - assert!(!heard.contains(&"type:ownership".to_string())); - assert!(!heard.iter().any(|facet| facet.starts_with("inventory_object:"))); - - let concrete = extract_memory_facets( - "User: I currently have a Korg B1 digital piano in my studio.", - None, - &[], - ); - assert!(concrete.contains(&"type:ownership".to_string())); - assert!(concrete.contains(&"inventory_object:korg".to_string())); - assert!(concrete.contains(&"inventory_object:b1".to_string())); -} - -#[test] -fn extracts_homegrown_ingredient_facets_from_real_relations() { - let facets = extract_memory_facets( - "User: I've been using basil and mint in my cooking lately. I've even harvested some cherry tomatoes from my garden.", - None, - &[], - ); - - assert!(facets.contains(&"type:ingredient".to_string())); - assert!(facets.contains(&"type:homegrown".to_string())); - - let unrelated = extract_memory_facets( - "User: I'm looking for inspiration for new cocktail ingredients this weekend.", - None, - &[], - ); - - assert!(unrelated.contains(&"type:ingredient".to_string())); - assert!(!unrelated.contains(&"type:homegrown".to_string())); -} - -#[test] -fn extracts_list_facets_from_inline_numbered_lists() { - let facets = extract_memory_facets( - "Assistant: 1. Virtual customer service representative 2. Remote bookkeeper 3. Transcriptionist 4. Social media manager", - None, - &[], - ); - - assert!(facets.contains(&"has:list".to_string())); -} - -#[test] -fn extracts_navigation_facets_from_routes_transit_passes_and_apps() { - let route = extract_memory_facets( - "How do I get to Shinjuku Station from Narita Airport using my Suica card?", - None, - &[], - ); - assert!(route.contains(&"type:navigation".to_string())); - assert!(route.contains(&"travel:route".to_string())); - assert!(route.contains(&"travel:station".to_string())); - assert!(route.contains(&"travel:pass".to_string())); - - let transit = extract_memory_facets( - "Take the train from Union Station, transfer to the metro, and check the fare before you go.", - None, - &[], - ); - assert!(transit.contains(&"type:navigation".to_string())); - assert!(transit.contains(&"travel:transit".to_string())); - assert!(transit.contains(&"travel:station".to_string())); - assert!(transit.contains(&"travel:fare".to_string())); - - let app = extract_memory_facets( - "I downloaded a travel app to keep my tour meeting point and itinerary organized.", - None, - &[], - ); - assert!(app.contains(&"type:navigation".to_string())); - assert!(app.contains(&"travel:route".to_string())); - assert!(app.contains(&"travel:app".to_string())); -} - -#[test] -fn generic_location_recommendations_do_not_get_navigation_facets() { - let facets = extract_memory_facets( - "Can you recommend some good restaurants near the Park Hyatt Tokyo?", - None, - &[], - ); - - assert!(!facets.contains(&"type:navigation".to_string())); - assert!(!facets.iter().any(|facet| facet.starts_with("travel:"))); -} - -#[test] -fn extracts_age_and_education_facets_from_structured_age_language() { - let current_age = extract_memory_facets( - "As a 32-year-old Digital Marketing Specialist, I'm considering an MBA.", - None, - &[], - ); - assert!(current_age.contains(&"has:age".to_string())); - assert!(current_age.contains(&"age:current".to_string())); - assert!(!current_age.contains(&"age:event".to_string())); - - let graduation_age = extract_memory_facets( - "I have a Bachelor's degree from the University of California, which I completed at the age of 25.", - None, - &[], - ); - assert!(graduation_age.contains(&"has:age".to_string())); - assert!(graduation_age.contains(&"age:event".to_string())); - assert!(graduation_age.contains(&"education:degree".to_string())); - assert!(graduation_age.contains(&"education:college".to_string())); - assert!(graduation_age.contains(&"education:graduation".to_string())); - assert!(graduation_age.contains(&"education:undergraduate".to_string())); -} - -#[test] -fn extracts_undergraduate_education_facets_from_undergrad_language() { - let facets = extract_memory_facets( - "User: I completed my undergrad in CS from UCLA before moving to Seattle.", - None, - &[], - ); - - assert!(facets.contains(&"education:degree".to_string())); - assert!(facets.contains(&"education:undergraduate".to_string())); - assert!(facets.contains(&"education:graduation".to_string())); - assert!(facets.contains(&"entity:cs".to_string())); - assert!(facets.contains(&"entity:ucla".to_string())); -} - -#[test] -fn bachelors_degree_query_targets_undergraduate_facets_and_initialisms() { - let available = |cue: &str| { - matches!( - cue, - "education:degree" - | "education:undergraduate" - | "education:graduation" - | "source_role:user" - | "entity:computer_science" - | "entity:cs" - ) - }; - - let intent = compile_query_intent( - "Where did I complete my Bachelor's degree in Computer Science?", - available, - ); - - assert!(intent.labels.contains(&"education_query".to_string())); - for expected in [ - "education:degree", - "education:undergraduate", - "education:graduation", - "source_role:user", - "entity:computer_science", - "entity:cs", - ] { - assert!( - intent.weighted_cues.iter().any(|(cue, _)| cue == expected), - "missing education query cue {expected}" - ); - } -} - -#[test] -fn extracts_family_relation_facets_from_self_scoped_sibling_facts() { - let sisters = extract_memory_facets( - "I come from a family with 3 sisters, so I have always had a strong female presence in my life.", - None, - &[], - ); - assert!(sisters.contains(&"has:number".to_string())); - assert!(sisters.contains(&"family_relation:sibling".to_string())); - assert!(sisters.contains(&"sibling_kind:sister".to_string())); - assert!(sisters.contains(&"family_scope:self".to_string())); - assert!(sisters.contains(&"family_count:sibling".to_string())); - - let brother = extract_memory_facets( - "I should mention that I have a brother, which might influence my social circle dynamics.", - None, - &[], - ); - assert!(brother.contains(&"family_relation:sibling".to_string())); - assert!(brother.contains(&"sibling_kind:brother".to_string())); - assert!(brother.contains(&"family_scope:self".to_string())); - assert!(brother.contains(&"family_count:sibling".to_string())); - - let movie = extract_memory_facets( - "The film follows twin siblings who uncover a family secret.", - None, - &[], - ); - assert!(movie.contains(&"family_relation:sibling".to_string())); - assert!(!movie.contains(&"family_scope:self".to_string())); - assert!(!movie.contains(&"family_count:sibling".to_string())); -} - -#[test] -fn extracts_co_residence_facets_from_staying_with_self_language() { - let facets = extract_memory_facets( - "User: My parents have been a big help; they've been staying with me for nine months now.", - None, - &[], - ); - - assert!(facets.contains(&"family_relation:parent".to_string())); - assert!(facets.contains(&"co_residence:with_self".to_string())); - assert!(facets.contains(&"has:duration".to_string())); -} - -#[test] -fn extracts_update_facets_from_discourse_markers() { - let correction = - extract_memory_facets("I'm actually planning to stay on Oahu instead.", None, &[]); - let switch = extract_memory_facets( - "I have just wrapped up a model and switched to a Ford F-150 pickup truck.", - None, - &[], - ); - - assert!(correction.contains(&"type:update".to_string())); - assert!(switch.contains(&"type:update".to_string())); -} - -#[test] -fn does_not_extract_question_words_or_source_labels_as_entities() { - let facets = extract_memory_facets( - "User: What breed is Max? Any tips? Assistant: Max is a Golden Retriever.", - None, - &[], - ); - - assert!(facets.contains(&"source_role:user".to_string())); - assert!(facets.contains(&"entity:max".to_string())); - assert!(facets.contains(&"entity:golden_retriever".to_string())); - assert!(!facets.contains(&"entity:what".to_string())); - assert!(!facets.contains(&"entity:any".to_string())); - assert!(!facets.contains(&"entity:user".to_string())); - assert!(!facets.contains(&"entity:assistant".to_string())); -} - -#[test] -fn query_intent_does_not_treat_stopword_sentence_openers_as_entities() { - let available = |cue: &str| matches!(cue, "entity:any" | "entity:max"); - let intent = compile_query_intent("Any tips for Max?", available); - - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "entity:any")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "entity:max")); -} - -#[test] -fn extracts_person_role_facets_from_titles_and_role_phrases() { - let facets = extract_memory_facets( - "I saw Dr. Patel for chronic sinusitis. My primary care physician Dr. Smith prescribed antibiotics, dermatologist Dr. Lee handled the biopsy, and project manager Alice approved the plan.", - None, - &[], - ); - - assert!(facets.contains(&"person_title:dr".to_string())); - assert!(facets.contains(&"person_ref:named".to_string())); - assert!(facets.contains(&"person_role_phrase:primary_care_physician".to_string())); - assert!(facets.contains(&"person_role_phrase:dermatologist".to_string())); - assert!(facets.contains(&"person_role_phrase:project_manager".to_string())); - let role_phrases = facets - .iter() - .filter(|facet| facet.starts_with("person_role_phrase:")) - .cloned() - .collect::<HashSet<_>>(); - assert_eq!( - role_phrases, - HashSet::from([ - "person_role_phrase:dermatologist".to_string(), - "person_role_phrase:primary_care_physician".to_string(), - "person_role_phrase:project_manager".to_string(), - ]) - ); -} - -#[test] -fn person_role_facets_reject_clause_false_positives() { - for content in [ - "I cut the wood with a saw.", - "I cut the wood with a saw before Maya arrived.", - "I cut wooden boards with Alice.", - "I used a circular power saw beside Maya.", - "My friend saw Dr. Patel yesterday.", - "My neighbor visited Dr. Smith last week.", - "Ensure we only search the remaining HTML.", - ] { - let facets = extract_memory_facets(content, None, &[]); - assert!( - !facets - .iter() - .any(|facet| facet.starts_with("person_role_phrase:")), - "false role facet for {content:?}: {facets:?}" - ); - } - - let facets = extract_memory_facets("I met project manager Alice yesterday.", None, &[]); - assert!( - !facets.contains(&"person_role_phrase:met_project_manager".to_string()), - "clause verb leaked into role phrase: {facets:?}" - ); -} - -#[test] -fn person_role_facets_preserve_content_words_and_complete_phrases() { - for (content, expected) in [ - ( - "Circular saw specialist Maya arrived.", - "person_role_phrase:circular_saw_specialist", - ), - ( - "Head of product Maya approved the launch.", - "person_role_phrase:head_of_product", - ), - ( - "Chief information security officer Dr. Smith approved it.", - "person_role_phrase:chief_information_security_officer", - ), - ( - "My director of operations Dr. Smith approved it.", - "person_role_phrase:director_of_operations", - ), - ] { - let facets = extract_memory_facets(content, None, &[]); - assert!( - facets.contains(&expected.to_string()), - "missing {expected} for {content:?}: {facets:?}" - ); - } -} - -#[test] -fn extracts_numeric_object_facets_from_quantity_syntax() { - let facets = extract_memory_facets( - "I currently have 2 monitors, 3 laptops, and a 20-gallon freshwater community tank named Amazonia.", - None, - &[], - ); - - assert!(facets.contains(&"type:ownership".to_string())); - assert!(facets.contains(&"quantity_object:monitor".to_string())); - assert!(facets.contains(&"quantity_object:laptop".to_string())); - assert!(facets.contains(&"quantity_object:tank".to_string())); - assert!(facets.contains(&"quantity_count:object".to_string())); - assert!(facets.contains(&"inventory_count:contained".to_string())); - assert!(facets.contains(&"quantity_unit:gallon".to_string())); - assert!(facets.contains(&"quantity_unit_object:gallon_tank".to_string())); - assert!(facets.contains(&"inventory_object:monitor".to_string())); - assert!(facets.contains(&"inventory_object:tank".to_string())); - assert!(!facets.contains(&"quantity_object:week".to_string())); -} - -#[test] -fn extracts_completion_count_facets_from_word_number_syntax() { - let facets = extract_memory_facets( - "User: I've completed three courses on Coursera and finished two workshops last month.", - None, - &[], - ); - - assert!(facets.contains(&"quantity_count:object".to_string())); - assert!(facets.contains(&"completion_count:object".to_string())); - assert!(facets.contains(&"quantity_object:course".to_string())); - assert!(facets.contains(&"quantity_object:workshop".to_string())); -} - -#[test] -fn completion_count_query_weights_completed_quantity_evidence() { - let available = |cue: &str| { - matches!( - cue, - "completion_count:object" | "quantity_count:object" | "quantity_object:course" - ) - }; - let intent = compile_query_intent( - "How many online courses have I completed in total?", - available, - ); - - assert!(intent.labels.contains(&"completion_count".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "completion_count:object" && *weight > 7.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "quantity_object:course" && *weight > 4.0)); -} - -#[test] -fn count_query_with_possessive_collection_is_inventory_intent() { - let available = |cue: &str| { - matches!( - cue, - "quantity_count:object" - | "inventory_object:fish" - | "quantity_object:fish" - | "inventory_count:contained" - | "type:ownership" - | "source_role:user" - ) - }; - let intent = compile_query_intent( - "How many fish are there in total in both of my aquariums?", - available, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent.labels.contains(&"inventory".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "quantity_count:object" && *weight > 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "inventory_count:contained" && *weight > 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "inventory_object:fish" && *weight > 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 3.0)); -} - -#[test] -fn contained_singular_inventory_item_counts_as_quantity_evidence() { - let facets = extract_memory_facets( - "I upgraded my old 10-gallon tank, which has my betta fish, Bubbles.", - None, - &[], - ); - - assert!(facets.contains(&"quantity_count:object".to_string())); - assert!(facets.contains(&"inventory_count:contained".to_string())); - assert!(facets.contains(&"quantity_object:betta".to_string())); - assert!(facets.contains(&"quantity_object:fish".to_string())); - - let generic = extract_memory_facets( - "I added decorations to create more hiding places for my fish.", - None, - &[], - ); - assert!(!generic.contains(&"quantity_count:object".to_string())); -} - -#[test] -fn extracts_project_work_from_first_person_work_patterns() { - let active_project = extract_memory_facets( - "User: I've been working on a solo project for my Data Mining class.", - None, - &[], - ); - assert!(active_project.contains(&"type:project_work".to_string())); - assert!(active_project.contains(&"type:activity".to_string())); - - let research = extract_memory_facets( - "User: I recently presented a poster on my research at an academic conference.", - None, - &[], - ); - assert!(research.contains(&"type:project_work".to_string())); - assert!(research.contains(&"type:activity".to_string())); - - let generic_advice = extract_memory_facets( - "Assistant: A project timeline should include milestones and dependencies.", - None, - &[], - ); - assert!(!generic_advice.contains(&"type:project_work".to_string())); -} - -#[test] -fn inventory_count_query_prefers_quantity_object_over_temporal_current() { - let available = |cue: &str| { - matches!( - cue, - "quantity_object:tank" - | "inventory_object:tank" - | "type:ownership" - | "source_role:user" - | "type:update" - | "has:number" - | "temporal:current" - | "temporal:recent" - ) - }; - let intent = compile_query_intent( - "How many tanks do I currently have, including the one I set up for my friend's kid?", - available, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent.labels.contains(&"inventory".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "inventory_object:tank" && *weight > 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "quantity_object:tank" && *weight > 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:ownership" && *weight > 2.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 3.0)); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:update")); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:current" || cue == "temporal:recent")); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, weight)| cue == "currently" && *weight < 1.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, weight)| cue == "one" && *weight < 1.0)); -} - -#[test] -fn project_count_query_prefers_project_work_facets() { - let available = |cue: &str| { - matches!( - cue, - "project" - | "type:project_work" - | "type:activity" - | "source_role:user" - | "type:update" - | "temporal:recent" - ) - }; - let intent = compile_query_intent( - "How many projects have I led or am currently leading?", - available, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent - .labels - .contains(&"project_work_count".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:project_work" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 3.0)); -} - -#[test] -fn event_count_query_prefers_counted_object_without_inventory_or_state_update() { - let available = |cue: &str| { - matches!( - cue, - "wedding" - | "type:ownership" - | "type:update" - | "type:activity" - | "type:event" - | "temporal:relative" - | "temporal:last_week" - | "temporal:recent" - | "source_year:2023" - ) - }; - let intent = compile_query_intent_with_reference_time( - "How many weddings have I attended in this year?", - Some("2023/10/15 (Sun) 23:47"), - available, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent.labels.contains(&"activity_event".to_string())); - assert!(intent.labels.contains(&"temporal_window".to_string())); - assert!(intent - .labels - .contains(&"temporal_resolved_year".to_string())); - assert!(!intent.labels.contains(&"inventory".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "wedding" && *weight >= 8.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_year:2023" && *weight >= 1.0)); - for rejected in [ - "type:ownership", - "type:update", - "temporal:relative", - "temporal:last_week", - "temporal:recent", - ] { - assert!( - !intent.weighted_cues.iter().any(|(cue, _)| cue == rejected), - "event count query should not inject {rejected}" - ); - } -} - -#[test] -fn age_difference_query_prefers_age_and_education_over_duration_temporal_state() { - let available = |cue: &str| { - matches!( - cue, - "has:number" - | "has:duration" - | "has:date" - | "has:age" - | "age:current" - | "age:event" - | "education:graduation" - | "education:degree" - | "education:college" - | "type:update" - | "temporal:recent" - | "temporal:last_week" - ) - }; - let intent = compile_query_intent( - "How many years older am I than when I graduated from college?", - available, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent.labels.contains(&"age_query".to_string())); - assert!(intent.labels.contains(&"age_difference".to_string())); - assert!(!intent.labels.contains(&"duration".to_string())); - assert!(!intent.labels.contains(&"temporal_window".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - - for expected in [ - "has:age", - "age:current", - "age:event", - "education:graduation", - "education:degree", - ] { - assert!( - intent.weighted_cues.iter().any(|(cue, _)| cue == expected), - "missing age-difference cue {expected}" - ); - } - for rejected in ["has:duration", "temporal:recent", "temporal:last_week", "type:update"] { - assert!( - !intent.weighted_cues.iter().any(|(cue, _)| cue == rejected), - "age-difference query should not inject {rejected}" - ); - } - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "year" && *multiplier < 1.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "old" && *multiplier < 1.0)); -} - -#[test] -fn sibling_count_query_prefers_family_facets_over_inventory_ownership() { - let available = |cue: &str| { - matches!( - cue, - "has:number" - | "type:ownership" - | "family_count:sibling" - | "family_scope:self" - | "family_relation:sibling" - | "sibling_kind:brother" - | "sibling_kind:sister" - ) - }; - let intent = compile_query_intent("What is the total number of siblings I have?", available); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent - .labels - .contains(&"family_relation_count".to_string())); - assert!(!intent.labels.contains(&"inventory".to_string())); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:ownership")); - for expected in [ - "family_count:sibling", - "family_scope:self", - "family_relation:sibling", - "sibling_kind:brother", - "sibling_kind:sister", - ] { - assert!( - intent.weighted_cues.iter().any(|(cue, _)| cue == expected), - "missing family cue {expected}" - ); - } -} - -#[test] -fn weekly_routine_count_queries_use_schedule_facets_not_last_week_temporal_window() { - let facets = extract_memory_facets( - "User: I usually take Zumba classes on Tuesdays and Thursdays at 7:00 PM.", - None, - &[], - ); - assert!(facets.contains(&"has:weekday".to_string())); - assert!(facets.contains(&"has:time".to_string())); - assert!(facets.contains(&"schedule:weekly".to_string())); - assert!(facets.contains(&"type:routine".to_string())); - - let available = |cue: &str| { - matches!( - cue, - "fitness" - | "class" - | "has:frequency" - | "schedule:frequency" - | "frequency_unit:week" - | "schedule:weekly" - | "has:weekday" - | "has:time" - | "type:routine" - | "type:activity" - | "source_role:user" - | "temporal:last_week" - | "type:update" - ) - }; - let intent = compile_query_intent( - "How many fitness classes do I attend in a typical week?", - available, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent.labels.contains(&"weekly_routine".to_string())); - assert!(!intent.labels.contains(&"temporal_window".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:last_week" || cue == "type:update")); - for expected in [ - "fitness", - "class", - "has:frequency", - "schedule:frequency", - "frequency_unit:week", - "schedule:weekly", - "has:weekday", - "type:routine", - ] { - assert!( - intent.weighted_cues.iter().any(|(cue, _)| cue == expected), - "missing weekly routine cue {expected}" - ); - } -} - -#[test] -fn family_duration_queries_weight_relation_and_co_residence_evidence() { - let available = |cue: &str| { - matches!( - cue, - "has:duration" - | "family_relation:parent" - | "co_residence:with_self" - | "source_role:user" - ) - }; - let intent = compile_query_intent( - "How long have my parents been staying with me?", - available, - ); - - assert!(intent.labels.contains(&"duration".to_string())); - assert!(intent.labels.contains(&"family_relation".to_string())); - assert!(intent.labels.contains(&"co_residence".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "family_relation:parent" && *weight >= 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "co_residence:with_self" && *weight >= 4.0)); -} - -#[test] -fn family_duration_query_prefers_family_stay_duration_over_unrelated_duration() { - let engine = CueMapEngine::new(); - - let add = |content: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let unrelated = "User: I'm a marketing specialist and have been doing it for about nine months."; - let legal = "User: I'll ask the attorney how they can help with my parents overstaying their visa."; - let target = "User: My parents have been a big help while living with me in the US for nine months."; - - add(unrelated); - add(legal); - add(target); - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "How long have my parents been staying with me in the US?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|result| result.content.as_str()), Some(target)); -} - -#[test] -fn weekday_schedule_queries_use_schedule_facets_not_relative_temporal_window() { - let available = |cue: &str| { - matches!( - cue, - "has:weekday" - | "schedule:weekly" - | "type:routine" - | "source_role:user" - | "temporal:last_week" - | "temporal:recent" - | "type:update" - ) - }; - let intent = compile_query_intent( - "What day of the week do I take a cocktail-making class?", - available, - ); - - assert!(intent.labels.contains(&"weekday_schedule".to_string())); - assert!(!intent.labels.contains(&"temporal_window".to_string())); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:last_week" || cue == "temporal:recent")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "has:weekday" && *weight >= 4.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, weight)| cue == "week" && *weight < 1.0)); -} - -#[test] -fn weekday_schedule_query_prefers_recurring_weekday_class_over_last_week_class() { - let engine = CueMapEngine::new(); - - let add = |content: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let last_week = "User: We made an Indian-inspired feast in my cooking class last week."; - let target = "User: I have a cocktail-making class on Fridays, so maybe I can experiment with tequila recipes there."; - let recipe = "User: I love tequila cocktails and want refreshing summer drink ideas."; - - add(last_week); - add(recipe); - add(target); - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "What day of the week do I take a cocktail-making class?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|result| result.content.as_str()), Some(target)); -} - -#[test] -fn temporal_distance_questions_do_not_default_to_recent_or_last_week() { - let intent = compile_query_intent( - "How many months ago did I attend the photography workshop?", - |cue| { - matches!( - cue, - "has:date" - | "temporal:relative" - | "temporal:last_week" - | "temporal:recent" - | "source_role:user" - ) - }, - ); - - assert!(intent.labels.contains(&"temporal_distance".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "has:date")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:relative")); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:last_week")); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:recent")); - for cue in ["month", "months", "ago"] { - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, multiplier)| adjusted == cue && *multiplier < 1.0)); - } -} - -#[test] -fn professional_role_query_adds_weighted_role_without_dropping_visit() { - let available = |cue: &str| matches!(cue, "dr" | "person_title:dr" | "person_ref:named"); - let intent = compile_query_intent("How many different doctors did I visit?", available); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent.weighted_cues.iter().any(|(cue, _)| cue == "dr")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "person_title:dr")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "person_ref:named")); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, weight)| cue == "many" && *weight < 1.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, weight)| cue == "different" && *weight < 1.0)); - assert!(!intent - .cue_weight_adjustments - .iter() - .any(|(cue, _)| cue == "visit")); -} - -#[test] -fn where_visit_query_keeps_visit_as_primary_cue() { - let intent = compile_query_intent("Where did I visit?", |_| false); - - assert!(!intent.labels.contains(&"count".to_string())); - assert!(!intent - .cue_weight_adjustments - .iter() - .any(|(cue, _)| cue == "visit")); -} - -#[test] -fn doctor_appointment_event_query_does_not_expand_to_doctor_title() { - let intent = compile_query_intent( - "What time did I go to bed on the day before I had a doctor's appointment?", - |cue| { - matches!( - cue, - "dr" - | "person_title:dr" - | "routine:bed_time" - | "has:time" - | "time_of_day:night" - | "source_role:user" - ) - }, - ); - - assert!(intent.labels.contains(&"bed_time".to_string())); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "dr" || cue == "person_title:dr")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "routine:bed_time" && *weight >= 4.0)); -} - -#[test] -fn title_abbreviation_query_expansion_is_not_doctor_specific() { - let available = |cue: &str| matches!(cue, "prof" | "person_title:prof" | "person_ref:named"); - let intent = compile_query_intent("How many professors did I meet?", available); - - assert!(intent.labels.contains(&"person_role".to_string())); - assert!(intent.weighted_cues.iter().any(|(cue, _)| cue == "prof")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "person_title:prof")); -} - -#[test] -fn present_state_queries_compile_update_intent_when_available() { - let available = |cue: &str| matches!(cue, "type:update" | "temporal:recent"); - let travel_intent = compile_query_intent( - "Where am I planning to stay for my birthday trip to Hawaii?", - available, - ); - let model_intent = compile_query_intent( - "What type of vehicle model am I currently working on?", - available, - ); - - assert!(travel_intent.labels.contains(&"state_update".to_string())); - assert!(travel_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:update" && *weight > 2.0)); - assert!(model_intent.labels.contains(&"state_update".to_string())); - assert!(model_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:update" && *weight > 2.0)); - assert!(model_intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "type" && *multiplier < 1.0)); -} - -#[test] -fn future_scheduled_advice_queries_do_not_compile_temporal_window() { - let available = |cue: &str| { - matches!( - cue, - "has:date" - | "temporal:relative" - | "temporal:last_week" - | "temporal:recent" - | "type:homegrown" - | "type:ingredient" - ) - }; - - for query in [ - "What should I serve for dinner this weekend with my homegrown ingredients?", - "I was thinking about rearranging the furniture in my bedroom this weekend. Any tips?", - "I'm getting excited about my visit to the music store this weekend. Any tips on what to look for in a new guitar?", - ] { - let intent = compile_query_intent(query, available); - assert!( - !intent.labels.contains(&"temporal_window".to_string()), - "future advice query was treated as temporal recall: {query}" - ); - assert!( - !intent.weighted_cues.iter().any(|(cue, _)| matches!( - cue.as_str(), - "has:date" | "temporal:relative" | "temporal:last_week" | "temporal:recent" - )), - "future advice query injected temporal facets: {query}" - ); - } - - let homegrown_intent = compile_query_intent( - "What should I serve for dinner this weekend with my homegrown ingredients?", - available, - ); - assert!(homegrown_intent.labels.contains(&"homegrown".to_string())); - assert!(homegrown_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:homegrown" && *weight > 3.0)); - assert!(homegrown_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:ingredient" && *weight > 2.0)); - assert!(homegrown_intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "weekend" && *multiplier < 1.0)); -} - -#[test] -fn past_time_window_queries_still_compile_temporal_window() { - let available = |cue: &str| { - matches!( - cue, - "has:date" | "temporal:relative" | "temporal:last_week" | "temporal:recent" - ) - }; - - for query in [ - "What gardening-related activity did I do two weeks ago?", - "What did I buy last weekend?", - ] { - let intent = compile_query_intent(query, available); - assert!( - intent.labels.contains(&"temporal_window".to_string()), - "past recall query lost temporal intent: {query}" - ); - assert!( - intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:relative" || cue == "temporal:last_week"), - "past recall query did not inject temporal facets: {query}" - ); - } -} - -#[test] -fn temporal_order_queries_do_not_become_latest_state_updates() { - let available = |cue: &str| { - matches!( - cue, - "source_time:dated" | "source_role:user" | "type:activity" | "type:event" - ) - }; - let intent = compile_query_intent( - "What is the order of the six museums I visited from earliest to latest?", - available, - ); - - assert!(intent.labels.contains(&"temporal_order".to_string())); - assert!(intent.labels.contains(&"activity_event".to_string())); - assert!(!intent.labels.contains(&"latest_current".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_time:dated" && *weight >= 2.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:activity" && *weight >= 3.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "latest" && *multiplier < 1.0)); -} - -#[test] -fn transport_mode_comparison_queries_target_concrete_transport_events() { - let available = |cue: &str| { - matches!( - cue, - "source_time:dated" - | "source_role:user" - | "type:activity" - | "type:event" - | "transport_event:bus" - | "transport_event:train" - | "transport_mode:bus" - | "transport_mode:train" - | "bus" - | "train" - | "temporal:recent" - | "type:update" - ) - }; - let intent = compile_query_intent( - "Which mode of transport did I use most recently, a bus or a train?", - available, - ); - - assert!(intent - .labels - .contains(&"transport_mode_comparison".to_string())); - assert!(!intent.labels.contains(&"latest_current".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "transport_event:bus" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "transport_event:train" && *weight >= 4.0)); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:update")); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "recently" && *multiplier < 1.0)); -} - -#[test] -fn relative_temporal_query_resolves_against_reference_time() { - let available = |cue: &str| { - matches!( - cue, - "has:date" - | "has:duration" - | "temporal:relative" - | "temporal:last_week" - | "temporal:recent" - | "temporal:today" - | "source_date:2023_04_21" - | "source_week:2023_w16" - | "source_month:2023_04" - ) - }; - let intent = compile_query_intent_with_reference_time( - "What gardening-related activity did I do two weeks ago?", - Some("2023/05/05 (Fri) 16:42"), - available, - ); - - assert!(intent.labels.contains(&"temporal_window".to_string())); - assert!(intent - .labels - .contains(&"temporal_resolved_date".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_date:2023_04_21" && *weight >= 1.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_week:2023_w16")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:today")); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "ago" && *multiplier < 1.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "two" && *multiplier < 1.0)); -} - -#[test] -fn charity_event_temporal_query_targets_exact_date_and_domain() { - let available = |cue: &str| { - matches!( - cue, - "source_date:2023_03_19" - | "source_week:2023_w11" - | "source_month:2023_03" - | "temporal:today" - | "type:activity" - | "type:event" - | "source_role:user" - | "event_domain:charity" - ) - }; - let intent = compile_query_intent_with_reference_time( - "What charity event did I participate in a month ago?", - Some("2023/04/18 (Tue) 18:34"), - available, - ); - - assert!(intent.labels.contains(&"temporal_window".to_string())); - assert!(intent - .labels - .contains(&"temporal_resolved_date".to_string())); - assert!(intent.labels.contains(&"charity_event".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_date:2023_03_19" && *weight >= 1.8)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "event_domain:charity" && *weight >= 3.0)); -} - -#[test] -fn last_week_query_targets_source_week_not_single_monday() { - let available = |cue: &str| { - matches!( - cue, - "source_date:2023_04_03" - | "source_week:2023_w14" - | "source_month:2023_04" - | "type:activity" - | "type:event" - | "source_role:user" - | "activity_domain:religion" - | "topic:religion" - ) - }; - let intent = compile_query_intent_with_reference_time( - "Where did I attend the religious activity last week?", - Some("2023/04/10 (Mon) 12:00"), - available, - ); - - assert!(intent.labels.contains(&"temporal_resolved_week".to_string())); - assert!(intent.labels.contains(&"activity_event".to_string())); - assert!(intent.labels.contains(&"religious_activity".to_string())); - assert!(!intent - .labels - .contains(&"temporal_resolved_date".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_week:2023_w14" && *weight >= 1.8)); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_date:2023_04_03")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:activity" && *weight >= 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "activity_domain:religion" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 1.0)); -} - -#[test] -fn past_weekend_query_targets_weekend_source_dates() { - let available = |cue: &str| { - matches!( - cue, - "source_date:2023_03_18" - | "source_date:2023_03_19" - | "source_week:2023_w11" - | "temporal:relative" - | "temporal:last_week" - | "temporal:recent" - ) - }; - let intent = compile_query_intent_with_reference_time( - "Which bike did I fixed or serviced the past weekend?", - Some("2023/03/21 (Tue) 21:43"), - available, - ); - - assert!(intent.labels.contains(&"temporal_window".to_string())); - assert!(intent - .labels - .contains(&"temporal_resolved_weekend".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_date:2023_03_18" && *weight >= 1.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_date:2023_03_19" && *weight >= 1.0)); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "temporal:last_week" || cue == "temporal:recent")); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "past" && *multiplier < 1.0)); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "weekend" && *multiplier < 1.0)); -} - -#[test] -fn past_month_query_targets_previous_source_month() { - let available = |cue: &str| { - matches!( - cue, - "source_month:2023_06" - | "source_month:2023_07" - | "source_year:2023" - | "type:competition_event" - | "activity_domain:sport" - | "type:activity" - | "type:event" - | "source_role:user" - ) - }; - let intent = compile_query_intent_with_reference_time( - "What is the order of the three sports events I participated in during the past month, from earliest to latest?", - Some("2023/07/01 (Sat) 20:43"), - available, - ); - - assert!(intent.labels.contains(&"temporal_order".to_string())); - assert!(intent - .labels - .contains(&"temporal_resolved_month".to_string())); - assert!(intent.labels.contains(&"competition_event".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_month:2023_06" && *weight >= 1.5)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:competition_event")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "activity_domain:sport")); - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(cue, multiplier)| cue == "participated" && *multiplier > 1.0)); -} - -#[test] -fn streaming_service_query_targets_usage_not_global_recency() { - let available = |cue: &str| { - matches!( - cue, - "media:streaming" - | "media:music_streaming" - | "media:music" - | "media:watching" - | "type:usage" - | "source_role:user" - | "type:update" - | "temporal:recent" - ) - }; - let intent = compile_query_intent( - "Which streaming service did I start using most recently?", - available, - ); - - assert!(intent - .labels - .contains(&"streaming_service_usage".to_string())); - assert!(!intent.labels.contains(&"latest_current".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "media:streaming" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:usage" && *weight >= 3.0)); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:update")); - - let music_intent = compile_query_intent( - "What is the name of the music streaming service have I been using lately?", - available, - ); - assert!(music_intent - .labels - .contains(&"music_streaming_service_usage".to_string())); - assert!(music_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "media:music_streaming" && *weight >= 4.0)); - assert!(music_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "media:music")); -} - -#[test] -fn wake_time_routine_query_targets_time_weekday_and_user_routine() { - let available = |cue: &str| { - matches!( - cue, - "routine:wake_time" - | "has:time" - | "has:weekday" - | "type:routine" - | "source_role:user" - | "type:update" - ) - }; - let intent = compile_query_intent("What time do I wake up on Saturday mornings?", available); - - assert!(intent.labels.contains(&"wake_time_routine".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "routine:wake_time" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "has:time" && *weight >= 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "has:weekday" && *weight >= 2.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:routine" && *weight >= 2.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 2.0)); -} - -#[test] -fn current_reading_query_targets_book_reading_state() { - let available = |cue: &str| { - matches!( - cue, - "reading:current" - | "media:book_reading" - | "media:book" - | "source_role:user" - | "temporal:current" - | "type:update" - ) - }; - let intent = compile_query_intent("What book am I currently reading?", available); - - assert!(intent.labels.contains(&"current_reading".to_string())); - assert!(!intent.labels.contains(&"state_update".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "reading:current" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "media:book_reading" && *weight >= 4.0)); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:update")); -} - -#[test] -fn personal_attribute_queries_target_entity_attribute_facets() { - let available = |cue: &str| { - matches!( - cue, - "type:entity_attribute" - | "attribute:class_relation" - | "source_role:user" - | "type:ownership" - ) - }; - let intent = compile_query_intent("What breed is my dog?", available); - - assert!(intent.labels.contains(&"entity_attribute".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:entity_attribute" && *weight >= 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "attribute:class_relation" && *weight >= 2.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 1.0)); - - let preference_intent = compile_query_intent("What brand are my favorite running shoes?", |cue| { - matches!( - cue, - "type:entity_attribute" - | "attribute:class_relation" - | "source_role:user" - | "type:preference" - ) - }); - assert!(preference_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 3.0)); -} - -#[test] -fn personal_attribute_recall_prefers_class_relation_over_generic_object_mentions() { - let engine = CueMapEngine::new(); - let dog_walker = "User: I need help finding a good dog walker in my area."; - let dog_toy = "User: I'm also thinking of getting Max a new toy, something interactive for dogs."; - let breed_memory = "User: I'm thinking of getting Max a new collar with a nice name tag. Do you have any recommendations for a good collar brand or type that would suit a Golden Retriever like Max?"; - - for content in [dog_walker, dog_toy, breed_memory] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "What breed is my dog?"), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(breed_memory)); -} - -#[test] -fn favorite_attribute_recall_prefers_user_fact_over_generic_advice() { - let engine = CueMapEngine::new(); - let advice = - "Assistant: Nike is a great choice for running shoes, and there are many brands to compare."; - let generic = - "Assistant: When buying gym shoes, compare brand, fit, cushioning, and durability."; - let target = - "User: Nike has been my favourite brand so far for running shoes."; - - for content in [advice, generic, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "What brand are my favorite running shoes?"), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn purchase_source_query_weights_acquired_object_and_source_facet() { - let intent = compile_query_intent( - "Where did I buy my new tennis racket from?", - |cue| { - matches!( - cue, - "tennis" - | "racket" - | "tennis_racket" - | "purchase:source" - | "type:ownership" - | "type:activity" - | "type:event" - | "source_role:user" - ) - }, - ); - - assert!(intent.labels.contains(&"purchase".to_string())); - assert!(intent.labels.contains(&"purchase_source".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "tennis_racket" && *weight >= 4.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "purchase:source" && *weight >= 3.0)); -} - -#[test] -fn received_from_whom_query_uses_acquisition_source_intent() { - let available = |cue: &str| { - matches!( - cue, - "purchase:source" | "type:ownership" | "type:activity" | "type:event" | "source_role:user" - ) - }; - let intent = compile_query_intent( - "I received a piece of jewelry last Saturday from whom?", - available, - ); - - assert!(intent.labels.contains(&"purchase".to_string())); - assert!(intent.labels.contains(&"purchase_source".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "purchase:source" && *weight > 3.0)); -} - -#[test] -fn purchase_query_weights_actual_acquisition_over_same_date_activity() { - let available = |cue: &str| { - matches!( - cue, - "purchase:acquired" - | "type:ownership" - | "type:activity" - | "type:event" - | "source_role:user" - ) - }; - let intent = compile_query_intent( - "I mentioned an investment for a competition four weeks ago. What did I buy?", - available, - ); - - assert!(intent.labels.contains(&"purchase".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "purchase:acquired" && *weight > 4.0)); -} - -#[test] -fn purchase_query_ranks_acquired_item_over_temporal_activity() { - let engine = CueMapEngine::new(); - let distractor = "User: We've been cooking a lot of Indian dishes, and last Sunday, we made chicken biryani."; - let target = "User: I actually got my own set of sculpting tools, including a modeling tool set, a wire cutter, and a sculpting mat today."; - - for content in [distractor, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let query = "I mentioned an investment for a competition four weeks ago. What did I buy?"; - let mut weighted_cues: Vec<(String, f64)> = - tokenize_to_cues(query).into_iter().map(|cue| (cue, 1.0)).collect(); - let intent = compile_query_intent(query, |cue| engine.get_cue_frequency(cue) > 0); - for (cue, weight) in intent.weighted_cues { - if let Some((_, existing)) = weighted_cues.iter_mut().find(|(existing, _)| existing == &cue) - { - if *existing < weight { - *existing = weight; - } - } else { - weighted_cues.push((cue, weight)); - } - } - - let results = engine.recall_weighted( - weighted_cues, - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn acquisition_source_query_ranks_source_memory_over_temporal_distractor() { - let engine = CueMapEngine::new(); - let distractor = "User: Last Saturday I hit a new personal best of 12,345 steps while running errands."; - let target = "User: I also got a stunning crystal chandelier from my aunt today, which used to belong to my great-grandmother."; - - for content in [distractor, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let query = "I received a piece last Saturday from whom?"; - let mut weighted_cues: Vec<(String, f64)> = - tokenize_to_cues(query).into_iter().map(|cue| (cue, 1.0)).collect(); - let intent = compile_query_intent(query, |cue| engine.get_cue_frequency(cue) > 0); - for (cue, weight) in intent.weighted_cues { - if let Some((_, existing)) = weighted_cues.iter_mut().find(|(existing, _)| existing == &cue) - { - if *existing < weight { - *existing = weight; - } - } else { - weighted_cues.push((cue, weight)); - } - } - - let results = engine.recall_weighted( - weighted_cues, - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn purchase_source_recall_prefers_matching_item_over_other_purchases() { - let engine = CueMapEngine::new(); - let other_purchase = - "User: I recently bought an eyeshadow palette at Sephora and earned loyalty points."; - let tennis_followup = - "User: I'll check the weather forecast online. Do you know warmups before playing tennis?"; - let target = - "User: I'm really happy with my new tennis racket, which I got from a sports store downtown."; - - for content in [other_purchase, tennis_followup, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "Where did I buy my new tennis racket from?"), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn purchase_source_recall_handles_item_is_from_source_statement() { - let engine = CueMapEngine::new(); - let other_purchase = - "User: I bought old light bulbs from Home Depot and checked for spares."; - let generic_bookshelf = - "User: I got a new bookshelf, which helped me declutter the living room."; - let target = - "User: The new bookshelf is from IKEA, and I'm really happy with it."; - - for content in [other_purchase, generic_bookshelf, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "Where did I buy my new bookshelf from?"), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn wake_time_routine_recall_prefers_timed_user_routine_over_weather_mentions() { - let engine = CueMapEngine::new(); - let weather = "User: I'm planning to go for a jog on Saturday morning, what's the weather forecast like?"; - let assistant_schedule = "Assistant: Your desired wake-up times include a consistent bedtime and Saturday morning routine."; - let target = "User: I've been waking up around 8:30 am on Saturdays, which gives me enough time to fit in a 30-minute jog."; - - for content in [weather, assistant_schedule, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "What time do I wake up on Saturday mornings?"), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn bed_time_query_prefers_timed_bed_memory_over_doctor_title_mentions() { - let engine = CueMapEngine::new(); - let doctor_lecture = - "User: I attended a lecture series downtown where the speaker, Dr. Khan, discussed stress."; - let appointment = - "User: I had a doctor's appointment at 10 AM last Thursday, and that's when I got my blood test results."; - let target = - "User: I'm feeling sluggish because I didn't get to bed until 2 AM last Wednesday."; - - for content in [doctor_lecture, appointment, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "What time did I go to bed on the day before I had a doctor's appointment?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn media_watch_recommendations_target_media_memories() { - let available = |cue: &str| matches!(cue, "media:watching" | "source_role:user"); - let intent = compile_query_intent( - "Can you recommend a show or movie for me to watch tonight?", - available, - ); - - assert!(intent.labels.contains(&"recommendation".to_string())); - assert!(intent - .labels - .contains(&"media_watch_recommendation".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "media:watching" && *weight >= 3.0)); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| matches!(cue.as_str(), "show" | "movie" | "watch"))); -} - -#[test] -fn inspiration_recommendation_prefers_source_memory_over_same_topic_advice() { - let engine = CueMapEngine::new(); - let pricing = "User: I'm having trouble pricing my paintings online and want advice on setting fair prices."; - let flowers = "User: I saw flower paintings on Instagram and asked for tips to paint realistic flowers."; - let texture = "User: I've been trying to add texture into my paintings with palette knives."; - let target = "User: I have been getting inspiration from social media and recently started a 30-day painting challenge."; - - for content in [pricing, flowers, texture, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "I've been feeling a bit stuck with my paintings lately. Do you have any ideas on how I can find new inspiration?", - ), - 4, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn compiles_query_intent_only_for_available_facets() { - let available = |cue: &str| matches!(cue, "has:number" | "has:money" | "type:preference"); - let intent = compile_query_intent( - "How much money did I spend on my favorite camera?", - available, - ); - - assert!(intent.labels.contains(&"money".to_string())); - assert!(intent.labels.contains(&"preference".to_string())); - assert!(intent.suppress_generic); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "has:money" && *weight > 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:preference")); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "has:duration")); -} - -#[test] -fn how_much_time_is_duration_not_money_intent() { - let available = |cue: &str| matches!(cue, "has:number" | "has:money" | "has:duration"); - let intent = compile_query_intent( - "How much time do I dedicate to practicing guitar every day?", - available, - ); - - assert!(intent.labels.contains(&"duration".to_string())); - assert!(!intent.labels.contains(&"money".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "has:duration")); - assert!(!intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "has:money")); -} - -#[test] -fn activity_duration_queries_keep_duration_intent_with_temporal_windows() { - let available = |cue: &str| { - matches!( - cue, - "has:number" | "has:duration" | "jog" | "yoga" | "source_week:2023_w21" - ) - }; - let intent = compile_query_intent_with_reference_time( - "How many hours of jogging and yoga did I do last week?", - Some("2023/05/30 (Tue) 21:24"), - available, - ); - - assert!(intent.labels.contains(&"duration".to_string())); - assert!(intent.labels.contains(&"temporal_window".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "has:duration")); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "jog" && *weight >= 6.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "yoga" && *weight >= 6.0)); -} - -#[test] -fn question_words_are_not_query_entity_intents() { - let available = |cue: &str| { - matches!( - cue, - "entity:what" | "entity:how" | "entity:user" | "entity:assistant" - ) - }; - - for query in [ - "What degree did I graduate with?", - "How much time do I practice guitar?", - "Assistant: what did you recommend?", - ] { - let intent = compile_query_intent(query, available); - assert!( - !intent.weighted_cues.iter().any(|(cue, _)| matches!( - cue.as_str(), - "entity:what" | "entity:how" | "entity:user" | "entity:assistant" - )), - "generic entity leaked for query: {query}" - ); - } -} - -#[test] -fn conversational_source_intent_uses_structured_roles_when_available() { - let assistant_available = - |cue: &str| matches!(cue, "source_role:assistant" | "type:recommendation"); - let assistant_intent = compile_query_intent( - "You mentioned a store and recommended a fabric supplier. What was it?", - assistant_available, - ); - - assert!(assistant_intent - .labels - .contains(&"source_assistant".to_string())); - assert!(assistant_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:assistant")); - - let say_intent = compile_query_intent( - "How many eggs did you say we need for the recipe?", - assistant_available, - ); - assert!(say_intent.labels.contains(&"source_answer".to_string())); - assert!(say_intent - .labels - .contains(&"source_assistant".to_string())); - assert!(say_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:assistant")); - - let remind_intent = compile_query_intent( - "Could you remind me of the name of that restaurant in Cihampelas Walk?", - assistant_available, - ); - assert!(remind_intent.labels.contains(&"source_answer".to_string())); - assert!(remind_intent - .labels - .contains(&"source_assistant".to_string())); - assert!(remind_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:assistant")); - - let created_available = |cue: &str| { - matches!( - cue, - "source_role:assistant" | "type:answer" | "type:iteration" - ) - }; - let created_intent = compile_query_intent( - "Can you remind me what was in the second version you created?", - created_available, - ); - assert!(created_intent - .labels - .contains(&"source_answer".to_string())); - assert!(created_intent - .labels - .contains(&"source_assistant".to_string())); - assert!(created_intent - .labels - .contains(&"iteration_reference".to_string())); - assert!(created_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:iteration")); - - let request_intent = compile_query_intent( - "Can you suggest some accessories that would complement my current photography setup?", - assistant_available, - ); - assert!(request_intent - .labels - .contains(&"recommendation".to_string())); - assert!(!request_intent.labels.contains(&"source_answer".to_string())); - assert!(!request_intent - .labels - .contains(&"source_assistant".to_string())); - assert!(!request_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:assistant")); - - let profile_available = |cue: &str| { - matches!( - cue, - "source_role:user" - | "type:recommendation" - | "type:preference" - | "type:ownership" - | "type:usage" - | "type:update" - | "photography" - | "complement" - ) - }; - let personal_request_intent = compile_query_intent( - "Can you suggest some accessories that would complement my current photography setup?", - profile_available, - ); - assert!(personal_request_intent - .labels - .contains(&"personal_recommendation_context".to_string())); - assert!(!personal_request_intent - .labels - .contains(&"state_update".to_string())); - assert!(personal_request_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:user")); - assert!(personal_request_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:ownership")); - - let purchase_consideration_available = |cue: &str| { - matches!( - cue, - "source_role:user" - | "type:purchase_consideration" - | "type:ownership" - | "type:preference" - | "guitar" - ) - }; - let purchase_consideration_intent = compile_query_intent( - "Any tips on what to look for in a new guitar?", - purchase_consideration_available, - ); - assert!(purchase_consideration_intent - .labels - .contains(&"purchase_consideration".to_string())); - assert!(purchase_consideration_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:purchase_consideration")); - - let research_interest_available = |cue: &str| { - matches!( - cue, - "source_role:user" - | "type:preference" - | "type:interest" - | "type:expertise" - | "publication" - | "conference" - | "might" - | "find" - | "interest" - ) - }; - let research_interest_intent = compile_query_intent( - "Can you recommend some recent publications or conferences that I might find interesting?", - research_interest_available, - ); - assert!(research_interest_intent - .labels - .contains(&"vague_interest_recommendation".to_string())); - assert!(research_interest_intent - .labels - .contains(&"research_interest_recommendation".to_string())); - assert!(research_interest_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:expertise")); - assert!(research_interest_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:user")); - for cue in ["publication", "conference", "might", "find", "interest"] { - assert!( - !research_interest_intent - .weighted_cues - .iter() - .any(|(weighted, _)| weighted == cue), - "vague interest recommendation should not treat {cue} as a profile topic" - ); - } - - let inspiration_available = |cue: &str| { - matches!( - cue, - "source_role:user" - | "type:inspiration_source" - | "type:interest" - | "type:recommendation" - | "type:ownership" - | "painting" - | "inspiration" - ) - }; - let inspiration_intent = compile_query_intent( - "I've been feeling a bit stuck with my paintings lately. Do you have any ideas on how I can find new inspiration?", - inspiration_available, - ); - assert!(inspiration_intent - .labels - .contains(&"inspiration_recommendation".to_string())); - assert!(inspiration_intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "type:inspiration_source" && *weight >= 4.0)); - assert!(inspiration_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:interest")); - - let decision_available = |cue: &str| { - matches!( - cue, - "type:decision" | "type:selection" | "type:naming" - ) - }; - let decision_intent = compile_query_intent( - "What did we finally decide to name it?", - decision_available, - ); - assert!(decision_intent - .labels - .contains(&"decision_selection".to_string())); - assert!(decision_intent - .labels - .contains(&"naming_decision".to_string())); - assert!(decision_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:decision")); - assert!(decision_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "type:naming")); - - let user_available = |cue: &str| matches!(cue, "source_role:user" | "has:date"); - let user_intent = compile_query_intent("What did I mention last week?", user_available); - - assert!(user_intent.labels.contains(&"source_user".to_string())); - assert!(user_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:user")); - - let brought_up_intent = compile_query_intent( - "Can you list the order in which I brought up the deployment issues?", - user_available, - ); - assert!(brought_up_intent - .labels - .contains(&"source_user".to_string())); - assert!(brought_up_intent - .weighted_cues - .iter() - .any(|(cue, _)| cue == "source_role:user")); -} - -#[test] -fn recommendation_queries_downweight_prompt_scaffolding_not_topic_terms() { - let intent = compile_query_intent( - "Can you suggest some useful accessories for my phone?", - |cue| matches!(cue, "type:recommendation" | "type:preference" | "phone"), - ); - - assert!(intent.labels.contains(&"recommendation".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "phone" && *weight >= 2.4)); - for cue in ["can", "suggest", "useful", "some"] { - assert!(intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, multiplier)| adjusted == cue && *multiplier < 1.0)); - } - assert!(!intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, _)| adjusted == "phone")); - assert!(!intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, _)| adjusted == "accessory")); - - let recipe_intent = compile_query_intent( - "I was thinking of trying a new coffee creamer recipe. Any recommendations?", - |cue| matches!(cue, "type:recipe" | "coffee" | "creamer" | "recipe" | "think" | "try"), - ); - - assert!(recipe_intent.labels.contains(&"recommendation".to_string())); - for cue in ["think", "try", "new", "recipe"] { - assert!(recipe_intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, multiplier)| adjusted == cue && *multiplier < 1.0)); - assert!(!recipe_intent - .weighted_cues - .iter() - .any(|(weighted, _)| weighted == cue)); - } - assert!(recipe_intent - .weighted_cues - .iter() - .any(|(weighted, _)| weighted == "type:recipe")); - assert!(recipe_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "creamer" && *weight >= 4.0)); - assert!(recipe_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "coffee" && *weight < 2.4)); - - let advice_intent = compile_query_intent( - "I'm a bit anxious about getting around Tokyo. Do you have any helpful tips?", - |cue| { - matches!( - cue, - "type:recommendation" - | "type:preference" - | "type:navigation" - | "travel:route" - | "tokyo" - | "helpful" - ) - }, - ); - - assert!(advice_intent.labels.contains(&"recommendation".to_string())); - assert!(advice_intent - .labels - .contains(&"personal_recommendation_context".to_string())); - assert!(advice_intent - .labels - .contains(&"navigation".to_string())); - assert!(!advice_intent - .weighted_cues - .iter() - .any(|(weighted, _)| weighted == "helpful")); - for cue in ["bit", "got", "around", "helpful", "tip"] { - assert!(advice_intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, multiplier)| adjusted == cue && *multiplier < 1.0)); - } - - let hotel_intent = compile_query_intent( - "Can you suggest a hotel for my upcoming trip to Miami?", - |cue| { - matches!( - cue, - "type:recommendation" - | "type:preference" - | "hotel" - | "upcoming" - | "trip" - | "miami" - ) - }, - ); - - assert!(hotel_intent.labels.contains(&"recommendation".to_string())); - assert!(hotel_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "hotel" && *weight >= 2.4)); - assert!(hotel_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "miami" && *weight >= 2.4)); - for cue in ["upcoming", "trip"] { - assert!(!hotel_intent - .weighted_cues - .iter() - .any(|(weighted, _)| weighted == cue)); - assert!(hotel_intent - .cue_weight_adjustments - .iter() - .any(|(adjusted, multiplier)| adjusted == cue && *multiplier < 1.0)); - } - - let cocktail_intent = compile_query_intent( - "I've been thinking about making a cocktail for an upcoming get-together, but I'm not sure which one to choose. Any suggestions?", - |cue| { - matches!( - cue, - "type:recommendation" - | "type:preference" - | "cocktail" - | "make" - | "choose" - | "sure" - | "one" - ) - }, - ); - - assert!(cocktail_intent.labels.contains(&"recommendation".to_string())); - assert!(cocktail_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "cocktail" && *weight >= 4.0)); - for cue in ["make", "choose", "sure", "one"] { - assert!( - !cocktail_intent - .weighted_cues - .iter() - .any(|(weighted, _)| weighted == cue), - "recommendation scaffold cue should not be a topic: {cue}" - ); - } - - let bake_intent = compile_query_intent( - "I'm thinking of inviting my colleagues over for a small gathering. Any tips on what to bake?", - |cue| matches!(cue, "gather" | "bake" | "colleague" | "type:recipe"), - ); - - assert!(bake_intent.labels.contains(&"recommendation".to_string())); - assert!(bake_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "bake" && *weight >= 4.0)); - assert!(bake_intent - .weighted_cues - .iter() - .any(|(weighted, weight)| weighted == "gather" && *weight < 2.4)); -} - -#[test] -fn weighted_facet_query_reranks_structured_evidence() { - let engine = CueMapEngine::new(); - engine.add_memory( - "Camera maintenance notes and lens cleaning checklist.".to_string(), - vec!["camera".to_string()], - None, - MainStats::default(), - false, - ); - engine.add_memory( - "I prefer Fuji cameras for street photography.".to_string(), - vec!["camera".to_string()], - None, - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - vec![ - ("camera".to_string(), 1.0), - ("type:preference".to_string(), 3.0), - ], - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!( - results.first().map(|r| r.content.as_str()), - Some("I prefer Fuji cameras for street photography.") - ); -} - -#[test] -fn update_facet_reranks_current_state_over_older_topic_match() { - let engine = CueMapEngine::new(); - let older = "I'm planning a birthday trip to Hawaii and I was wondering if you could recommend some good hiking trails on Kauai?"; - let updated = - "I'm actually planning to stay on Oahu, so Hanauma Bay and Shark's Cove sound perfect."; - let generic_stay = "I'm planning a trip to Seoul and looking for the best areas to stay."; - - for content in [older, updated, generic_stay] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "Where am I planning to stay for my birthday trip to Hawaii?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(updated)); -} - -#[test] -fn update_facet_reranks_switched_current_model_over_previous_project() { - let engine = CueMapEngine::new(); - let noise = "Thanks for the SSD advice. I recently upgraded my PC and will check out those drive models."; - let previous = - "I'm looking for tips on weathering effects for my current project, a Ford Mustang Shelby GT350R model."; - let updated = "I have just wrapped up a model and switched to a Ford F-150 pickup truck."; - - for content in [noise, previous, updated] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "What type of vehicle model am I currently working on?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(updated)); -} - -#[test] -fn person_role_facets_help_count_query_rank_structured_role_evidence() { - let engine = CueMapEngine::new(); - let travel = "And any scenic drives or lookouts that I should visit?"; - let generic_doctor = "What questions should I ask my doctor before the colonoscopy?"; - let dr_patel = "I had an appointment with Dr. Patel, the ENT specialist, who diagnosed chronic sinusitis and prescribed nasal spray."; - let dr_smith = "My primary care physician Dr. Smith prescribed antibiotics for a UTI."; - let dr_lee = "Dermatologist Dr. Lee handled the biopsy and said it was benign."; - - for content in [travel, generic_doctor, dr_patel, dr_smith, dr_lee] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let query = "How many different doctors did I visit?"; - let mut weighted_cues: Vec<(String, f64)> = tokenize_to_cues(query) - .into_iter() - .map(|cue| (cue, 1.0)) - .collect(); - let total_memories = engine.total_memories().max(1); - let intent = compile_query_intent(query, |cue| { - let df = engine.get_cue_frequency(cue); - df > 0 && (df <= 16 || df * 5 <= total_memories) - }); - - for (cue, multiplier) in &intent.cue_weight_adjustments { - if let Some((_, weight)) = weighted_cues - .iter_mut() - .find(|(existing, _)| existing == cue) - { - *weight *= *multiplier; - } - } - for (cue, weight) in &intent.weighted_cues { - if let Some((_, existing_weight)) = weighted_cues - .iter_mut() - .find(|(existing, _)| existing == cue) - { - if *existing_weight < *weight { - *existing_weight = *weight; - } - } else { - weighted_cues.push((cue.clone(), *weight)); - } - } - let results = engine.recall_weighted( - weighted_cues, - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top = results.first().map(|result| result.content.as_str()); - assert!( - matches!(top, Some(content) if content == dr_patel || content == dr_smith || content == dr_lee), - "unexpected top result: {top:?}" - ); - assert!(results - .iter() - .take(3) - .all(|result| result.content != travel && result.content != generic_doctor)); -} - -#[test] -fn event_count_query_ranks_counted_object_over_generic_recent_activity() { - let engine = CueMapEngine::new(); - let workshop_today = "User: I'm attending another theater workshop today, focusing on improvisation techniques."; - let lecture_recent = "User: I attended a lecture series at the National Gallery recently, which was enlightening."; - let expected_barn = "User: I'm planning my own wedding and I just got back from a friend's wedding last weekend at a rustic barn."; - let expected_vineyard = "User: I'm getting married soon and I've been to a few weddings recently, including my cousin's wedding at a vineyard."; - - let dated = |date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_date".to_string(), json!(date)); - Some(metadata) - }; - - for (content, date) in [ - (workshop_today, "2023/10/15 (Sun) 07:23"), - (lecture_recent, "2023/10/15 (Sun) 14:36"), - (expected_barn, "2023/10/15 (Sun) 19:23"), - (expected_vineyard, "2023/10/15 (Sun) 05:48"), - ] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - dated(date), - MainStats::default(), - false, - ); - } - - let query = "How many weddings have I attended in this year?"; - let weighted_cues = compile_weighted_query_at(&engine, query, Some("2023/10/15 (Sun) 23:47")); - let results = engine.recall_weighted( - weighted_cues, - 4, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_three = results - .iter() - .take(3) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_three.contains(&expected_barn), "top results: {top_three:?}"); - assert!(top_three.contains(&expected_vineyard), "top results: {top_three:?}"); - assert!(!top_three.contains(&workshop_today), "top results: {top_three:?}"); -} - -#[test] -fn explicit_month_query_weights_content_month_facet() { - let intent = compile_query_intent( - "How many different museums or galleries did I visit in the month of February?", - |cue| matches!(cue, "content_month:02" | "museum" | "gallery" | "has:date"), - ); - - assert!(intent.labels.contains(&"temporal_window".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "content_month:02" && *weight >= 3.0)); -} - -#[test] -fn explicit_month_visit_count_ranks_numeric_month_visits() { - let engine = CueMapEngine::new(); - let january = - "User: I attended a guided workshop at the Modern Art Museum in January."; - let expected_museum = - "User: I took my niece to the Natural History Museum on 2/8 and she loved it."; - let expected_gallery = - "User: I recently saw work when I visited The Art Cube on 2/15."; - - for content in [january, expected_museum, expected_gallery] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "How many different museums or galleries did I visit in the month of February?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_two = results - .iter() - .take(2) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_two.contains(&expected_museum), "top results: {top_two:?}"); - assert!(top_two.contains(&expected_gallery), "top results: {top_two:?}"); -} - -#[test] -fn count_query_boosts_structural_scope_terms() { - let intent = compile_query_intent( - "How many different types of citrus fruits have I used in my cocktail recipes?", - |cue| { - matches!( - cue, - "citrus" - | "fruit" - | "cocktail" - | "recipe" - | "quantity_object:citrus" - | "quantity_object:fruit" - | "type:recipe" - | "type:ingredient" - | "source_role:user" - ) - }, - ); - - assert!(intent.labels.contains(&"count".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "cocktail" && *weight >= 3.0)); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "recipe" && *weight >= 3.0)); -} - -#[test] -fn scoped_count_query_ranks_scope_matches_over_object_only_matches() { - let engine = CueMapEngine::new(); - let citrus_syrup = "User: I made a citrus honey syrup with orange and lemon for iced tea."; - let sangria = "User: I served sangria with slices of citrus fruit at the gathering."; - let expected = - "User: I used orange bitters in my cocktail recipe, and lime in another cocktail recipe."; - - for content in [citrus_syrup, sangria, expected] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let weighted_cues = compile_weighted_query( - &engine, - "How many different types of citrus fruits have I used in my cocktail recipes?", - ); - let results = engine.recall_weighted( - weighted_cues, - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top = results.first().map(|result| result.content.as_str()); - assert_eq!(top, Some(expected)); -} - -#[test] -fn duration_count_query_boosts_activity_scope_terms() { - let intent = compile_query_intent( - "How many days did I spend on camping trips in the United States this year?", - |cue| { - matches!( - cue, - "camp" - | "trip" - | "camp_trip" - | "has:duration" - | "source_year:2023" - | "entity:united_states" - ) - }, - ); - - assert!(intent.labels.contains(&"duration".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "camp_trip" && *weight >= 3.0)); -} - -#[test] -fn duration_count_query_ranks_scoped_duration_memories_over_entity_only_matches() { - let engine = CueMapEngine::new(); - let entity_noise = "User: How does the United States Senate hold a filibuster?"; - let duration_noise = - "User: I've been taking my antibiotics for 10 days now and feel better."; - let target = - "User: I just got back from a 5-day camping trip to Yellowstone National Park."; - - for content in [entity_noise, duration_noise, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "How many days did I spend on camping trips in the United States this year?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn numeric_object_facets_help_count_inventory_query_rank_owned_objects() { - let engine = CueMapEngine::new(); - let current_noise = "I'm currently on Season 4 Episode 10 and loving this show."; - let kid_noise = "What are some popular kid-friendly flooring options for heavy traffic?"; - let friend_noise = "I met a friend at a board game cafe and bought the starter set."; - let one_gallon = - "I've also been taking care of a small 1-gallon tank that I set up for a friend's kid."; - let five_gallon = "I have a 5-gallon tank with a solitary betta fish named Finley."; - let twenty_gallon = - "I've finally set up my 20-gallon freshwater community tank named Amazonia."; - let plant_tank = "I've got an anacharis and a java moss in my community tank."; - - for content in [ - current_noise, - kid_noise, - friend_noise, - one_gallon, - five_gallon, - twenty_gallon, - plant_tank, - ] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let query = - "How many tanks do I currently have, including the one I set up for my friend's kid?"; - let mut weighted_cues: Vec<(String, f64)> = tokenize_to_cues(query) - .into_iter() - .map(|cue| (cue, 1.0)) - .collect(); - let total_memories = engine.total_memories().max(1); - let intent = compile_query_intent(query, |cue| { - let df = engine.get_cue_frequency(cue); - df > 0 && (df <= 16 || df * 5 <= total_memories) - }); - - for (cue, multiplier) in &intent.cue_weight_adjustments { - if let Some((_, weight)) = weighted_cues - .iter_mut() - .find(|(existing, _)| existing == cue) - { - *weight *= *multiplier; - } - } - for (cue, weight) in &intent.weighted_cues { - if let Some((_, existing_weight)) = weighted_cues - .iter_mut() - .find(|(existing, _)| existing == cue) - { - if *existing_weight < *weight { - *existing_weight = *weight; - } - } else { - weighted_cues.push((cue.clone(), *weight)); - } - } - - let results = engine.recall_weighted( - weighted_cues, - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_contents = results - .iter() - .take(3) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_contents.contains(&one_gallon)); - assert!(top_contents.contains(&five_gallon)); - assert!(top_contents.contains(&twenty_gallon)); - assert!(!top_contents.contains(¤t_noise)); - assert!(!top_contents.contains(&kid_noise)); - assert!(!top_contents.contains(&friend_noise)); -} - -#[test] -fn first_person_inventory_queries_penalize_non_user_source_matches() { - let engine = CueMapEngine::new(); - - let user_owned = "User: I'm looking to find a piano technician to service my Korg B1, which I've had for about 3 years."; - let assistant_advice = "Assistant: The Korg B1 is a digital piano, not an acoustic piano, so you will want a technician for musical instruments."; - - for content in [user_owned, assistant_advice] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let weighted_cues = - compile_weighted_query(&engine, "How many musical instruments do I currently own?"); - assert!(weighted_cues - .iter() - .any(|(cue, weight)| cue == "source_role:user" && *weight >= 3.0)); - - let results = engine.recall_weighted( - weighted_cues, - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(user_owned)); -} - -#[test] -fn navigation_facets_help_getting_around_query_rank_transit_resources() { - let engine = CueMapEngine::new(); - let restaurant = - "I'm heading to Tokyo soon and was wondering if you could recommend restaurants near my hotel."; - let shopping = - "I'm planning to do some shopping while I'm in Tokyo. Can you recommend popular malls?"; - let tofu = "Do you have any tips for making sure tofu gets crispy in the stir-fry?"; - let suica = - "I'm visiting Tsukiji. What's the best way to get there from Shinjuku Station using my Suica card?"; - let trip_app = "I'm taking a guided tour tomorrow. How can I get to the meeting point using my transit app and rail pass?"; - - for content in [restaurant, shopping, tofu, suica, trip_app] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "I'm anxious about getting around Tokyo. Do you have any helpful tips?", - ), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_two = results - .iter() - .take(2) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_two.contains(&suica)); - assert!(top_two.contains(&trip_app)); - assert!(!top_two.contains(&restaurant)); - assert!(!top_two.contains(&shopping)); - assert!(!top_two.contains(&tofu)); -} - -#[test] -fn source_session_answer_projection_cues_rank_inline_list_answer_over_user_prompt() { - let engine = CueMapEngine::new(); - let prompt = "User: Brainstorm ideas for work from home jobs for seniors"; - let answer = "Assistant: 1. Virtual customer service representative 2. Telehealth professional 3. Remote bookkeeper 4. Virtual tutor or teacher 5. Freelance writer or editor 6. Online survey taker 7. Transcriptionist 8. Social media manager"; - let distractor = "Assistant: Giving a presentation to students who want to work in logistics can be valuable. Here are a few tips: 1. Define logistics 2. Explain supply chains 3. Share examples"; - - let mut prompt_metadata = HashMap::new(); - prompt_metadata.insert("source_role".to_string(), json!("user")); - prompt_metadata.insert("source_session_id".to_string(), json!("answer_sharegpt_hA7AkP3_0")); - engine.add_memory( - prompt.to_string(), - tokenize_to_cues(prompt), - Some(prompt_metadata), - MainStats::default(), - false, - ); - - let mut answer_metadata = HashMap::new(); - answer_metadata.insert("source_role".to_string(), json!("assistant")); - answer_metadata.insert("source_session_id".to_string(), json!("answer_sharegpt_hA7AkP3_0")); - engine.add_memory( - answer.to_string(), - tokenize_to_cues(answer), - Some(answer_metadata), - MainStats::default(), - false, - ); - - let mut distractor_metadata = HashMap::new(); - distractor_metadata.insert("source_role".to_string(), json!("assistant")); - distractor_metadata.insert("source_session_id".to_string(), json!("answer_sharegpt_other_0")); - engine.add_memory( - distractor.to_string(), - tokenize_to_cues(distractor), - Some(distractor_metadata), - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - vec![ - ("source_session:answer_sharegpt_ha7akp3_0".to_string(), 4.0), - ("source_role:assistant".to_string(), 3.0), - ("has:list".to_string(), 2.8), - ("has:number".to_string(), 1.0), - ], - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(answer)); -} - -#[test] -fn age_difference_query_ranks_current_and_event_age_evidence() { - let engine = CueMapEngine::new(); - let old_table = "User: I'm restoring an old oak coffee table and need antique furniture care tips."; - let workout_duration = "User: I completed a 5K run two Sundays ago in 27 minutes and 12 seconds."; - let mba_answer = "Assistant: An MBA can be useful for long-term career goals and leadership roles."; - let current_age = "User: I'm considering pursuing the CDMP certification. As a 32-year-old Digital Marketing Specialist at TechSavvy Inc., I believe it will prepare me for an MBA."; - let graduation_age = "User: I have a Bachelor's degree in Business Administration from the University of California, Berkeley, which I completed at the age of 25."; - - for content in [old_table, workout_duration, mba_answer, current_age, graduation_age] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "How many years older am I than when I graduated from college?", - ), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_two = results - .iter() - .take(2) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_two.contains(¤t_age)); - assert!(top_two.contains(&graduation_age)); - assert!(!top_two.contains(&old_table)); - assert!(!top_two.contains(&workout_duration)); -} - -#[test] -fn sibling_count_query_ranks_brother_and_sister_facts() { - let engine = CueMapEngine::new(); - let board_game = "User: Actually, I have been playing a lot of board games recently."; - let sweet_tooth = "User: I have a bit of a sweet tooth and want dessert spots nearby."; - let sister_gift = "User: Can you remind me about the necklace I got for my sister's birthday?"; - let twin_movie = "Assistant: The film follows twin siblings who uncover a family secret."; - let sisters = "User: I come from a family with 3 sisters, so I've always had a strong female presence in my life."; - let brother = "User: I should mention that I have a brother, which might be influencing my social circle dynamics."; - - for content in [board_game, sweet_tooth, sister_gift, twin_movie, sisters, brother] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "What is the total number of siblings I have?"), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_two = results - .iter() - .take(2) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_two.contains(&sisters)); - assert!(top_two.contains(&brother)); - assert!(!top_two.contains(&board_game)); - assert!(!top_two.contains(&sweet_tooth)); - assert!(!top_two.contains(&sister_gift)); - assert!(!top_two.contains(&twin_movie)); -} - -#[test] -fn relative_temporal_query_ranks_source_dated_memory_without_exact_domain_word() { - let engine = CueMapEngine::new(); - let museum = "User: I attended an art museum exhibition three weeks ago and enjoyed the guided tour."; - let garden_app = "User: I've been using a gardening app to track weather and soil moisture levels."; - let fertilizer = "User: I attended a gardening workshop recently and learned about companion planting."; - let expected = "User: I'm looking for advice on keeping my tomato plants healthy and pest-free. I just planted 12 new tomato saplings today."; - - let dated = |date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_date".to_string(), json!(date)); - Some(metadata) - }; - - engine.add_memory( - museum.to_string(), - tokenize_to_cues(museum), - dated("2023/04/14 (Fri) 12:00"), - MainStats::default(), - false, - ); - engine.add_memory( - garden_app.to_string(), - tokenize_to_cues(garden_app), - dated("2023/05/01 (Mon) 12:00"), - MainStats::default(), - false, - ); - engine.add_memory( - fertilizer.to_string(), - tokenize_to_cues(fertilizer), - dated("2023/04/28 (Fri) 12:00"), - MainStats::default(), - false, - ); - engine.add_memory( - "User: hello".to_string(), - tokenize_to_cues("User: hello"), - dated("2023/04/21 (Fri) 00:29"), - MainStats::default(), - false, - ); - engine.add_memory( - "User: I will provide context data in the next several queries.".to_string(), - tokenize_to_cues("User: I will provide context data in the next several queries."), - dated("2023/04/21 (Fri) 00:29"), - MainStats::default(), - false, - ); - engine.add_memory( - expected.to_string(), - tokenize_to_cues(expected), - dated("2023/04/21 (Fri) 00:30"), - MainStats::default(), - false, - ); - engine.add_memory( - "User: I recently attended a documentary filmmaking panel and completed a workshop." - .to_string(), - tokenize_to_cues( - "User: I recently attended a documentary filmmaking panel and completed a workshop.", - ), - None, - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "What gardening-related activity did I do two weeks ago?", - Some("2023/05/05 (Fri) 16:42"), - ), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(expected)); -} - -#[test] -fn religious_activity_query_ranks_religious_service_over_generic_attendance() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let museum = "User: I attended a ceramics workshop at the Museum of Craft and Design."; - let volunteer = "User: I helped out at an Easter Egg Hunt event last week."; - let expected = - "User: I attended the Maundy Thursday service at the Episcopal Church last week."; - - add(museum, "2023/03/26 (Sun) 21:45"); - add(volunteer, "2023/04/06 (Thu) 12:00"); - add(expected, "2023/04/06 (Thu) 05:36"); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "Where did I attend the religious activity last week?", - Some("2023/04/10 (Mon) 12:00"), - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(expected)); -} - -#[test] -fn streaming_service_query_ranks_media_usage_over_recent_noise() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let recent_noise = - "User: I recently attended a writing workshop and want more prompts for memoir writing."; - let netflix = - "User: I've been using Netflix, Hulu, and Amazon Prime for the past 6 months while looking for new shows to watch."; - let disney = "User: I saw a documentary on Disney+ during my free trial last month."; - let apple = - "User: I've also been using Apple TV+ for a few months now and finished watching For All Mankind."; - - add(recent_noise, "2023/05/26 (Fri) 23:59"); - add(netflix, "2023/05/26 (Fri) 08:25"); - add(disney, "2023/05/26 (Fri) 01:08"); - add(apple, "2023/05/26 (Fri) 23:40"); - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "Which streaming service did I start using most recently?", - ), - 4, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_three = results - .iter() - .take(3) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_three.contains(&netflix)); - assert!(top_three.contains(&disney)); - assert!(top_three.contains(&apple)); - assert!(!top_three.contains(&recent_noise)); -} - -#[test] -fn music_streaming_service_query_prefers_music_usage_over_video_streaming() { - let engine = CueMapEngine::new(); - - let netflix = - "User: I've been using Netflix for a while now and watching more original content lately."; - let documentaries = "User: I've been meaning to try out some documentaries on Netflix."; - let target = - "User: I've been listening to their songs a lot on Spotify lately."; - - for content in [netflix, documentaries, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "What is the name of the music streaming service have I been using lately?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn counted_item_facets_rank_inventory_counts_over_generic_collection_mentions() { - let engine = CueMapEngine::new(); - let generic = "I'm thinking of adding decorations to the 20-gallon tank to create more hiding places for the fish."; - let counted = "My new 20-gallon tank currently has 10 neon tetras, 5 golden honey gouramis, and a small pleco catfish."; - - for content in [generic, counted] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let mut weighted_cues: Vec<(String, f64)> = - tokenize_to_cues("How many fish are in my tank?").into_iter().map(|cue| (cue, 1.0)).collect(); - let intent = compile_query_intent("How many fish are in my tank?", |cue| { - engine.get_cue_frequency(cue) > 0 - }); - for (cue, weight) in intent.weighted_cues { - if let Some((_, existing)) = weighted_cues.iter_mut().find(|(existing, _)| existing == &cue) - { - if *existing < weight { - *existing = weight; - } - } else { - weighted_cues.push((cue, weight)); - } - } - - let results = engine.recall_weighted( - weighted_cues, - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(counted)); -} - -#[test] -fn current_book_query_prefers_current_reading_over_finished_book_mentions() { - let engine = CueMapEngine::new(); - let finished = "User: We're going to discuss \"The Last House Guest\" by Megan Miranda, which I've already read and really enjoyed."; - let reading_habit = "User: I love making reading a habit loop and read before bed every night."; - let target = "User: I'm currently devouring \"The Seven Husbands of Evelyn Hugo\" and it's hard to put down."; - - for content in [finished, reading_habit, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query(&engine, "What book am I currently reading?"), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn temporal_order_query_ranks_visited_events_over_latest_update_noise() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let latest_noise = "User: I'm planning to attend another art-related event soon and need the latest trends and exhibition updates."; - let science = "User: I visited the Science Museum's Space Exploration exhibition today."; - let history = "User: I participated in a behind-the-scenes tour of the Museum of History's conservation lab today."; - - add(latest_noise, "2023/03/10 (Fri) 10:00"); - add(science, "2023/01/15 (Sun) 16:31"); - add(history, "2023/02/15 (Wed) 12:20"); - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "What is the order of the six museums I visited from earliest to latest?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert!(results[0].content.contains("Science Museum")); - assert!(results - .iter() - .take(2) - .any(|result| result.content.contains("Museum of History"))); - assert!(!results[0].content.contains("latest trends")); -} - -#[test] -fn sports_event_order_query_ranks_competitions_in_past_month() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let old_progress = "User: I've been tracking body fat percentage and made progress over the past month."; - let old_workout = "User: I've been thinking about trying a quick 20-minute workout before work."; - let soccer = "User: I participate in the company's annual charity soccer tournament today."; - let triathlon = "User: I just completed the Spring Sprint Triathlon today, which included a 20K bike ride."; - let run = "User: I just finished a 5K run with a personal best time at the Midsummer 5K Run."; - - add(old_progress, "2023/05/12 (Fri) 15:34"); - add(old_workout, "2023/05/21 (Sun) 12:34"); - add(triathlon, "2023/06/02 (Fri) 15:29"); - add(run, "2023/06/10 (Sat) 15:00"); - add(soccer, "2023/06/17 (Sat) 11:09"); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "What is the order of the three sports events I participated in during the past month, from earliest to latest?", - Some("2023/07/01 (Sat) 20:43"), - ), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_three = results - .iter() - .take(3) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_three.contains(&triathlon)); - assert!(top_three.contains(&run)); - assert!(top_three.contains(&soccer)); - assert!(!top_three.contains(&old_progress)); - assert!(!top_three.contains(&old_workout)); -} - -#[test] -fn relative_charity_event_query_prefers_matching_date_and_domain() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - Some(metadata), - MainStats::default(), - false, - ); - }; - - let wrong_date = "User: I just got back from the \"24-Hour Bike Ride\" charity event, where I cycled for 4 hours."; - let same_date_noise = "User: I've played Ticket to Ride and Settlers of Catan before, and they're great games."; - let target = "User: I just did the \"Walk for Hunger\" charity event today with my colleagues from work."; - - add(wrong_date, "2023/02/14 (Tue) 06:22"); - add(same_date_noise, "2023/03/19 (Sun) 04:24"); - add(target, "2023/03/19 (Sun) 15:44"); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "What charity event did I participate in a month ago?", - Some("2023/04/18 (Tue) 18:34"), - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn extracts_companion_facets_from_first_person_event_language() { - let facets = extract_memory_facets( - "User: I just saw Queen live with Adam Lambert at the Prudential Center with my parents.", - None, - &[], - ); - - assert!(facets.contains(&"type:activity".to_string())); - assert!(facets.contains(&"companion:with".to_string())); -} - -#[test] -fn companion_query_weights_with_companion_evidence() { - let available = |cue: &str| matches!(cue, "companion:with" | "source_role:user" | "type:activity" | "type:event"); - let intent = compile_query_intent( - "Who did I go with to the music event last Saturday?", - available, - ); - - assert!(intent.labels.contains(&"companion".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "companion:with" && *weight >= 5.0)); -} - -#[test] -fn companion_query_prefers_events_with_with_companion_evidence() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let generic_music = - "User: I've been listening to a lot of music and planning to check out jazz clubs."; - let target = "User: I just saw Queen live with Adam Lambert at the Prudential Center with my parents, and I want more classic rock playlists."; - let friend_festival = - "User: I went to a music festival in Brooklyn with a group of friends recently."; - - add(generic_music, "2023/05/20 (Sat) 12:00"); - add(friend_festival, "2023/05/22 (Mon) 12:00"); - add(target, "2023/05/27 (Sat) 21:00"); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "Who did I go with to the music event last Saturday?", - Some("2023/06/03 (Sat) 12:00"), - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert!(results.iter().take(2).any(|result| result.content == target)); - assert!(!results[0].content.contains("jazz clubs")); -} - -#[test] -fn extracts_completed_clean_facets_from_first_person_action_language() { - let facets = extract_memory_facets( - "User: I'm glad I finally got around to cleaning my white Adidas sneakers last month.", - None, - &[], - ); - - assert!(facets.contains(&"type:activity".to_string())); - assert!(facets.contains(&"completed_action:clean".to_string())); -} - -#[test] -fn completed_clean_query_weights_completed_action_evidence() { - let available = |cue: &str| { - matches!( - cue, - "completed_action:clean" | "source_role:user" | "type:activity" | "type:event" - ) - }; - let intent = compile_query_intent("Which pair of shoes did I clean last month?", available); - - assert!(intent.labels.contains(&"completed_action".to_string())); - assert!(intent - .weighted_cues - .iter() - .any(|(cue, weight)| cue == "completed_action:clean" && *weight >= 5.0)); -} - -#[test] -fn completed_clean_query_prefers_completed_cleaning_over_cleaning_advice() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let advice = "User: What's the best way to clean and maintain my new hiking boots?"; - let lent = "User: I lent my spare pair of running shoes to my sister a few weeks ago."; - let target = "User: I'm glad I finally got around to cleaning my white Adidas sneakers last month."; - - add(lent, "2023/04/11 (Tue) 12:00"); - add(advice, "2023/05/05 (Fri) 12:00"); - add(target, "2023/05/21 (Sun) 12:00"); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "Which pair of shoes did I clean last month?", - Some("2023/06/15 (Thu) 12:00"), - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|result| result.content.as_str()), Some(target)); -} - -#[test] -fn transport_mode_comparison_ranks_ride_events_over_habitual_transport_mentions() { - let engine = CueMapEngine::new(); - - let add = |content: &str, date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_role".to_string(), json!("user")); - metadata.insert("source_date".to_string(), json!(date)); - let cues = [ - tokenize_to_cues(content), - extract_memory_facets(content, Some(&metadata), &[]), - ] - .concat(); - engine.add_memory( - content.to_string(), - cues, - Some(metadata), - MainStats::default(), - false, - ); - }; - - let generic = "User: I have been tracking my modes of transport and have been taking more trains and buses instead of driving."; - let bus = "User: I just got back from a bus ride to attend a friend's wedding today."; - let train = "User: I took a train ride to visit my family today, and it was a nice 2-hour journey."; - - add(generic, "2023/03/03 (Fri) 19:17"); - add(bus, "2023/02/27 (Mon) 06:17"); - add(train, "2023/03/03 (Fri) 19:17"); - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "Which mode of transport did I use most recently, a bus or a train?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - let top_two = results - .iter() - .take(2) - .map(|result| result.content.as_str()) - .collect::<Vec<_>>(); - assert!(top_two.contains(&bus)); - assert!(top_two.contains(&train)); - assert!(!top_two.contains(&generic)); -} - -#[test] -fn milestone_temporal_query_ranks_first_client_contract() { - let engine = CueMapEngine::new(); - let instagram = "User: I recently collaborated with an influencer who promoted my product to 10,000 followers."; - let plant = "User: I recently repotted my spider plant 3 weeks ago and it is doing better."; - let expected = "User: I'm looking for advice on creating a solid contract for my freelance clients. I just signed a contract with my first client today."; - - let dated = |date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_date".to_string(), json!(date)); - Some(metadata) - }; - - engine.add_memory( - instagram.to_string(), - tokenize_to_cues(instagram), - dated("2023/02/28 (Tue) 12:00"), - MainStats::default(), - false, - ); - engine.add_memory( - plant.to_string(), - tokenize_to_cues(plant), - dated("2023/03/01 (Wed) 12:00"), - MainStats::default(), - false, - ); - engine.add_memory( - expected.to_string(), - tokenize_to_cues(expected), - dated("2023/03/01 (Wed) 02:43"), - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "What was the significant buisiness milestone I mentioned four weeks ago?", - Some("2023/03/28 (Tue) 20:35"), - ), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(expected)); -} - -#[test] -fn purchase_temporal_query_ranks_acquired_object_memory() { - let engine = CueMapEngine::new(); - let wood = "User: I'm thinking of trying out a mix of hickory and apple wood today."; - let jacket = "User: I also have a blue denim jacket from Zara that I've been loving lately."; - let expected = "User: I'm looking for BBQ sauce recipes. By the way, I just got a smoker today and I'm excited to experiment with different woods."; - - let dated = |date: &str| { - let mut metadata = HashMap::new(); - metadata.insert("source_date".to_string(), json!(date)); - Some(metadata) - }; - - engine.add_memory( - wood.to_string(), - tokenize_to_cues(wood), - dated("2023/03/15 (Wed) 09:00"), - MainStats::default(), - false, - ); - engine.add_memory( - jacket.to_string(), - tokenize_to_cues(jacket), - dated("2023/03/14 (Tue) 09:00"), - MainStats::default(), - false, - ); - engine.add_memory( - expected.to_string(), - tokenize_to_cues(expected), - dated("2023/03/15 (Wed) 10:00"), - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - compile_weighted_query_at( - &engine, - "What kitchen appliance did I buy 10 days ago?", - Some("2023/03/25 (Sat) 20:00"), - ), - 5, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(expected)); -} - -#[test] -fn homegrown_ingredient_query_ranks_harvested_garden_memory() { - let engine = CueMapEngine::new(); - let cocktail = - "User: I'm looking for inspiration for new cocktail recipes and ingredients this weekend."; - let family_dinner = - "User: I'm trying to plan a family dinner and need healthy meal ideas."; - let expected = "User: I've been using basil and mint in my cooking lately. I've even harvested some cherry tomatoes from my garden. Do you have suggestions for companion plants?"; - let garden_noise = - "User: I've been thinking about introducing beneficial insects to my garden."; - - for content in [cocktail, family_dinner, expected, garden_noise] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "What should I serve for dinner this weekend with my homegrown ingredients?", - ), - 4, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(expected)); - assert_ne!(results.first().map(|r| r.content.as_str()), Some(cocktail)); -} - -#[test] -fn generated_facets_do_not_create_bare_value_aliases() { - let engine = CueMapEngine::new(); - let mut metadata = HashMap::new(); - metadata.insert("source_date".to_string(), json!("2023/04/21 (Fri) 00:30")); - engine.add_memory( - "Doctor: As a 32-year-old commuter, I currently practice 20 minutes daily, paid $15 yesterday, prefer Fuji X100V, use a transit app, and I have a brother." - .to_string(), - vec![], - Some(metadata), - MainStats::default(), - false, - ); - - assert_eq!(engine.get_cue_frequency("source_role:doctor"), 1); - assert_eq!(engine.get_cue_frequency("source_date:2023_04_21"), 1); - assert_eq!(engine.get_cue_frequency("source_week:2023_w16"), 1); - assert_eq!(engine.get_cue_frequency("has:number"), 1); - assert_eq!(engine.get_cue_frequency("has:money"), 1); - assert_eq!(engine.get_cue_frequency("has:duration"), 1); - assert_eq!(engine.get_cue_frequency("has:age"), 1); - assert_eq!(engine.get_cue_frequency("age:current"), 1); - assert_eq!(engine.get_cue_frequency("type:preference"), 1); - assert_eq!(engine.get_cue_frequency("type:navigation"), 1); - assert_eq!(engine.get_cue_frequency("travel:app"), 1); - assert_eq!(engine.get_cue_frequency("family_relation:sibling"), 1); - assert_eq!(engine.get_cue_frequency("family_count:sibling"), 1); - assert_eq!(engine.get_cue_frequency("sibling_kind:brother"), 1); - assert_eq!(engine.get_cue_frequency("entity:fuji_x100v"), 1); - - for leaked_alias in [ - "doctor", - "2023_04_21", - "2023_w16", - "number", - "money", - "duration", - "age", - "current", - "preference", - "navigation", - "app", - "sibling", - "brother", - "fuji_x100v", +fn query_plan_emits_bounded_answer_shape_labels() { + for (query, expected) in [ + ("Who did I meet?", "answer_shape_person"), + ("Where did I go?", "answer_shape_location"), + ("When did it happen?", "answer_shape_time"), + ("How many memories mention Flask?", "answer_shape_count"), + ("How much did it cost?", "answer_shape_amount"), + ("Why did I choose it?", "answer_shape_reason"), + ("How long did it take?", "answer_shape_duration"), + ("Which option did I select?", "answer_shape_selection"), + ("What kind of file is this?", "answer_shape_category"), + ("Did I deploy it?", "answer_shape_boolean"), ] { - assert_eq!( - engine.get_cue_frequency(leaked_alias), - 0, - "generated facet leaked bare alias: {leaked_alias}" - ); - } -} - -#[test] -fn structured_facet_only_match_does_not_dominate_lexical_match() { - let engine = CueMapEngine::new(); - let relevant = "User: I practice guitar daily."; - let distractor = "User: I paid $15 at the market yesterday."; - - engine.add_memory( - distractor.to_string(), - tokenize_to_cues(distractor), - None, - MainStats::default(), - false, - ); - engine.add_memory( - relevant.to_string(), - tokenize_to_cues(relevant), - None, - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - vec![("guitar".to_string(), 1.0), ("has:money".to_string(), 3.5)], - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(relevant)); -} - -#[test] -fn weak_recommendation_scaffold_does_not_outrank_topic_match() { - let engine = CueMapEngine::new(); - let relevant = "User: Besides great views, I also like hotels with unique features, such as a rooftop pool or hot tub."; - let distractor = - "User: Can you suggest how to design a gamified conference challenge with hints?"; - - engine.add_memory( - distractor.to_string(), - tokenize_to_cues(distractor), - None, - MainStats::default(), - false, - ); - engine.add_memory( - relevant.to_string(), - tokenize_to_cues(relevant), - None, - MainStats::default(), - false, - ); - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "Can you suggest a hotel for my upcoming trip to Miami?", - ), - 2, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|result| result.content.as_str()), Some(relevant)); -} - -#[test] -fn generic_question_distractors_do_not_beat_specific_lexical_match() { - let engine = CueMapEngine::new(); - let relevant = "User: I graduated with a degree in Business Administration."; - let distractor = "User: What math problem do they solve?"; - - engine.add_memory( - distractor.to_string(), - tokenize_to_cues(distractor), - None, - MainStats::default(), - false, - ); - engine.add_memory( - relevant.to_string(), - tokenize_to_cues(relevant), - None, - MainStats::default(), - false, - ); - - let results = engine.recall( - tokenize_to_cues("What degree did I graduate with?"), - 2, - false, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(relevant)); -} - -#[test] -fn undergraduate_degree_query_prefers_matching_level_and_field_initialism() { - let engine = CueMapEngine::new(); - let target = - "User: I completed my undergrad in CS from UCLA before starting my first developer job."; - let masters_distractor = - "User: I am pursuing a Master's degree in Data Science while taking evening classes."; - let computer_science_distractor = - "Assistant: Computer Science programs often cover algorithms, systems, and theory."; - - for content in [masters_distractor, computer_science_distractor, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "Where did I complete my Bachelor's degree in Computer Science?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn shopping_advice_query_prefers_first_person_purchase_consideration() { - let engine = CueMapEngine::new(); - let tuning = "User: What are the differences between open D tuning and standard tuning?"; - let genre = "User: What are the most common types of music that people play on a Les Paul?"; - let target = "User: I'm considering upgrading from a Fender Stratocaster to a Gibson Les Paul. Can you tell me the main differences between these two guitars?"; - - for content in [tuning, genre, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); - } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "I'm getting excited about my visit to the music store this weekend. Any tips on what to look for in a new guitar?", - ), - 3, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); -} - -#[test] -fn assistant_created_second_version_query_prefers_iteration_answer() { - let engine = CueMapEngine::new(); - let user_prompt = "User: Create a sad song with notes"; - let first_song = "Assistant: Here's a sad song with notes for you:\n\nVerse 1:\nC D E E E D C C\nThe rain falls down on me\n\nChorus:\nG G G G A G F\nWhy did you leave?"; - let theory = "Assistant: Understanding music theory will help you create chord progressions and songs."; - let target = "Assistant: Sure, here's a more romantic and heart-felt song for you:\n\nVerse 1:\nG A B C D E D C B A G\nWhen I first saw you\n\nChorus:\nC D E F G A B A G F E D C\nYou're the one I want"; - - for content in [user_prompt, first_song, theory, target] { - engine.add_memory( - content.to_string(), - tokenize_to_cues(content), - None, - MainStats::default(), - false, - ); + let plan = compile_query_plan(query, |_| false); + assert!(plan.labels.iter().any(|label| label == expected), "missing {expected} for {query:?}: {plan:?}"); } - - let results = engine.recall_weighted( - compile_weighted_query( - &engine, - "I'm looking back at our previous conversation where you created two sad songs for me. Can you remind me what was the chord progression for the chorus in the second song?", - ), - 4, - false, - None, - 1, - true, - true, - None, - None, - ); - - assert_eq!(results.first().map(|r| r.content.as_str()), Some(target)); } diff --git a/tests/fixture_test.rs b/tests/fixture_test.rs index 202f5e4..1c27e51 100644 --- a/tests/fixture_test.rs +++ b/tests/fixture_test.rs @@ -2,13 +2,12 @@ use cuemap::engine::CueMapEngine; use cuemap::persistence::PersistenceManager; use cuemap::structures::MainStats; use cuemap::config::ServerConfig; -use std::fs; -use std::path::PathBuf; #[test] fn test_fixture_loading_and_recall() { // 1. Create a dummy snapshot file - let fixture_path = PathBuf::from("/private/tmp/cuemap_fixture_test.bin"); + let fixture_dir = tempfile::tempdir().expect("Failed to create fixture directory"); + let fixture_path = fixture_dir.path().join("cuemap_fixture_test.bin"); { // Scope to drop engine @@ -49,7 +48,4 @@ fn test_fixture_loading_and_recall() { let results = loaded_engine.recall(vec!["test_cue".to_string()], 5, false, None); assert!(!results.is_empty()); assert_eq!(results[0].content, "Test Content"); - - // Cleanup - let _ = fs::remove_file(fixture_path); } diff --git a/tests/ingester_filter_test.rs b/tests/ingester_filter_test.rs index d123ccb..5292bd3 100644 --- a/tests/ingester_filter_test.rs +++ b/tests/ingester_filter_test.rs @@ -1,9 +1,12 @@ use cuemap::agent::ingester::Ingester; use cuemap::agent::AgentConfig; +use cuemap::config::TuningConfig; use cuemap::jobs::{JobQueue, ProjectProvider}; +use cuemap::multi_tenant::MultiTenantEngine; use std::fs; use std::sync::Arc; use tempfile::tempdir; +use tokio::time::{sleep, timeout, Duration}; struct MockProvider; impl ProjectProvider for MockProvider { @@ -72,6 +75,7 @@ async fn test_ingester_filters_noise_and_ignore_files() { watch_dir: watch_path.to_string_lossy().to_string(), throttle_ms: 0, state_file: None, + included_paths: Vec::new(), ignored_patterns: vec!["custom_ignored.txt".to_string()], ignored_extensions: vec!["bak".to_string()], }; @@ -159,3 +163,252 @@ async fn test_ingester_filters_noise_and_ignore_files() { "sub/valid_in_sub.txt should be tracked" ); } + +#[tokio::test] +async fn nested_and_context_specific_ignore_files_do_not_hide_the_repository() { + let dir = tempdir().unwrap(); + let watch_path = dir.path().to_path_buf(); + fs::write(watch_path.join("root.rs"), "fn root() {}").unwrap(); + fs::write(watch_path.join(".dockerignore"), "*\n").unwrap(); + + let generated = watch_path.join("generated"); + fs::create_dir(&generated).unwrap(); + fs::write(generated.join(".gitignore"), "*\n").unwrap(); + fs::write(generated.join("ignored.rs"), "fn ignored() {}").unwrap(); + + let source = watch_path.join("src"); + fs::create_dir(&source).unwrap(); + fs::write(source.join("main.rs"), "fn main() {}").unwrap(); + + let job_queue = Arc::new(JobQueue::new(Arc::new(MockProvider), None, true)); + let config = AgentConfig { + project_id: "scoped_ignores".to_string(), + watch_dir: watch_path.to_string_lossy().to_string(), + throttle_ms: 0, + state_file: None, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let mut ingester = Ingester::new(config, job_queue); + + let preview = ingester.preview_scope().unwrap(); + assert_eq!(preview.supported_files, 2); + assert!(preview.entries.iter().any(|entry| entry.path == "root.rs")); + assert!(preview.entries.iter().any(|entry| entry.path == "src")); + assert!(!preview.entries.iter().any(|entry| entry.path == "generated")); + + ingester.scan_all().await.unwrap(); + let tracked = ingester.get_file_hashes(); + let root = fs::canonicalize(watch_path.join("root.rs")) + .unwrap() + .to_string_lossy() + .to_lowercase(); + let main = fs::canonicalize(source.join("main.rs")) + .unwrap() + .to_string_lossy() + .to_lowercase(); + let ignored = fs::canonicalize(generated.join("ignored.rs")) + .unwrap() + .to_string_lossy() + .to_lowercase(); + assert!(tracked.contains_key(&root)); + assert!(tracked.contains_key(&main)); + assert!(!tracked.contains_key(&ignored)); +} + +#[tokio::test] +async fn selected_scope_applies_to_initial_and_new_files() { + let dir = tempdir().unwrap(); + let watch_path = dir.path().to_path_buf(); + fs::create_dir(watch_path.join("src")).unwrap(); + fs::create_dir(watch_path.join("docs")).unwrap(); + fs::write(watch_path.join("src/main.rs"), "fn main() {}").unwrap(); + fs::write(watch_path.join("docs/guide.md"), "# Guide").unwrap(); + + let job_queue = Arc::new(JobQueue::new(Arc::new(MockProvider), None, true)); + let config = AgentConfig { + project_id: "selected_scope".to_string(), + watch_dir: watch_path.to_string_lossy().to_string(), + throttle_ms: 0, + state_file: None, + included_paths: vec!["src".to_string()], + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let mut ingester = Ingester::new(config, job_queue); + + let preview = ingester.preview_scope().unwrap(); + assert_eq!(preview.supported_files, 1); + assert_eq!(preview.entries.len(), 1); + assert_eq!(preview.entries[0].path, "src"); + + ingester.scan_all().await.unwrap(); + let main_path = fs::canonicalize(watch_path.join("src/main.rs")) + .unwrap() + .to_string_lossy() + .to_lowercase(); + let guide_path = fs::canonicalize(watch_path.join("docs/guide.md")) + .unwrap() + .to_string_lossy() + .to_lowercase(); + assert!(ingester.get_file_hashes().contains_key(&main_path)); + assert!(!ingester.get_file_hashes().contains_key(&guide_path)); + + let new_source = watch_path.join("src/new.rs"); + fs::write(&new_source, "pub fn new_file() {}").unwrap(); + ingester.process_file_path(new_source.clone()).await.unwrap(); + let new_source = fs::canonicalize(new_source) + .unwrap() + .to_string_lossy() + .to_lowercase(); + assert!(ingester.get_file_hashes().contains_key(&new_source)); + + let new_doc = watch_path.join("docs/new.md"); + fs::write(&new_doc, "# Not selected").unwrap(); + ingester.process_file_path(new_doc.clone()).await.unwrap(); + let new_doc = fs::canonicalize(new_doc) + .unwrap() + .to_string_lossy() + .to_lowercase(); + assert!(!ingester.get_file_hashes().contains_key(&new_doc)); +} + +#[tokio::test] +async fn changed_cuemapignore_is_reloaded_and_reconciled() { + let dir = tempdir().unwrap(); + let watch_path = dir.path().to_path_buf(); + fs::write(watch_path.join("keep.rs"), "pub fn keep() {}").unwrap(); + fs::write(watch_path.join("remove.rs"), "pub fn remove() {}").unwrap(); + + let job_queue = Arc::new(JobQueue::new(Arc::new(MockProvider), None, true)); + let config = AgentConfig { + project_id: "ignore_reload".to_string(), + watch_dir: watch_path.to_string_lossy().to_string(), + throttle_ms: 0, + state_file: None, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let mut ingester = Ingester::new(config, job_queue); + ingester.scan_all().await.unwrap(); + assert_eq!(ingester.get_file_hashes().len(), 2); + + fs::write(watch_path.join(".cuemapignore"), "remove.rs\n").unwrap(); + ingester.reload_filters_and_rescan().await.unwrap(); + assert_eq!(ingester.get_file_hashes().len(), 1); + + fs::remove_file(watch_path.join(".cuemapignore")).unwrap(); + ingester.reload_filters_and_rescan().await.unwrap(); + assert_eq!(ingester.get_file_hashes().len(), 2); +} + +#[tokio::test] +async fn replacing_saved_scope_prunes_previously_tracked_paths() { + let dir = tempdir().unwrap(); + let watch_path = dir.path().join("repo"); + let state_path = dir.path().join("agent-state.json"); + fs::create_dir(&watch_path).unwrap(); + fs::create_dir(watch_path.join("src")).unwrap(); + fs::create_dir(watch_path.join("docs")).unwrap(); + fs::write(watch_path.join("src/main.rs"), "fn main() {}").unwrap(); + fs::write(watch_path.join("docs/guide.md"), "# Guide").unwrap(); + + let first_queue = Arc::new(JobQueue::new(Arc::new(MockProvider), None, true)); + let first_config = AgentConfig { + project_id: "scope_replacement".to_string(), + watch_dir: watch_path.to_string_lossy().to_string(), + throttle_ms: 0, + state_file: Some(state_path.clone()), + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let mut first_ingester = Ingester::new(first_config, first_queue); + first_ingester.scan_all().await.unwrap(); + first_ingester.save_state(&state_path).unwrap(); + assert_eq!(first_ingester.get_file_hashes().len(), 2); + + let second_queue = Arc::new(JobQueue::new(Arc::new(MockProvider), None, true)); + let second_config = AgentConfig { + project_id: "scope_replacement".to_string(), + watch_dir: watch_path.to_string_lossy().to_string(), + throttle_ms: 0, + state_file: Some(state_path.clone()), + included_paths: vec!["src".to_string()], + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let mut second_ingester = Ingester::new(second_config, second_queue); + second_ingester.load_state(&state_path).unwrap(); + second_ingester.scan_all().await.unwrap(); + + assert_eq!(second_ingester.get_file_hashes().len(), 1); + let tracked_path = second_ingester + .get_file_hashes() + .keys() + .next() + .unwrap(); + assert!(tracked_path.ends_with("/src/main.rs")); +} + +#[tokio::test] +async fn repository_file_ingestion_records_explicit_source_type() { + let dir = tempdir().unwrap(); + let snapshots = dir.path().join("snapshots"); + fs::create_dir_all(&snapshots).unwrap(); + let watch_path = dir.path().join("repo"); + fs::create_dir(&watch_path).unwrap(); + fs::write(watch_path.join("main.rs"), "fn main() {}").unwrap(); + + let engine = Arc::new(MultiTenantEngine::with_snapshots_dir( + snapshots, + TuningConfig::default(), + )); + let project_id = "source_type_test".to_string(); + let context = engine.get_or_create_project(project_id.clone()).unwrap(); + let job_queue = Arc::new(JobQueue::new(engine, None, false)); + let config = AgentConfig { + project_id: project_id.clone(), + watch_dir: watch_path.to_string_lossy().to_string(), + throttle_ms: 0, + state_file: None, + included_paths: Vec::new(), + ignored_patterns: Vec::new(), + ignored_extensions: Vec::new(), + }; + let session = job_queue.session_manager.get_or_create(&project_id); + let mut ingester = Ingester::new(config, job_queue); + ingester.scan_all().await.unwrap(); + + timeout(Duration::from_secs(30), async { + loop { + let progress = session.get_progress(); + if progress.writes_total > 0 && progress.writes_completed >= progress.writes_total { + break; + } + sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("timed out waiting for repository ingestion to complete"); + + let memories = context.main.get_memories(); + let memory_entry = memories + .iter() + .next() + .expect("repository file was not ingested"); + let memory = memory_entry.value(); + assert_eq!( + memory + .metadata + .get("source_type") + .and_then(|value| value.as_str()), + Some("repository_file") + ); + assert!(memory + .cues + .iter() + .any(|cue| cue == "source_type:repository_file")); +} diff --git a/tests/jobs/mod.rs b/tests/jobs/mod.rs index d5b6fa6..7bd1b11 100644 --- a/tests/jobs/mod.rs +++ b/tests/jobs/mod.rs @@ -34,6 +34,36 @@ fn test_recall_requests_keep_extra_passes_off_by_default() { assert!(grounded.auto_reinforce); } +#[test] +fn progress_phase_waits_for_intent_jobs_after_writes() { + let session = IngestionSession::new("phase_test".to_string()); + + assert_eq!(session.get_progress().phase, "idle"); + + session.expect_write(); + assert_eq!(session.get_progress().phase, "writing"); + + session.write_complete(); + session.expect_intent(); + assert_eq!(session.get_progress().phase, "processing"); + + session.intent_complete(); + assert_eq!(session.get_progress().phase, "done"); +} + +#[test] +fn failed_intent_jobs_reach_a_terminal_phase() { + let session = IngestionSession::new("intent-failure".to_string()); + session.expect_intent(); + session.intent_failed(); + + let progress = session.get_progress(); + assert_eq!(progress.phase, "done"); + assert_eq!(progress.intent_completed, 0); + assert_eq!(progress.intent_failed, 1); + assert_eq!(progress.intent_total, 1); +} + #[tokio::test] async fn extract_and_ingest_preserves_metadata_for_ordered_recall() { let dir = tempdir().unwrap(); @@ -62,6 +92,7 @@ async fn extract_and_ingest_preserves_metadata_for_ordered_recall() { file_path: "thread-job".to_string(), structural_cues: vec!["source_type:chat_message".to_string()], metadata: Some(metadata), + embedding: None, category: cuemap::agent::chunker::ChunkCategory::Prose, }) .await; diff --git a/tests/lemmatization_quality_test.rs b/tests/lemmatization_quality_test.rs index 3a9f3a1..a17e3f7 100644 --- a/tests/lemmatization_quality_test.rs +++ b/tests/lemmatization_quality_test.rs @@ -5,10 +5,30 @@ use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; #[test] +fn test_embedded_dictionary_smoke() { + let cases = [ + ("favourites", "favorite"), + ("night-watchmen", "night-watchman"), + ("sustaining", "sustain"), + ("homeworlds", "homeworld"), + ]; + + for (word, expected) in cases { + assert_eq!(stem_word(word), expected, "failed to lemmatize {word}"); + } +} + +#[test] +#[ignore = "requires the optional tests/data/verbs.csv and tests/data/nouns.csv quality corpus"] fn test_generate_dictionary_and_verify() { let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); let data_dir = PathBuf::from(manifest_dir).join("tests/data"); + assert!( + data_dir.join("verbs.csv").is_file() && data_dir.join("nouns.csv").is_file(), + "optional lemmatization corpus is missing; add tests/data/verbs.csv and tests/data/nouns.csv before running this ignored quality test" + ); + let mut overrides: HashMap<String, String> = HashMap::new(); let standalone_lemmas = collect_standalone_lemmas(&data_dir); let mut total_checks = 0; diff --git a/tests/multi_tenant/mod.rs b/tests/multi_tenant/mod.rs index a78b130..654032a 100644 --- a/tests/multi_tenant/mod.rs +++ b/tests/multi_tenant/mod.rs @@ -2,7 +2,9 @@ use cuemap::config::TuningConfig; use cuemap::multi_tenant::*; use cuemap::structures::MainStats; use std::fs; +use std::time::Duration; use tempfile::tempdir; +use tokio::time::sleep; #[test] fn test_project_id_validation() { @@ -106,6 +108,28 @@ fn test_snapshot_roundtrip() { } } +#[test] +fn test_load_all_restores_every_project_snapshot() { + let dir = tempdir().unwrap(); + let project_id = "load-all-project".to_string(); + let first = MultiTenantEngine::with_snapshots_dir(dir.path(), TuningConfig::default()); + let context = first.get_or_create_project(project_id.clone()).unwrap(); + context.main.add_memory( + "load all content".to_string(), + vec!["load-all".to_string()], + None, + MainStats::default(), + true, + ); + first.save_project(&project_id).unwrap(); + + let second = MultiTenantEngine::with_snapshots_dir(dir.path(), TuningConfig::default()); + let results = second.load_all(); + assert!(matches!(results.get(&project_id), Some(Ok(())))); + let restored = second.get_project(&project_id).expect("project should be restored"); + assert_eq!(restored.main.total_memories(), 1); +} + #[test] fn test_delete_project() { let dir = tempdir().unwrap(); @@ -123,3 +147,83 @@ fn test_delete_project() { assert!(engine.delete_project(&project_id.to_string())); assert!(engine.get_project(&project_id.to_string()).is_none()); } + +#[test] +fn repository_ingestion_scope_is_persisted_in_project_metadata() { + let dir = tempdir().unwrap(); + let snapshots_dir = dir.path().join("snapshots"); + let watch_dir = dir.path().join("repo"); + fs::create_dir_all(&snapshots_dir).unwrap(); + fs::create_dir(&watch_dir).unwrap(); + let engine = MultiTenantEngine::with_snapshots_dir( + &snapshots_dir, + TuningConfig::default(), + ); + let project_id = "scope-persistence"; + engine + .get_or_create_project(project_id.to_string()) + .unwrap(); + + engine + .set_project_watch_config( + project_id, + watch_dir.to_string_lossy().to_string(), + vec!["src".to_string(), "README.md".to_string()], + vec!["docs/**".to_string()], + vec!["log".to_string()], + ) + .unwrap(); + + let metadata = engine + .load_project_meta(&project_id.to_string()) + .unwrap(); + assert!(metadata.agent_enabled); + assert_eq!(metadata.included_paths, vec!["src", "README.md"]); + assert_eq!(metadata.ignored_patterns, vec!["docs/**"]); + assert_eq!(metadata.ignored_extensions, vec!["log"]); +} + +#[tokio::test] +async fn periodic_snapshots_persist_projects_created_after_scheduler_start() { + let dir = tempdir().unwrap(); + let snapshots_dir = dir.path().join("snapshots"); + fs::create_dir_all(&snapshots_dir).unwrap(); + + let engine = MultiTenantEngine::with_snapshots_dir( + &snapshots_dir, + TuningConfig::default(), + ); + engine.start_periodic_snapshots(Duration::from_millis(20)); + + let project_id = "periodic-new-project".to_string(); + let context = engine.get_or_create_project(project_id.clone()).unwrap(); + context.main.add_memory( + "persisted by the periodic scheduler".to_string(), + vec!["snapshot:periodic".to_string()], + None, + MainStats::default(), + false, + ); + + for _ in 0..100 { + let reloaded = MultiTenantEngine::with_snapshots_dir( + &snapshots_dir, + TuningConfig::default(), + ); + if let Ok(reloaded_context) = reloaded.load_project(&project_id) { + let matches = reloaded_context.main.recall( + vec!["snapshot:periodic".to_string()], + 10, + false, + None, + ); + if matches.len() == 1 { + assert_eq!(matches[0].content, "persisted by the periodic scheduler"); + return; + } + } + sleep(Duration::from_millis(20)).await; + } + + panic!("periodic snapshot did not persist a project created after startup"); +} diff --git a/tests/projects/mod.rs b/tests/projects/mod.rs index 5518828..2a210f6 100644 --- a/tests/projects/mod.rs +++ b/tests/projects/mod.rs @@ -1,6 +1,27 @@ use cuemap::projects::*; use cuemap::structures::MainStats; +use cuemap::config::ServerConfig; +use cuemap::{normalization::NormalizationConfig, taxonomy::Taxonomy}; +use std::fs; use std::sync::Arc; +use std::sync::atomic::Ordering; + +fn write_artifact(dir: &std::path::Path, name: &str, content: &str) { + fs::create_dir_all(dir).unwrap(); + fs::write(dir.join(name), content).unwrap(); +} + +fn context_with_data_dir(project_id: &str, data_dir: &std::path::Path) -> ProjectContext { + let mut config = ServerConfig::default(); + config.server.data_dir = data_dir.to_string_lossy().to_string(); + ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(Default::default()), + config, + project_id.to_string(), + ) +} #[test] fn test_project_creation() { @@ -55,3 +76,143 @@ fn test_context_isolation() { // Verify they are different objects in memory (Arc pointers) assert!(!Arc::ptr_eq(&ctx1, &ctx2)); } + +#[test] +fn project_context_lifecycle_getters_and_artifact_reload_are_covered() { + let tmp = tempfile::tempdir().unwrap(); + let mut config = ServerConfig::default(); + config.server.data_dir = tmp.path().to_string_lossy().to_string(); + let ctx = ProjectContext::new_with_encoder( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(Default::default()), + config, + "health".to_string(), + None, + ); + + let before = ctx.get_last_activity(); + ctx.touch(); + assert!(ctx.get_last_activity() >= before); + assert_eq!(ctx.total_memories(), 0); + assert_eq!(ctx.get_cue_frequency("missing"), 0); + assert_eq!(ctx.cuebridge_artifact_summary().artifact_count, 0); + + let artifact_dir = tmp.path().join("artifacts").join("health"); + write_artifact( + &artifact_dir, + "aliases.json", + r#"{"schema_version":1,"artifact_type":"alias_pack","name":"health-aliases","entries":[{"id":"a1","from":"coffee","to":"tea","confidence":0.9}]}"#, + ); + let summary = ctx.reload_cuebridge_artifacts(&tmp.path().to_string_lossy(), "health"); + assert_eq!(summary.artifact_count, 1); + assert_eq!(summary.alias_entry_count, 1); + assert_eq!(ctx.cuebridge_artifact_summary().alias_entry_count, 1); +} + +#[test] +fn project_context_resolves_skip_lexicon_cache_and_language_paths() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = context_with_data_dir("resolve", tmp.path()); + assert_eq!(ctx.resolve_cues_from_text("", true), (Vec::new(), Vec::new(), Vec::new())); + + let skipped = ctx.resolve_cues_from_text("Coffee and tea", true); + assert!(!skipped.0.is_empty()); + assert!(skipped.1.is_empty()); + assert!(!skipped.2.is_empty()); + + let lex_id = ctx.lexicon.add_memory( + "beverage".to_string(), + vec!["coffee".to_string()], + None, + cuemap::structures::LexiconStats::default(), + true, + ); + let resolved = ctx.resolve_cues_from_text("coffee", false); + assert_eq!(resolved.1, vec![lex_id]); + assert!(resolved.0.iter().any(|cue| cue == "beverage")); + let cached = ctx.resolve_cues_from_text("coffee", false); + assert_eq!(cached.0, resolved.0); + assert!(cached.1.is_empty()); + + let python = ctx.resolve_cues_from_text_with_lang("Coffee", false, cuemap::nl::Language::Python); + assert!(!python.2.is_empty()); + assert!(ctx.query_cache.len() >= 2); + + let invalid = ctx.resolve_cues_from_text("bad:key", true); + assert!(!invalid.2.is_empty()); +} + +#[test] +fn project_context_symbol_router_routes_and_refreshes_after_index_changes() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = context_with_data_dir("symbols", tmp.path()); + ctx.main.add_memory( + "run implementation".to_string(), + vec!["defines_function:run".to_string()], + None, + MainStats::default(), + true, + ); + + let routed = ctx.resolve_cues_from_text("where is run used", true); + assert!(routed.0.iter().any(|cue| cue == "calls_function:run")); + assert!(routed.0.iter().any(|cue| cue == "calls_method:run")); + let generic = ctx.resolve_cues_from_text("run", true); + assert!(!generic.0.is_empty()); + + ctx.main.add_memory( + "new symbol".to_string(), + vec!["defines_function:deploy".to_string()], + None, + MainStats::default(), + true, + ); + let refreshed = ctx.resolve_cues_from_text("what does deploy do", true); + assert!(refreshed.0.iter().any(|cue| cue == "defines_function:deploy")); + assert!(ctx.main.cue_index_version() > 0); +} + +#[test] +fn project_context_inline_alias_expansion_filters_and_deduplicates() { + let tmp = tempfile::tempdir().unwrap(); + let ctx = context_with_data_dir("inline-alias", tmp.path()); + for cue in ["tea", "coffee", "cocoa"] { + ctx.main.add_memory(cue.to_string(), vec![cue.to_string()], None, MainStats::default(), true); + } + write_artifact( + &tmp.path().join("artifacts").join("inline-alias"), + "artifact-alias.json", + r#"{"schema_version":1,"artifact_type":"alias_pack","name":"artifact-alias","entries":[{"id":"bridge-1","from":"coffee","to":"tea","weight":0.7,"confidence":0.8}]}"#, + ); + ctx.reload_cuebridge_artifacts(&tmp.path().to_string_lossy(), "inline-alias"); + + let add_alias = |content: &str, cues: Vec<String>| { + ctx.aliases.add_memory(content.to_string(), cues, None, MainStats::default(), true); + }; + add_alias(r#"{"from":"coffee","to":"tea"}"#, vec!["type:alias".into(), "from:coffee".into(), "status:active".into()]); + add_alias(r#"{"from":"wrong","to":"cocoa","downweight":0.2}"#, vec!["type:alias".into(), "from:coffee".into(), "status:active".into()]); + add_alias("not-json", vec!["type:alias".into(), "from:coffee".into(), "status:active".into()]); + + let (expanded, trace) = ctx.expand_query_cues_with_trace(vec!["coffee".into(), "tea".into(), "missing".into()], &["coffee".into()]); + assert_eq!(expanded.iter().filter(|(cue, _)| cue == "tea").count(), 1); + assert!(expanded.iter().any(|(cue, weight)| cue == "coffee" && *weight == 1.0)); + assert_eq!(trace.len(), 1); + assert_eq!(trace[0].entry_id, "bridge-1"); + assert!(!expanded.iter().any(|(cue, _)| cue == "missing")); + assert_eq!(ctx.expand_query_cues(vec!["coffee".into()], &[]), vec![("coffee".into(), 1.0)]); +} + +#[test] +fn project_store_reuses_context_and_keeps_projects_isolated() { + let store = ProjectStore::new(); + let first = store.get_or_create("same"); + let second = store.get_or_create("same"); + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(store.projects.len(), 1); + first.last_activity.store(42, Ordering::Relaxed); + assert_eq!(second.get_last_activity(), 42); + let other = store.get_or_create("other"); + assert_eq!(store.projects.len(), 2); + assert!(!Arc::ptr_eq(&first, &other)); +} diff --git a/tests/recursive_crawl_test.rs b/tests/recursive_crawl_test.rs index ec65925..7985a7a 100644 --- a/tests/recursive_crawl_test.rs +++ b/tests/recursive_crawl_test.rs @@ -70,6 +70,7 @@ async fn test_recursive_crawl_depth_1() { watch_dir: String::new(), throttle_ms: 100, // Throttle to be polite state_file: None, + included_paths: Vec::new(), ignored_patterns: Vec::new(), ignored_extensions: Vec::new(), }; @@ -153,6 +154,7 @@ async fn test_job_phase_ordering() { watch_dir: String::new(), throttle_ms: 0, state_file: None, + included_paths: Vec::new(), ignored_patterns: Vec::new(), ignored_extensions: Vec::new(), }; diff --git a/tests/unit/api.rs b/tests/unit/api.rs new file mode 100644 index 0000000..39767b1 --- /dev/null +++ b/tests/unit/api.rs @@ -0,0 +1,3617 @@ + use super::*; + use crate::config::TuningConfig; + use crate::normalization::NormalizationConfig; + use crate::persistence::{CloudBackupConfig, CloudBackupManager}; + use crate::projects::ProjectContext; + use crate::structures::MainStats; + use crate::taxonomy::Taxonomy; + use axum::body::{to_bytes, Body}; + use axum::http::{Request, StatusCode}; + use std::collections::HashMap; + use std::sync::Arc; + use tower::ServiceExt; + + #[test] + fn source_event_time_prefers_explicit_and_reads_structured_metadata() { + let mut metadata = HashMap::new(); + metadata.insert( + "source_timestamp".to_string(), + serde_json::json!("2024-01-01T00:00:00.250Z"), + ); + + assert_eq!(source_event_time(Some(42.0), Some(&metadata)), Some(42.0)); + assert_eq!( + source_event_time(None, Some(&metadata)), + Some(1_704_067_200.25) + ); + } + + #[test] + fn api_helper_paths_cover_query_plan_cuebridge_and_parent_fusion() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "api-helper-paths".to_string(), + ); + + let mut metadata = HashMap::new(); + metadata.insert("source_role".to_string(), serde_json::json!("assistant")); + metadata.insert( + "source_session_id".to_string(), + serde_json::json!("helper-session"), + ); + metadata.insert( + "source_timestamp".to_string(), + serde_json::json!("2024-01-01T00:00:00Z"), + ); + + let first = ctx.main.add_memory( + "First sentence. Shared detail.".to_string(), + vec![ + "parent:helper-document".to_string(), + "chunk_idx:0".to_string(), + "source_role:assistant".to_string(), + "has:list".to_string(), + "has:number".to_string(), + ], + Some(metadata.clone()), + MainStats::default(), + true, + ); + let second = ctx.main.add_memory( + "Shared detail. Second sentence.".to_string(), + vec![ + "parent:helper-document".to_string(), + "chunk_idx:1".to_string(), + "source_role:assistant".to_string(), + "has:list".to_string(), + "has:number".to_string(), + ], + Some(metadata), + MainStats::default(), + true, + ); + + let mut expanded = vec![("has:list".to_string(), 1.0), ("today".to_string(), 1.0)]; + let plan = apply_query_plan( + &ctx, + Some("Can you list the deployment steps today in order?"), + Some("2024-01-02"), + &mut expanded, + ) + .expect("query plan should be compiled"); + assert!(plan.labels.iter().any(|label| label == "ordered_reconstruction")); + assert!(plan.labels.iter().any(|label| label == "multi_evidence_collection")); + assert!(expanded.iter().any(|(cue, _)| cue == "has:list")); + + let expansion = crate::cuebridge::CueBridgeGapExpansion { + artifact: "helper-pack".to_string(), + artifact_hash: "hash".to_string(), + entry_id: "entry".to_string(), + cue: "deployment".to_string(), + weight: 1.2, + confidence: 0.9, + score: 0.8, + }; + let mut existing = recall_result(0.5, 2); + existing.memory_id = first; + existing.score = 1.0; + existing.explain = Some(serde_json::json!({})); + let mut improved = recall_result(0.9, 3); + improved.memory_id = first; + improved.score = 2.0; + let mut added = recall_result(0.8, 2); + added.memory_id = second; + merge_cuebridge_gap_results( + &mut vec![existing], + vec![improved, added], + &[expansion], + true, + ); + + let mut candidates = Vec::new(); + for (memory_id, score, chunk_idx) in [(first, 2.0, 0), (second, 1.5, 1)] { + let mut result = recall_result(0.8, 3); + result.memory_id = memory_id; + result.score = score; + result.metadata.insert("chunk_idx".to_string(), serde_json::json!(chunk_idx)); + candidates.push(result); + } + let fused = build_parent_fusion_results(&ctx, candidates, 2, true); + assert_eq!(fused.len(), 1); + assert_eq!(fused[0].metadata["parent_fusion"], true); + assert!(fused[0].explain.is_some()); + + let mut merged = vec![fused[0].clone()]; + let mut weaker = fused[0].clone(); + weaker.score -= 10.0; + merge_parent_fusion_results(&mut merged, vec![weaker]); + let mut stronger = fused[0].clone(); + stronger.score += 10.0; + merge_parent_fusion_results(&mut merged, vec![stronger]); + assert_eq!(merged.len(), 1); + } + + #[test] + fn projection_helpers_cover_source_instruction_preference_and_decision_paths() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "api-projection-paths".to_string(), + ); + + let mut assistant_meta = HashMap::new(); + assistant_meta.insert("source_role".to_string(), serde_json::json!("assistant")); + assistant_meta.insert( + "source_session_id".to_string(), + serde_json::json!("projection-session"), + ); + assistant_meta.insert("source_turn_index".to_string(), serde_json::json!(2)); + let assistant_id = ctx.main.add_memory( + "Dessert migration answer with probability details.".to_string(), + vec![ + "source_role:assistant".to_string(), + "type:answer".to_string(), + "has:list".to_string(), + "dessert".to_string(), + "migration".to_string(), + "probability".to_string(), + ], + Some(assistant_meta.clone()), + MainStats::default(), + true, + ); + + let mut user_meta = HashMap::new(); + user_meta.insert("source_role".to_string(), serde_json::json!("user")); + user_meta.insert( + "source_session_id".to_string(), + serde_json::json!("projection-session"), + ); + user_meta.insert("source_turn_index".to_string(), serde_json::json!(1)); + let user_id = ctx.main.add_memory( + "User dessert migration preference.".to_string(), + vec![ + "source_role:user".to_string(), + "dessert".to_string(), + "migration".to_string(), + ], + Some(user_meta), + MainStats::default(), + true, + ); + + let standing_id = ctx.main.add_memory( + "Always use the migration probability threshold.".to_string(), + vec![ + "type:standing_instruction".to_string(), + "instruction:conditional".to_string(), + "instruction:always".to_string(), + "instruction_trigger:probability".to_string(), + "probability".to_string(), + "migration".to_string(), + ], + Some(assistant_meta.clone()), + MainStats::default(), + true, + ); + let preference_id = ctx.main.add_memory( + "I prefer dessert migration options.".to_string(), + vec![ + "type:preference".to_string(), + "preference:explicit".to_string(), + "preference_value:dessert".to_string(), + "preference_topic:dessert".to_string(), + "preference_contrast:migration".to_string(), + "dessert".to_string(), + "migration".to_string(), + "source_role:user".to_string(), + ], + Some(assistant_meta.clone()), + MainStats::default(), + true, + ); + let decision_id = ctx.main.add_memory( + "The naming decision selected migration.".to_string(), + vec![ + "type:decision".to_string(), + "type:selection".to_string(), + "type:naming".to_string(), + "migration".to_string(), + ], + Some(assistant_meta), + MainStats::default(), + true, + ); + + let plan = crate::facets::StructuralQueryPlan { + labels: vec![ + "__semantic_facets_removed__".to_string(), + "source_answer".to_string(), + "source_assistant".to_string(), + "personal_recommendation_context".to_string(), + "naming_decision".to_string(), + ], + ..Default::default() + }; + let mut assistant_result = recall_result(0.9, 3); + assistant_result.memory_id = assistant_id; + assistant_result.content = "Dessert migration answer with probability details.".to_string(); + assistant_result.created_at = 2.0; + assistant_result.metadata = [ + ("source_role".to_string(), serde_json::json!("assistant")), + ("source_session_id".to_string(), serde_json::json!("projection-session")), + ] + .into_iter() + .collect(); + let mut user_result = recall_result(0.6, 5); + user_result.memory_id = user_id; + user_result.content = "User dessert migration preference.".to_string(); + user_result.created_at = 1.0; + user_result.metadata = [ + ("source_role".to_string(), serde_json::json!("user")), + ("source_session_id".to_string(), serde_json::json!("projection-session")), + ("user_context_projection".to_string(), serde_json::json!(true)), + ] + .into_iter() + .collect(); + let all_results = vec![assistant_result.clone(), user_result.clone()]; + + assert!(source_answer_projection_requested(Some(&plan), Some("what did the assistant answer"))); + assert!(!source_answer_projection_cues(&ctx, Some(&plan), Some("list the answer"), &all_results).is_empty()); + assert!(!source_prompt_projection_cues( + &ctx, + Some(&plan), + Some("assistant answer about dessert migration"), + &all_results, + ) + .is_empty()); + assert!(user_context_projection_requested( + Some(&crate::facets::StructuralQueryPlan { + labels: vec!["__semantic_facets_removed__".to_string()], + ..Default::default() + }), + Some("what advice about dessert migration should I use"), + )); + assert!(!user_context_projection_cues( + &ctx, + Some(&crate::facets::StructuralQueryPlan { + labels: vec!["__semantic_facets_removed__".to_string(), "personal_recommendation_context".to_string()], + ..Default::default() + }), + Some("what advice about dessert migration should I use"), + &all_results, + ) + .is_empty()); + + let standing = standing_instruction_projection_cues( + &ctx, + Some(&plan), + Some("what probability migration should we use"), + ); + assert!(!standing.cues.is_empty()); + assert!(!standing.anchors.is_empty()); + let preference = preference_projection_cues( + &ctx, + Some(&plan), + Some("which dessert migration do I prefer"), + ); + assert!(!preference.cues.is_empty()); + assert!(!preference.anchors.is_empty()); + assert!(!decision_projection_cues(&ctx, Some(&plan), &all_results).is_empty()); + + let mut projected = all_results.clone(); + let mut standing_result = recall_result(0.4, 3); + standing_result.memory_id = standing_id; + standing_result.intersection_count = 5; + let mut preference_result = recall_result(0.4, 3); + preference_result.memory_id = preference_id; + preference_result.intersection_count = 5; + let mut decision_result = recall_result(0.4, 3); + decision_result.memory_id = decision_id; + decision_result.intersection_count = 5; + merge_source_answer_projection_results(&mut projected, vec![assistant_result.clone()]); + merge_source_prompt_projection_results( + &mut projected, + vec![user_result.clone()], + Some("assistant answer about dessert migration"), + ); + merge_user_context_projection_results(&mut projected, vec![user_result.clone()]); + merge_standing_instruction_projection_results( + &ctx, + &mut projected, + vec![standing_result], + &standing.anchors, + ); + merge_preference_projection_results( + &ctx, + &mut projected, + vec![preference_result], + &preference.anchors, + ); + merge_decision_projection_results(&mut projected, vec![decision_result]); + apply_source_role_preference(&mut projected, Some(&plan)); + apply_source_answer_adjacency_preference(&mut projected, Some(&plan)); + apply_user_context_adjacency_preference( + &mut projected, + Some(&crate::facets::StructuralQueryPlan { + labels: vec!["__semantic_facets_removed__".to_string()], + ..Default::default() + }), + Some("what advice about dessert migration should I use"), + ); + assert!(projected.iter().any(|result| result.metadata.contains_key("decision_projection"))); + } + + fn recall_result( + match_integrity: f64, + intersection_count: usize, + ) -> crate::engine::RecallResult { + crate::engine::RecallResult { + memory_id: 1, + content: "content".to_string(), + score: 1.0, + match_integrity, + intersection_count, + recency_score: 0.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata: HashMap::new(), + explain: None, + } + } + + #[test] + fn parent_fusion_defaults_off_and_force_runs() { + let results = vec![recall_result(0.95, 4)]; + + assert!(!should_run_parent_fusion( + &results, + ParentFusionMode::Off, + None, + Some("summarize the key points"), + )); + assert!(should_run_parent_fusion( + &results, + ParentFusionMode::Force, + None, + Some("plain lookup"), + )); + } + + #[test] + fn parent_fusion_auto_requires_synthesis_query() { + assert!(!should_run_parent_fusion( + &[recall_result(0.4, 1)], + ParentFusionMode::Auto, + None, + Some("what is my favorite dessert"), + )); + assert!(should_run_parent_fusion( + &[recall_result(0.4, 1)], + ParentFusionMode::Auto, + None, + Some("summarize my language service progress in order"), + )); + } + + #[test] + fn ordered_reconstruction_is_opt_in_and_intent_gated() { + let mut intent = crate::facets::StructuralQueryPlan::default(); + assert!(!should_run_ordered_reconstruction( + OrderedReconstructionMode::Off, + Some(&intent) + )); + assert!(should_run_ordered_reconstruction( + OrderedReconstructionMode::Force, + None + )); + assert!(!should_run_ordered_reconstruction( + OrderedReconstructionMode::Auto, + Some(&intent) + )); + + intent.labels.push("ordered_reconstruction".to_string()); + assert!(should_run_ordered_reconstruction( + OrderedReconstructionMode::Auto, + Some(&intent) + )); + + intent.labels.clear(); + intent + .labels + .push("multi_evidence_collection".to_string()); + assert!(should_run_ordered_reconstruction( + OrderedReconstructionMode::Auto, + Some(&intent) + )); + } + + #[test] + fn evidence_coverage_is_opt_in_and_intent_gated() { + let mut intent = crate::facets::StructuralQueryPlan::default(); + assert!(!should_run_evidence_coverage( + EvidenceCoverageMode::Off, + Some(&intent) + )); + assert!(should_run_evidence_coverage( + EvidenceCoverageMode::Force, + None + )); + assert!(!should_run_evidence_coverage( + EvidenceCoverageMode::Auto, + Some(&intent) + )); + + intent.labels.push("multi_evidence_summary".to_string()); + assert!(should_run_evidence_coverage( + EvidenceCoverageMode::Auto, + Some(&intent) + )); + + intent.labels.clear(); + intent + .labels + .push("multi_evidence_collection".to_string()); + assert!(should_run_evidence_coverage( + EvidenceCoverageMode::Auto, + Some(&intent) + )); + + intent.labels.clear(); + intent.labels.push("ordered_reconstruction".to_string()); + assert!(should_run_evidence_coverage( + EvidenceCoverageMode::Auto, + Some(&intent) + )); + } + + #[test] + fn evidence_coverage_selects_diverse_session_evidence() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "evidence_coverage_test".to_string(), + ); + + let add_turn = |session: &str, + order: i64, + plan: Option<i64>, + content: &str, + cues: &[&str]| + -> MemoryId { + let mut metadata = HashMap::new(); + metadata.insert("source_session_id".to_string(), serde_json::json!(session)); + metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); + if let Some(plan) = plan { + metadata.insert("source_plan_idx".to_string(), serde_json::json!(plan)); + } + ctx.main.add_memory( + content.to_string(), + cues.iter().map(|cue| cue.to_string()).collect(), + Some(metadata), + MainStats::default(), + false, + ) + }; + + let integration = add_turn( + "thread-a", + 1, + Some(1), + "We designed language service integration.", + &["source_role:assistant", "type:answer", "has:list", "language", "service", "integration", "architecture"], + ); + let deployment = add_turn( + "thread-a", + 2, + Some(2), + "We planned deployment and release steps.", + &["source_role:assistant", "type:answer", "deployment", "release", "service"], + ); + let performance = add_turn( + "thread-a", + 3, + Some(3), + "We improved performance and latency.", + &["source_role:assistant", "type:answer", "performance", "latency", "service"], + ); + let unrelated = add_turn( + "thread-a", + 4, + Some(4), + "We discussed a lunch menu.", + &["source_role:assistant", "type:answer", "lunch", "menu"], + ); + let distractor = add_turn( + "thread-b", + 1, + Some(2), + "A different deployment discussion happened elsewhere.", + &["source_role:assistant", "type:answer", "deployment", "service"], + ); + + let pivot = crate::engine::RecallResult { + memory_id: deployment, + content: "We planned deployment and release steps.".to_string(), + score: 140.0, + match_integrity: 0.6, + intersection_count: 2, + recency_score: 1.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata: HashMap::new(), + explain: None, + }; + let pivot_score = pivot.score; + + let evidence = evidence_coverage_results( + &ctx, + &[ + ("language".to_string(), 1.0), + ("service".to_string(), 0.8), + ("integration".to_string(), 1.0), + ("deployment".to_string(), 1.0), + ("performance".to_string(), 1.0), + ], + &[pivot], + 10, + 100, + 1, + true, + ); + + let ids: Vec<MemoryId> = evidence.iter().map(|result| result.memory_id).collect(); + assert!(ids.contains(&integration)); + assert!(ids.contains(&deployment)); + assert!(ids.contains(&performance)); + assert!(!ids.contains(&unrelated)); + assert!(!ids.contains(&distractor)); + assert!(evidence + .iter() + .all(|result| result.metadata.contains_key("evidence_coverage"))); + assert!(evidence.iter().any(|result| result + .metadata + .contains_key("evidence_coverage_source_plan"))); + assert!(evidence + .iter() + .all(|result| result.score < pivot_score)); + } + + #[test] + fn slate_rerank_is_mode_and_intent_gated() { + let mut intent = crate::facets::StructuralQueryPlan::default(); + intent.labels.push("multi_evidence_summary".to_string()); + + assert!(!slate_rerank_requested( + OrderedReconstructionMode::Off, + EvidenceCoverageMode::Off, + Some(&intent) + )); + assert!(slate_rerank_requested( + OrderedReconstructionMode::Auto, + EvidenceCoverageMode::Off, + Some(&intent) + )); + + let plain_intent = crate::facets::StructuralQueryPlan::default(); + assert!(!slate_rerank_requested( + OrderedReconstructionMode::Auto, + EvidenceCoverageMode::Off, + Some(&plain_intent) + )); + } + + #[test] + fn slate_rerank_promotes_coverage_candidates_below_protected_top() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "slate_rerank_test".to_string(), + ); + + let add_turn = |session: &str, + order: i64, + role: &str, + content: &str, + cues: &[&str]| + -> MemoryId { + let mut metadata = HashMap::new(); + metadata.insert("source_session_id".to_string(), serde_json::json!(session)); + metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); + metadata.insert("source_role".to_string(), serde_json::json!(role)); + ctx.main.add_memory( + content.to_string(), + cues.iter().map(|cue| cue.to_string()).collect(), + Some(metadata), + MainStats::default(), + false, + ) + }; + let make_result = |memory_id: MemoryId, + score: f64, + metadata: HashMap<String, serde_json::Value>| + -> crate::engine::RecallResult { + crate::engine::RecallResult { + memory_id, + content: format!("memory {memory_id}"), + score, + match_integrity: 0.2, + intersection_count: 1, + recency_score: 0.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata, + explain: None, + } + }; + + let protected_a = add_turn( + "thread-a", + 1, + "assistant", + "Protected top result A.", + &["overview"], + ); + let protected_b = add_turn( + "thread-a", + 2, + "assistant", + "Protected top result B.", + &["overview"], + ); + let protected_c = add_turn( + "thread-a", + 3, + "assistant", + "Protected top result C.", + &["overview"], + ); + let mut results = vec![ + make_result(protected_a, 300.0, HashMap::new()), + make_result(protected_b, 290.0, HashMap::new()), + make_result(protected_c, 280.0, HashMap::new()), + ]; + + for rank in 0..25 { + let id = add_turn( + "thread-b", + rank, + "assistant", + "Generic distractor.", + &["generic", "discussion"], + ); + results.push(make_result(id, 270.0 - rank as f64, HashMap::new())); + } + + let relevant_late = add_turn( + "thread-a", + 24, + "assistant", + "We covered deployment and latency.", + &["deployment", "latency", "service", "type:answer"], + ); + let relevant_later = add_turn( + "thread-a", + 40, + "assistant", + "We also covered integration architecture.", + &["integration", "architecture", "service", "type:answer"], + ); + let mut evidence_metadata = HashMap::new(); + evidence_metadata.insert("evidence_coverage".to_string(), serde_json::json!(true)); + results.push(make_result(relevant_late, 150.0, evidence_metadata.clone())); + results.push(make_result(relevant_later, 149.0, evidence_metadata)); + + let mut intent = crate::facets::StructuralQueryPlan::default(); + intent.labels.push("multi_evidence_summary".to_string()); + let moved = apply_slate_rerank( + &ctx, + &mut results, + &[ + ("deployment".to_string(), 1.0), + ("latency".to_string(), 1.0), + ("integration".to_string(), 1.0), + ("architecture".to_string(), 1.0), + ("service".to_string(), 0.8), + ], + OrderedReconstructionMode::Auto, + EvidenceCoverageMode::Off, + Some(&intent), + 100, + ); + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let ids: Vec<MemoryId> = results.iter().map(|result| result.memory_id).collect(); + assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); + assert!(ids.iter().position(|id| *id == relevant_late).unwrap() < 20); + assert!(ids.iter().position(|id| *id == relevant_later).unwrap() < 20); + assert!(moved >= 2); + assert!(results + .iter() + .any(|result| result.memory_id == relevant_late + && result.metadata.contains_key("slate_rerank"))); + } + + #[test] + fn slate_rerank_promotes_strong_summary_candidates_without_helper_metadata() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "slate_rerank_summary_signal_test".to_string(), + ); + + let add_turn = |session: &str, + order: i64, + content: &str, + cues: &[&str]| + -> MemoryId { + let mut metadata = HashMap::new(); + metadata.insert("source_session_id".to_string(), serde_json::json!(session)); + metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); + ctx.main.add_memory( + content.to_string(), + cues.iter().map(|cue| cue.to_string()).collect(), + Some(metadata), + MainStats::default(), + false, + ) + }; + let make_result = |memory_id: MemoryId, + score: f64| + -> crate::engine::RecallResult { + crate::engine::RecallResult { + memory_id, + content: format!("memory {memory_id}"), + score, + match_integrity: 0.2, + intersection_count: 1, + recency_score: 0.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata: HashMap::new(), + explain: None, + } + }; + + let protected_a = add_turn("thread-a", 1, "Protected A.", &["overview"]); + let protected_b = add_turn("thread-a", 2, "Protected B.", &["overview"]); + let protected_c = add_turn("thread-a", 3, "Protected C.", &["overview"]); + let mut results = vec![ + make_result(protected_a, 300.0), + make_result(protected_b, 290.0), + make_result(protected_c, 280.0), + ]; + + for rank in 0..25 { + let id = add_turn( + "thread-b", + rank, + "Generic project discussion.", + &["generic", "project"], + ); + results.push(make_result(id, 270.0 - rank as f64)); + } + + let relevant = add_turn( + "thread-c", + 8, + "City autocomplete in the weather app uses a debounced API lookup.", + &["city", "autocomplete", "weather", "app", "lookup"], + ); + results.push(make_result(relevant, 150.0)); + + let mut intent = crate::facets::StructuralQueryPlan::default(); + intent.labels.push("multi_evidence_summary".to_string()); + let moved = apply_slate_rerank( + &ctx, + &mut results, + &[ + ("city".to_string(), 1.0), + ("autocomplete".to_string(), 1.0), + ("weather".to_string(), 1.0), + ("app".to_string(), 0.8), + ("implementation".to_string(), 0.8), + ], + OrderedReconstructionMode::Auto, + EvidenceCoverageMode::Off, + Some(&intent), + 100, + ); + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let ids: Vec<MemoryId> = results.iter().map(|result| result.memory_id).collect(); + assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); + assert!(ids.iter().position(|id| *id == relevant).unwrap() < 20); + assert!(moved >= 1); + assert!(results + .iter() + .any(|result| result.memory_id == relevant + && result.metadata.contains_key("slate_rerank"))); + } + + #[test] + fn slate_rerank_promotes_standing_instruction_for_instruction_query() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "slate_rerank_instruction_test".to_string(), + ); + + let add_turn = |content: &str, cues: &[&str]| -> MemoryId { + ctx.main.add_memory( + content.to_string(), + cues.iter().map(|cue| cue.to_string()).collect(), + None, + MainStats::default(), + false, + ) + }; + let make_result = |memory_id: MemoryId, + score: f64| + -> crate::engine::RecallResult { + crate::engine::RecallResult { + memory_id, + content: format!("memory {memory_id}"), + score, + match_integrity: 0.1, + intersection_count: 1, + recency_score: 0.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata: HashMap::new(), + explain: None, + } + }; + + let protected_a = add_turn("Protected A", &["layout"]); + let protected_b = add_turn("Protected B", &["layout"]); + let protected_c = add_turn("Protected C", &["layout"]); + let mut results = vec![ + make_result(protected_a, 300.0), + make_result(protected_b, 290.0), + make_result(protected_c, 280.0), + ]; + for rank in 0..45 { + let id = add_turn("Generic layout discussion", &["layout", "project"]); + results.push(make_result(id, 270.0 - rank as f64)); + } + + let instruction = add_turn( + "Always include semantic HTML5 tag usage details when I ask about markup structure.", + &[ + "type:standing_instruction", + "instruction_trigger:markup", + "semantic", + "html5", + "tag", + "structure", + ], + ); + results.push(make_result(instruction, 120.0)); + + let mut intent = crate::facets::StructuralQueryPlan::default(); + intent.labels.push("instruction_applicable".to_string()); + let moved = apply_slate_rerank( + &ctx, + &mut results, + &[ + ("blog".to_string(), 1.0), + ("layout".to_string(), 1.0), + ("header".to_string(), 1.0), + ("navigation".to_string(), 1.0), + ("footer".to_string(), 1.0), + ], + OrderedReconstructionMode::Auto, + EvidenceCoverageMode::Off, + Some(&intent), + 100, + ); + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let ids: Vec<MemoryId> = results.iter().map(|result| result.memory_id).collect(); + assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); + assert!(ids.iter().position(|id| *id == instruction).unwrap() < 20); + assert!(moved >= 1); + assert!(results + .iter() + .any(|result| result.memory_id == instruction + && result.metadata.contains_key("slate_rerank"))); + } + + #[test] + fn slate_rerank_orders_selected_ordered_candidates_after_selection() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "slate_rerank_ordered_test".to_string(), + ); + + let add_turn = |session: &str, order: i64, content: &str, cues: &[&str]| -> MemoryId { + let mut metadata = HashMap::new(); + metadata.insert("source_session_id".to_string(), serde_json::json!(session)); + metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); + ctx.main.add_memory( + content.to_string(), + cues.iter().map(|cue| cue.to_string()).collect(), + Some(metadata), + MainStats::default(), + false, + ) + }; + let make_result = |memory_id: MemoryId, + score: f64, + ordered: bool| + -> crate::engine::RecallResult { + let mut metadata = HashMap::new(); + if ordered { + metadata.insert("ordered_reconstruction".to_string(), serde_json::json!(true)); + } + crate::engine::RecallResult { + memory_id, + content: format!("memory {memory_id}"), + score, + match_integrity: 0.3, + intersection_count: 1, + recency_score: 0.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata, + explain: None, + } + }; + + let protected_a = add_turn("thread-a", 1, "Protected A", &["bootstrap"]); + let protected_b = add_turn("thread-a", 2, "Protected B", &["bootstrap"]); + let protected_c = add_turn("thread-a", 3, "Protected C", &["bootstrap"]); + let mut results = vec![ + make_result(protected_a, 300.0, false), + make_result(protected_b, 290.0, false), + make_result(protected_c, 280.0, false), + ]; + for rank in 0..25 { + let id = add_turn("thread-b", rank, "Generic project discussion", &["project"]); + results.push(make_result(id, 270.0 - rank as f64, false)); + } + + let first = add_turn("thread-a", 5, "Bootstrap CDN setup", &["bootstrap", "cdn"]); + let second = add_turn("thread-a", 7, "Bootstrap form classes", &["bootstrap", "form"]); + let third = add_turn("thread-a", 11, "Bootstrap modal upgrade", &["bootstrap", "modal"]); + results.push(make_result(third, 151.0, true)); + results.push(make_result(first, 150.0, true)); + results.push(make_result(second, 149.0, true)); + + let mut intent = crate::facets::StructuralQueryPlan::default(); + intent.labels.push("ordered_reconstruction".to_string()); + let moved = apply_slate_rerank( + &ctx, + &mut results, + &[ + ("bootstrap".to_string(), 1.0), + ("cdn".to_string(), 1.0), + ("form".to_string(), 1.0), + ("modal".to_string(), 1.0), + ], + OrderedReconstructionMode::Auto, + EvidenceCoverageMode::Off, + Some(&intent), + 100, + ); + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let ids: Vec<MemoryId> = results.iter().map(|result| result.memory_id).collect(); + assert_eq!(&ids[..3], &[protected_a, protected_b, protected_c]); + let first_pos = ids.iter().position(|id| *id == first).unwrap(); + let second_pos = ids.iter().position(|id| *id == second).unwrap(); + let third_pos = ids.iter().position(|id| *id == third).unwrap(); + assert!(first_pos < 20); + assert!(second_pos < 20); + assert!(third_pos < 20); + assert!(first_pos < second_pos); + assert!(second_pos < third_pos); + assert!(moved >= 3); + } + + #[test] + fn ordered_reconstruction_scans_selected_session_in_order() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "ordered_test".to_string(), + ); + + let add_turn = |session: &str, order: i64, content: &str, cues: &[&str]| -> MemoryId { + let mut metadata = HashMap::new(); + metadata.insert("source_session_id".to_string(), serde_json::json!(session)); + metadata.insert("source_turn_index".to_string(), serde_json::json!(order)); + ctx.main.add_memory( + content.to_string(), + cues.iter().map(|cue| cue.to_string()).collect(), + Some(metadata), + MainStats::default(), + false, + ) + }; + + let first = add_turn( + "thread-a", + 1, + "We integrated the language detection service.", + &["language", "service", "integrate"], + ); + let second = add_turn( + "thread-a", + 2, + "Then we optimized translation service latency.", + &["translation", "service", "optimize"], + ); + let distractor = add_turn( + "thread-b", + 1, + "A different service discussion happened elsewhere.", + &["translation", "service", "discussion"], + ); + + let mut pivot_metadata = HashMap::new(); + pivot_metadata.insert("source_session_id".to_string(), serde_json::json!("thread-a")); + pivot_metadata.insert("source_turn_index".to_string(), serde_json::json!(2)); + let pivot = crate::engine::RecallResult { + memory_id: second, + content: "Then we optimized translation service latency.".to_string(), + score: 120.0, + match_integrity: 0.6, + intersection_count: 2, + recency_score: 1.0, + reinforcement_score: 0.0, + salience_score: 0.0, + created_at: 0.0, + metadata: pivot_metadata, + explain: None, + }; + + let ordered = ordered_reconstruction_results( + &ctx, + &[ + ("language".to_string(), 1.0), + ("translation".to_string(), 1.0), + ("service".to_string(), 1.0), + ("optimize".to_string(), 1.0), + ], + &[pivot], + 10, + 100, + 1, + true, + ); + + let ids: Vec<MemoryId> = ordered.iter().map(|result| result.memory_id).collect(); + assert!(ids.contains(&first)); + assert!(ids.contains(&second)); + assert!(!ids.contains(&distractor)); + assert!(ordered + .iter() + .all(|result| result.metadata.contains_key("ordered_reconstruction"))); + } + + #[test] + fn segment_link_requires_parent_and_chunk_idx() { + assert_eq!( + segment_link_from_cues(&[ + "parent:abc".to_string(), + "chunk_idx:7".to_string(), + "source_role:user".to_string(), + ]), + Some(("parent:abc".to_string(), 7)) + ); + assert_eq!( + segment_link_from_cues(&["parent:abc".to_string()]), + None + ); + } + + #[test] + fn stitched_chunk_join_removes_overlapped_sentences() { + let joined = join_stitched_chunk_contents(&[ + "First sentence. Shared sentence.".to_string(), + "Shared sentence. Final sentence.".to_string(), + ]); + + assert_eq!(joined, "First sentence. Shared sentence. Final sentence."); + } + + #[cfg(any())] + #[test] + fn source_answer_projection_requires_assistant_answer_language() { + let source_answer_intent = crate::facets::StructuralQueryPlan { + labels: vec!["source_answer".to_string()], + ..Default::default() + }; + assert!(!source_answer_projection_requested( + Some(&source_answer_intent), + Some("What did I buy last week?") + )); + assert!(source_answer_projection_requested( + Some(&source_answer_intent), + Some("What was in the assistant answer?") + )); + + let assistant_intent = crate::facets::StructuralQueryPlan { + labels: vec!["source_assistant".to_string()], + ..Default::default() + }; + assert!(source_answer_projection_requested( + Some(&assistant_intent), + Some("Can you remind me?") + )); + } + + #[cfg(any())] + #[test] + fn user_context_projection_targets_advice_without_source_intents() { + assert!(user_context_projection_requested( + None, + Some("I've been having trouble with battery life. Any tips?") + )); + + let recommendation_intent = crate::facets::StructuralQueryPlan { + labels: vec!["recommendation".to_string()], + ..Default::default() + }; + assert!(user_context_projection_requested( + Some(&recommendation_intent), + Some("Can you recommend something for me?") + )); + assert!(!user_context_projection_requested( + Some(&recommendation_intent), + Some("Can you suggest a hotel for my upcoming trip to Miami?") + )); + + for label in ["source_answer", "source_assistant", "source_user", "decision_selection"] { + let intent = crate::facets::StructuralQueryPlan { + labels: vec![label.to_string()], + ..Default::default() + }; + assert!( + !user_context_projection_requested(Some(&intent), Some("Any tips?")), + "source-specific query should not request user context projection for {label}" + ); + } + } + + #[cfg(any())] + #[test] + fn user_context_projection_anchors_require_specific_context() { + let phone_accessory_anchors = + projection_anchor_cues(Some("Can you suggest some useful accessories for my phone?")); + assert_eq!( + phone_accessory_anchors, + vec!["accessory".to_string(), "phone".to_string()] + ); + + let media_recommendation_anchors = + projection_anchor_cues(Some("Can you recommend a show or movie for me to watch tonight?")); + assert!(media_recommendation_anchors.is_empty()); + + let troubleshooting_anchors = projection_anchor_cues(Some( + "I've been having trouble with the battery life on my phone lately. Any tips?", + )); + assert!(troubleshooting_anchors.contains(&"battery".to_string())); + assert!(troubleshooting_anchors.contains(&"life".to_string())); + assert!(troubleshooting_anchors.contains(&"phone".to_string())); + + let navigation_anchors = projection_anchor_cues(Some( + "I'm a bit anxious about getting around Tokyo. Do you have any helpful tips?", + )); + assert_eq!( + navigation_anchors, + vec!["anxious".to_string(), "tokyo".to_string()] + ); + + let relevant = "assistant: A power bank can help with phone battery life while traveling."; + let incidental = "assistant: You could schedule a phone call during the morning."; + + assert!(projection_anchor_match_count(relevant, &troubleshooting_anchors) >= 2); + assert!(projection_anchor_match_count(incidental, &troubleshooting_anchors) < 2); + assert!(!projection_pivot_matches_context( + "assistant: A camera bag can complement your Sony setup.", + 4, + &phone_accessory_anchors, + true + )); + assert!(!projection_pivot_matches_context( + "assistant: A camera bag can complement your Sony setup.", + 3, + &phone_accessory_anchors, + true + )); + assert!(projection_pivot_matches_context( + "assistant: A phone case is a useful accessory for your phone setup.", + 2, + &phone_accessory_anchors, + true + )); + assert!(!projection_pivot_matches_context( + "assistant: A camera bag can complement your Sony setup.", + 4, + &phone_accessory_anchors, + false + )); + + let vague_interest_intent = crate::facets::StructuralQueryPlan { + labels: vec!["vague_interest_recommendation".to_string()], + ..Default::default() + }; + assert!(suppress_user_context_projection_for_intent(Some( + &vague_interest_intent + ))); + } + + #[cfg(any())] + #[test] + fn standing_instruction_projection_is_intent_gated() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "standing_instruction_test".to_string(), + ); + + let instruction_id = ctx.main.add_memory( + "Always provide fallback strategies when I ask about error handling in API services." + .to_string(), + vec!["api".to_string(), "error_handling".to_string()], + None, + MainStats::default(), + false, + ); + + assert!(standing_instruction_projection_cues( + &ctx, + None, + Some("What are some ways I can manage problems that come up when my API calls fail?") + ) + .cues + .is_empty()); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec!["instruction_applicable".to_string()], + ..Default::default() + }; + let projection = standing_instruction_projection_cues( + &ctx, + Some(&intent), + Some("What are some ways I can manage problems that come up when my API calls fail?"), + ); + + assert!(projection + .cues + .iter() + .any(|(cue, _)| cue == "type:standing_instruction")); + assert!(projection + .cues + .iter() + .any(|(cue, _)| cue == "instruction_trigger:api")); + + let projection_results = ctx.main.recall_weighted( + projection.cues.clone(), + 10, + false, + None, + 1, + false, + true, + None, + None, + ); + let mut all_results = Vec::new(); + merge_standing_instruction_projection_results( + &ctx, + &mut all_results, + projection_results, + &projection.anchors, + ); + + let projected = all_results + .iter() + .find(|result| result.memory_id == instruction_id) + .expect("standing instruction should be projected"); + assert!(projected + .metadata + .contains_key("standing_instruction_projection")); + } + + #[cfg(any())] + #[test] + fn standing_instruction_projection_uses_morphological_anchor_variants() { + let anchors = + standing_instruction_projection_anchors(Some("How do I implement a login feature?")); + assert!(anchors.contains(&"implement".to_string())); + assert!(anchors.contains(&"implementation".to_string())); + + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "standing_instruction_morphology_test".to_string(), + ); + + ctx.main.add_memory( + "Always format code snippets with syntax highlighting when I ask about implementation details." + .to_string(), + Vec::new(), + None, + MainStats::default(), + false, + ); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec!["instruction_applicable".to_string()], + ..Default::default() + }; + let projection = standing_instruction_projection_cues( + &ctx, + Some(&intent), + Some("How do I implement a login feature?"), + ); + + assert!(projection + .cues + .iter() + .any(|(cue, _)| cue == "instruction_trigger:implementation")); + } + + #[cfg(any())] + #[test] + fn standing_instruction_projection_maps_chance_to_probability_anchor() { + let anchors = standing_instruction_projection_anchors(Some( + "How do I calculate the chance of drawing a red card from a standard deck?", + )); + assert!(anchors.contains(&"chance".to_string())); + assert!(anchors.contains(&"probability".to_string())); + + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "standing_instruction_probability_test".to_string(), + ); + + ctx.main.add_memory( + "Always provide step-by-step explanations with concrete examples when I ask about probability concepts." + .to_string(), + Vec::new(), + None, + MainStats::default(), + false, + ); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec!["instruction_applicable".to_string()], + ..Default::default() + }; + let projection = standing_instruction_projection_cues( + &ctx, + Some(&intent), + Some("How do I calculate the chance of drawing a red card from a standard deck?"), + ); + + assert!(projection + .cues + .iter() + .any(|(cue, _)| cue == "instruction_trigger:probability")); + } + + #[cfg(any())] + #[test] + fn preference_projection_is_intent_gated() { + let ctx = ProjectContext::new( + NormalizationConfig::default(), + Taxonomy::default(), + Arc::new(TuningConfig::default()), + crate::config::ServerConfig::default(), + "preference_projection_test".to_string(), + ); + + let memory_id = ctx.main.add_memory( + "I prefer geometric vector methods over purely trigonometric formulas for clarity, so can you explain how to use vector algebra to calculate geodesic length between two points on a sphere?".to_string(), + vec![ + "sphere".to_string(), + "two_point".to_string(), + "vector".to_string(), + "geodesic".to_string(), + ], + None, + MainStats::default(), + false, + ); + + let query = "Can you show me how to find the shortest path between two points on a sphere?"; + assert!(preference_projection_cues(&ctx, None, Some(query)) + .cues + .is_empty()); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec!["preference_applicable".to_string()], + ..Default::default() + }; + let projection = preference_projection_cues(&ctx, Some(&intent), Some(query)); + + assert!(projection + .cues + .iter() + .any(|(cue, _)| cue == "type:preference")); + assert!(projection + .cues + .iter() + .any(|(cue, _)| cue == "sphere" || cue == "two_point")); + + let projection_results = ctx.main.recall_weighted( + projection.cues.clone(), + 10, + false, + None, + 1, + false, + true, + None, + None, + ); + let mut all_results = Vec::new(); + merge_preference_projection_results( + &ctx, + &mut all_results, + projection_results, + &projection.anchors, + ); + + let projected = all_results + .iter() + .find(|result| result.memory_id == memory_id) + .expect("matching preference should be projected"); + assert!(projected.metadata.contains_key("preference_projection")); + } + + #[cfg(any())] + #[test] + fn user_context_projection_merge_marks_and_updates_results() { + let mut existing = recall_result(0.2, 1); + existing.memory_id = 10; + existing.score = 10.0; + + let mut projected = recall_result(0.8, 3); + projected.memory_id = 10; + projected.score = 50.0; + projected + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + + let mut all_results = vec![existing]; + merge_user_context_projection_results(&mut all_results, vec![projected]); + + assert_eq!(all_results.len(), 1); + assert_eq!(all_results[0].score, 50.0); + assert!(all_results[0] + .metadata + .contains_key("user_context_projection")); + } + + #[cfg(any())] + #[test] + fn source_prompt_projection_filters_short_scaffold_prompts() { + let mut scaffold = recall_result(0.1, 1); + scaffold.memory_id = 20; + scaffold.content = "user: Write another scene".to_string(); + scaffold.score = 5000.0; + scaffold + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + + let mut source = recall_result(0.8, 9); + source.memory_id = 21; + source.content = + "user: Write a comedy movie scene. Andy wears an untidy stained white shirt." + .to_string(); + source.score = 600.0; + source + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + + let mut results = Vec::new(); + merge_source_prompt_projection_results( + &mut results, + vec![scaffold, source], + Some("what was Andy wearing in the script you wrote for the comedy movie scene?"), + ); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].memory_id, 21); + assert!(results[0] + .metadata + .contains_key("source_prompt_projection")); + assert!(results[0].score > 600.0); + } + + #[cfg(any())] + #[test] + fn user_context_adjacency_prefers_nearest_prior_user_turn() { + let mut expected = recall_result(0.2, 1); + expected.memory_id = 30; + expected.score = 100.0; + expected.created_at = 1.0; + expected + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + expected.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-3"), + ); + expected + .metadata + .insert("user_context_projection".to_string(), serde_json::json!(true)); + + let mut pivot = recall_result(0.6, 2); + pivot.memory_id = 31; + pivot.score = 500.0; + pivot.created_at = 2.0; + pivot + .metadata + .insert("source_role".to_string(), serde_json::json!("assistant")); + pivot.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-3"), + ); + + let mut later_user = recall_result(0.2, 1); + later_user.memory_id = 32; + later_user.score = 900.0; + later_user.created_at = 3.0; + later_user + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + later_user.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-3"), + ); + later_user + .metadata + .insert("user_context_projection".to_string(), serde_json::json!(true)); + + let mut results = vec![expected, pivot, later_user]; + apply_user_context_adjacency_preference(&mut results, None, Some("Any tips?")); + + assert!(results[0].score > results[2].score); + assert!(results[0] + .metadata + .contains_key("user_context_adjacency_boost")); + assert!(!results[2] + .metadata + .contains_key("user_context_adjacency_boost")); + } + + #[cfg(any())] + #[test] + fn user_context_adjacency_considers_bounded_multiple_pivots() { + fn with_source( + mut result: crate::engine::RecallResult, + role: &str, + session: &str, + projected: bool, + ) -> crate::engine::RecallResult { + result + .metadata + .insert("source_role".to_string(), serde_json::json!(role)); + result.metadata.insert( + "source_session_id".to_string(), + serde_json::json!(session), + ); + if projected { + result + .metadata + .insert("user_context_projection".to_string(), serde_json::json!(true)); + } + result + } + + let mut first_user = recall_result(0.2, 1); + first_user.memory_id = 40; + first_user.score = 90.0; + first_user.created_at = 1.0; + + let mut first_pivot = recall_result(0.6, 2); + first_pivot.memory_id = 41; + first_pivot.score = 900.0; + first_pivot.created_at = 2.0; + + let mut second_user = recall_result(0.2, 1); + second_user.memory_id = 42; + second_user.score = 80.0; + second_user.created_at = 3.0; + + let mut second_pivot = recall_result(0.6, 2); + second_pivot.memory_id = 43; + second_pivot.score = 800.0; + second_pivot.created_at = 4.0; + + let mut expected = recall_result(0.2, 1); + expected.memory_id = 44; + expected.score = 70.0; + expected.created_at = 5.0; + + let mut expected_pivot = recall_result(0.6, 2); + expected_pivot.memory_id = 45; + expected_pivot.score = 500.0; + expected_pivot.created_at = 6.0; + + let mut results = vec![ + with_source(first_user, "user", "conversation-7", true), + with_source(first_pivot, "assistant", "conversation-7", false), + with_source(second_user, "user", "conversation-7", true), + with_source(second_pivot, "assistant", "conversation-7", false), + with_source(expected, "user", "conversation-7", true), + with_source(expected_pivot, "assistant", "conversation-7", false), + ]; + + apply_user_context_adjacency_preference( + &mut results, + None, + Some("Any helpful tips?"), + ); + + assert!(results[4] + .metadata + .contains_key("user_context_adjacency_boost")); + assert!(results[4].score > 70.0); + } + + #[test] + fn source_session_cue_is_derived_from_structured_metadata() { + let mut metadata = HashMap::new(); + metadata.insert( + "source_session_id".to_string(), + serde_json::json!("Answer ShareGPT hA7AkP3 0"), + ); + + assert_eq!( + source_session_cue_from_metadata(&metadata).as_deref(), + Some("source_session:answer_sharegpt_ha7akp3_0") + ); + } + + #[test] + fn list_answer_detection_covers_ordinals_without_topic_words() { + assert!(query_wants_list_answer(Some( + "What was the 7th item you listed?" + ))); + assert!(query_wants_list_answer(Some( + "Remind me what was in the list you provided." + ))); + assert!(!query_wants_list_answer(Some( + "What did I purchase yesterday?" + ))); + } + + #[test] + fn source_role_preference_demotes_structured_role_mismatches() { + let mut user_result = recall_result(1.0, 3); + user_result.score = 100.0; + user_result + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + + let mut assistant_result = recall_result(1.0, 3); + assistant_result.memory_id = 50; + assistant_result.score = 80.0; + assistant_result + .metadata + .insert("source_role".to_string(), serde_json::json!("assistant")); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec!["source_assistant".to_string()], + ..Default::default() + }; + let mut results = vec![user_result, assistant_result]; + apply_source_role_preference(&mut results, Some(&intent)); + + assert!(results[0].score < results[1].score); + assert_eq!(results[1].score, 80.0); + } + + #[cfg(any())] + #[test] + fn source_answer_adjacency_prefers_immediate_assistant_reply() { + let mut pivot = recall_result(1.0, 5); + pivot.score = 1000.0; + pivot.created_at = 1.0; + pivot.metadata + .insert("source_role".to_string(), serde_json::json!("user")); + pivot.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-1"), + ); + + let mut immediate_answer = recall_result(1.0, 2); + immediate_answer.memory_id = 60; + immediate_answer.score = 300.0; + immediate_answer.created_at = 2.0; + immediate_answer + .metadata + .insert("source_role".to_string(), serde_json::json!("assistant")); + immediate_answer.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-1"), + ); + + let mut later_answer = recall_result(1.0, 8); + later_answer.memory_id = 61; + later_answer.score = 1000.0; + later_answer.created_at = 6.0; + later_answer + .metadata + .insert("source_role".to_string(), serde_json::json!("assistant")); + later_answer.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-1"), + ); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec!["source_answer".to_string(), "source_assistant".to_string()], + ..Default::default() + }; + let mut results = vec![pivot, immediate_answer, later_answer]; + apply_source_answer_adjacency_preference(&mut results, Some(&intent)); + + assert!(results[1].score > results[2].score); + assert!(results[1] + .metadata + .contains_key("source_answer_adjacency_boost")); + } + + #[cfg(any())] + #[test] + fn decision_adjacency_prefers_selection_after_proposal() { + let mut proposal = recall_result(1.0, 6); + proposal.score = 3000.0; + proposal.created_at = 1.0; + proposal.content = + "assistant: Here are some potential names: Radik, Nucleus, Fissionator.".to_string(); + proposal + .metadata + .insert("source_role".to_string(), serde_json::json!("assistant")); + proposal.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-2"), + ); + + let mut selected = recall_result(1.0, 1); + selected.memory_id = 70; + selected.score = 300.0; + selected.created_at = 2.0; + selected.content = "user: Fissionator is a really cool one.".to_string(); + selected + .metadata + .insert("source_role".to_string(), serde_json::json!("user")); + selected.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-2"), + ); + + let mut later = recall_result(1.0, 4); + later.memory_id = 71; + later.score = 900.0; + later.created_at = 5.0; + later.content = "assistant: Fissionator could have radioactive attacks.".to_string(); + later + .metadata + .insert("source_role".to_string(), serde_json::json!("assistant")); + later.metadata.insert( + "source_session_id".to_string(), + serde_json::json!("conversation-2"), + ); + + let intent = crate::facets::StructuralQueryPlan { + labels: vec![ + "decision_selection".to_string(), + "naming_decision".to_string(), + ], + ..Default::default() + }; + let mut results = vec![proposal, selected, later]; + apply_decision_adjacency_preference(&mut results, Some(&intent)); + + assert!(results[1].score > results[0].score); + assert!(results[1].score > results[2].score); + assert!(results[1] + .metadata + .contains_key("decision_adjacency_boost")); + } + + #[test] + fn project_headers_and_source_metadata_are_normalized_safely() { + let mut headers = axum::http::HeaderMap::new(); + assert!(extract_project_id(&headers).is_err()); + headers.insert("X-Project-ID", "valid_project".parse().unwrap()); + assert_eq!(extract_project_id(&headers).unwrap(), "valid_project"); + assert_eq!(extract_project_id_optional(&headers).as_deref(), Some("valid_project")); + headers.insert("X-Project-ID", "bad/project".parse().unwrap()); + assert!(extract_project_id(&headers).is_err()); + assert!(extract_project_id_optional(&headers).is_none()); + + assert_eq!(normalize_source_value(" Assistant Role! "), Some("assistant_role".to_string())); + assert!(normalize_source_value("-").is_none()); + let mut metadata = HashMap::new(); + metadata.insert("role".to_string(), serde_json::json!("assistant")); + metadata.insert("thread_id".to_string(), serde_json::json!("thread-42")); + assert_eq!(metadata_string(&metadata, &["missing", "role"]), Some("assistant")); + assert_eq!(source_role_from_metadata(&metadata).as_deref(), Some("assistant")); + assert_eq!(source_session_cue_from_metadata(&metadata).as_deref(), Some("source_session:thread_42")); + } + + #[test] + fn path_and_segment_defaults_are_safe_and_deterministic() { + assert_eq!(normalize_included_paths(Some(vec![ + "src\\lib".to_string(), + "./README.md".to_string(), + "src/lib".to_string(), + ])).unwrap(), vec!["README.md", "src/lib"]); + assert!(normalize_included_paths(Some(vec!["../outside".to_string()])).is_err()); + assert_eq!(normalize_included_paths(Some(vec![".".to_string()])).unwrap(), Vec::<String>::new()); + assert_eq!(normalize_ignored_extensions(Some(vec![ + ".RS".to_string(), + "rs".to_string(), + " ".to_string(), + ])), vec!["rs"]); + assert_eq!(default_depth(), 1); + assert_eq!(default_project_export_limit(), 1000); + assert_eq!(default_parent_fusion_limit(), 80); + assert_eq!(default_parent_fusion_min_chunks(), 2); + assert_eq!(default_ordered_reconstruction_limit(), 80); + assert_eq!(default_ordered_session_scan_limit(), 4096); + assert_eq!(default_ordered_max_sessions(), 3); + assert_eq!(default_evidence_coverage_limit(), 100); + assert_eq!(default_evidence_coverage_session_scan_limit(), 4096); + assert_eq!(default_evidence_coverage_max_sessions(), 3); + assert_eq!(default_cuebridge_gap_limit(), 6); + assert_eq!(default_filename(), "content.txt"); + } + + #[test] + fn source_event_time_rejects_invalid_numeric_values() { + let mut metadata = HashMap::new(); + metadata.insert("source_timestamp".to_string(), serde_json::json!(12.5)); + assert_eq!(source_event_time(None, Some(&metadata)), Some(12.5)); + metadata.insert("source_timestamp".to_string(), serde_json::json!(-1.0)); + assert_eq!(source_event_time(None, Some(&metadata)), None); + metadata.insert("source_timestamp".to_string(), serde_json::json!("not-a-time")); + assert_eq!(source_event_time(None, Some(&metadata)), None); + } + + fn test_router() -> axum::Router { + test_router_with_read_only(false) + } + + fn test_router_with_read_only(read_only: bool) -> axum::Router { + let snapshots = std::env::temp_dir().join(format!("cuemap-api-test-{}", uuid::Uuid::new_v4())); + test_router_with_snapshots(snapshots, read_only) + } + + fn test_router_with_snapshots( + snapshots: std::path::PathBuf, + read_only: bool, + ) -> axum::Router { + let mt_engine = Arc::new(MultiTenantEngine::with_snapshots_dir( + &snapshots, + TuningConfig::default(), + )); + let metrics = Arc::new(MetricsCollector::new()); + let provider: Arc<dyn crate::jobs::ProjectProvider> = mt_engine.clone(); + let job_queue = Arc::new(JobQueue::new(provider, Some(metrics.clone()), true)); + let agent_manager = Arc::new(crate::agent::manager::AgentManager::new( + job_queue.clone(), + mt_engine.clone(), + )); + routes( + mt_engine, + job_queue, + metrics, + AuthConfig::from_config(&crate::config::SecurityConfig::default()), + read_only, + snapshots.to_string_lossy().to_string(), + None, + None, + agent_manager, + ) + } + + async fn test_router_with_local_backup() -> axum::Router { + let root = std::env::temp_dir().join(format!("cuemap-api-backup-{}", uuid::Uuid::new_v4())); + let data_dir = root.join("data"); + let snapshots = data_dir.join("snapshots"); + let cloud_dir = root.join("cloud"); + std::fs::create_dir_all(&snapshots).unwrap(); + + let mt_engine = Arc::new(MultiTenantEngine::with_snapshots_dir( + &snapshots, + TuningConfig::default(), + )); + let metrics = Arc::new(MetricsCollector::new()); + let provider: Arc<dyn crate::jobs::ProjectProvider> = mt_engine.clone(); + let job_queue = Arc::new(JobQueue::new(provider, Some(metrics.clone()), true)); + let agent_manager = Arc::new(crate::agent::manager::AgentManager::new( + job_queue.clone(), + mt_engine.clone(), + )); + let config = CloudBackupConfig::from_args( + Some("local"), + Some(cloud_dir.to_string_lossy().as_ref()), + None, + None, + "cuemap/", + false, + ) + .unwrap(); + let backup = Arc::new(CloudBackupManager::new(config).await.unwrap()); + routes( + mt_engine, + job_queue, + metrics, + AuthConfig::from_config(&crate::config::SecurityConfig::default()), + false, + data_dir.to_string_lossy().to_string(), + Some(backup), + None, + agent_manager, + ) + } + + async fn json_body(response: axum::response::Response) -> serde_json::Value { + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + serde_json::from_slice(&bytes).unwrap() + } + + async fn local_http_url(body: &'static str) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + if let Ok((mut stream, _)) = listener.accept().await { + let mut request = [0_u8; 4096]; + let _ = stream.read(&mut request).await; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = stream.write_all(response.as_bytes()).await; + } + }); + format!("http://{address}/fixture") + } + + #[tokio::test] + async fn routes_cover_root_stats_and_memory_lifecycle() { + let router = test_router(); + let root = router + .clone() + .oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(root.status(), StatusCode::OK); + assert!(json_body(root).await["capabilities"].as_array().unwrap().len() >= 4); + + let missing_project = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .body(Body::from(r#"{"content":"hello"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_project.status(), StatusCode::BAD_REQUEST); + + let stored = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "api-test") + .body(Body::from( + r#"{"content":"hello world","cues":["greeting"],"metadata":{"source":"test"}}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(stored.status(), StatusCode::OK); + let stored_json = json_body(stored).await; + let id = stored_json["id"].as_u64().unwrap(); + + let fetched = router + .clone() + .oneshot( + Request::builder() + .uri(format!("/memories/{id}")) + .header("X-Project-ID", "api-test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(fetched.status(), StatusCode::OK); + assert_eq!(json_body(fetched).await["id"], id); + + let reinforced_with_cues = router + .clone() + .oneshot( + Request::builder() + .method("PATCH") + .uri(format!("/memories/{id}/reinforce")) + .header("content-type", "application/json") + .header("X-Project-ID", "api-test") + .body(Body::from(r#"{"cues":["greeting"]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(reinforced_with_cues.status(), StatusCode::OK); + + let recalled = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall") + .header("content-type", "application/json") + .header("X-Project-ID", "api-test") + .body(Body::from(r#"{"cues":["greeting"],"limit":5}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(recalled.status(), StatusCode::OK); + assert!(!json_body(recalled).await["results"].as_array().unwrap().is_empty()); + + let deleted = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri(format!("/memories/{id}")) + .header("X-Project-ID", "api-test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(deleted.status(), StatusCode::OK); + + let stats = router + .oneshot( + Request::builder() + .uri("/stats") + .header("X-Project-ID", "api-test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(stats.status(), StatusCode::OK); + } + + #[tokio::test] + async fn routes_cover_projects_aliases_lexicon_and_directory_preview() { + let router = test_router(); + let project = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"api-project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(project.status(), StatusCode::CREATED); + + let invalid_project = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"x"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_project.status(), StatusCode::BAD_REQUEST); + + let listed = router + .clone() + .oneshot(Request::builder().uri("/projects").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(listed.status(), StatusCode::OK); + assert!(json_body(listed).await.as_array().unwrap().iter().any(|p| p["project_id"] == "api-project")); + + let alias = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/aliases") + .header("content-type", "application/json") + .header("X-Project-ID", "api-project") + .body(Body::from(r#"{"from":"rust","to":"rust_language"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(alias.status(), StatusCode::OK); + + let aliases = router + .clone() + .oneshot( + Request::builder() + .uri("/aliases?cue=rust") + .header("X-Project-ID", "api-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(aliases.status(), StatusCode::OK); + + let merged = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/aliases/merge") + .header("content-type", "application/json") + .header("X-Project-ID", "api-project") + .body(Body::from(r#"{"cues":["rust","rs"],"to":"rust_language"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(merged.status(), StatusCode::OK); + + let wired = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/lexicon/wire") + .header("content-type", "application/json") + .header("X-Project-ID", "api-project") + .body(Body::from(r#"{"token":"rs","canonical":"rust"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(wired.status(), StatusCode::OK); + let lexicon_id = json_body(wired).await["memory_id"].as_u64().unwrap(); + + for uri in ["/lexicon/inspect/rust", "/lexicon/graph"] { + let response = router + .clone() + .oneshot( + Request::builder() + .uri(uri) + .header("X-Project-ID", "api-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + let deleted_lexicon = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri(format!("/lexicon/entry/{lexicon_id}")) + .header("X-Project-ID", "api-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(deleted_lexicon.status(), StatusCode::OK); + + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("notes.md"), "hello\nworld").unwrap(); + let preview = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/directory/preview") + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({"watch_dir": dir.path(), "included_paths":["notes.md"]}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(preview.status(), StatusCode::OK); + + let watch = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects/api-project/watch-dir") + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "watch_dir": dir.path(), + "included_paths": ["notes.md"], + "ignored_extensions": [".log"] + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(watch.status(), StatusCode::OK); + let watch_info = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/api-project/watch-dir") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(watch_info.status(), StatusCode::OK); + assert_eq!(json_body(watch_info).await["initialized"], true); + + let deleted_project = router + .oneshot( + Request::builder() + .method("DELETE") + .uri("/projects/api-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(deleted_project.status(), StatusCode::OK); + } + + #[tokio::test] + async fn routes_cover_project_guards_reload_and_directory_validation() { + let router = test_router(); + + let missing_classify_project = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/intent/classify") + .header("content-type", "application/json") + .body(Body::from(r#"{"text":"classify this"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_classify_project.status(), StatusCode::BAD_REQUEST); + + let created = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects/reload-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(created.status(), StatusCode::METHOD_NOT_ALLOWED); + + let project = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"reload-project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(project.status(), StatusCode::CREATED); + + let reloaded = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects/reload-project/artifacts") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(reloaded.status(), StatusCode::OK); + + for uri in [ + "/projects/bad!id/artifacts", + "/projects/bad!id/export", + ] { + let response = router + .clone() + .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{uri}"); + } + + let uninitialized_watch = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/bad!id/watch-dir") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(uninitialized_watch.status(), StatusCode::OK); + assert_eq!(json_body(uninitialized_watch).await["initialized"], false); + + let missing_watch = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/not-created/watch-dir") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_watch.status(), StatusCode::OK); + assert_eq!(json_body(missing_watch).await["initialized"], false); + + let missing_delete = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/projects/not-created") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_delete.status(), StatusCode::NOT_FOUND); + + let dir = tempfile::tempdir().unwrap(); + for uri in ["/ingest/directory/preview", "/projects/reload-project/watch-dir"] { + let response = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(uri) + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "watch_dir": dir.path(), + "included_paths": ["../outside"] + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{uri}"); + } + } + + #[tokio::test] + async fn routes_cover_batch_ingestion_debug_grounding_export_and_metrics() { + let router = test_router(); + let batch = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories/batch") + .header("content-type", "application/json") + .header("X-Project-ID", "batch-project") + .body(Body::from( + r#"{"memories":[{"content":"first item","cues":["batch"]},{"content":"second item","cues":["batch"]}],"minimal_response":true,"trace_timing":true}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(batch.status(), StatusCode::OK); + let batch_json = json_body(batch).await; + let ids = batch_json["ids"].as_array().unwrap(); + assert_eq!(ids.len(), 2); + + let reinforce = router + .clone() + .oneshot( + Request::builder() + .method("PATCH") + .uri(format!("/memories/{}/reinforce", ids[0].as_u64().unwrap())) + .header("content-type", "application/json") + .header("X-Project-ID", "batch-project") + .body(Body::from(r#"{"cues":[]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(reinforce.status(), StatusCode::OK); + + let jobs = router + .clone() + .oneshot( + Request::builder() + .uri("/jobs/status") + .header("X-Project-ID", "batch-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(jobs.status(), StatusCode::OK); + assert!(json_body(jobs).await.get("intent_ready").is_some()); + + let debug = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/debug/analyze-text") + .header("content-type", "application/json") + .header("X-Project-ID", "batch-project") + .body(Body::from(r#"{"text":"First sentence. Second sentence.","filename":"notes.md","segmenter":"sentence_window","segment_window_size":2}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(debug.status(), StatusCode::OK); + assert!(json_body(debug).await["chunks"].as_array().unwrap().len() >= 1); + + let ingested = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/content") + .header("content-type", "application/json") + .header("X-Project-ID", "batch-project") + .body(Body::from(r#"{"content":"A longer note with enough content to create a chunk.","filename":"notes.md","source_key":"notes.md"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(ingested.status(), StatusCode::OK); + + let grounded = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall/grounded") + .header("content-type", "application/json") + .header("X-Project-ID", "batch-project") + .body(Body::from(r#"{"query_text":"batch","token_budget":64,"limit":5}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(grounded.status(), StatusCode::OK); + assert_eq!(json_body(grounded).await["signature_alg"], "none"); + + let export = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/batch-project/export?limit=1&include_metadata=false") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(export.status(), StatusCode::OK); + let export_json = json_body(export).await; + assert_eq!(export_json["count"], 1); + assert_eq!(export_json["include_metadata"], false); + + let metrics = router + .clone() + .oneshot(Request::builder().uri("/metrics").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(metrics.status(), StatusCode::OK); + let metrics_body = to_bytes(metrics.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(metrics_body.to_vec()).unwrap().contains("cuemap_total_memories")); + + let intent = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/intent/classify") + .header("content-type", "application/json") + .header("X-Project-ID", "batch-project") + .body(Body::from(r#"{"text":"What happened yesterday?","target":"query"}"#)) + .unwrap(), + ) + .await + .unwrap(); + #[cfg(feature = "semantic-encoder")] + assert_eq!(intent.status(), StatusCode::OK); + #[cfg(not(feature = "semantic-encoder"))] + assert_eq!(intent.status(), StatusCode::SERVICE_UNAVAILABLE); + + let global_stats = router + .clone() + .oneshot(Request::builder().uri("/stats").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(global_stats.status(), StatusCode::OK); + + let artifacts = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/batch-project/artifacts") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(artifacts.status(), StatusCode::OK); + + let backup_list = router + .clone() + .oneshot(Request::builder().uri("/backup/list").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(backup_list.status(), StatusCode::SERVICE_UNAVAILABLE); + + let boundary = "coverage-boundary"; + let multipart = format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"filename\"\r\n\r\nupload-test.md\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"upload-test.md\"\r\nContent-Type: text/plain\r\n\r\nUploaded content for multipart ingestion.\r\n--{boundary}--\r\n" + ); + let uploaded = router + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/file") + .header("content-type", format!("multipart/form-data; boundary={boundary}")) + .header("X-Project-ID", "batch-project") + .body(Body::from(multipart)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(uploaded.status(), StatusCode::OK); + } + + #[tokio::test] + async fn routes_cover_read_only_api_paths() { + let router = test_router_with_read_only(true); + let cases = [ + ("POST", "/memories", r#"{"content":"blocked"}"#), + ("POST", "/ingest/content", r#"{"content":"blocked"}"#), + ("POST", "/ingest/url", r#"{"url":"not-a-url"}"#), + ("POST", "/aliases", r#"{"from":"a","to":"b"}"#), + ("POST", "/lexicon/wire", r#"{"token":"a","canonical":"b"}"#), + ( + "POST", + "/projects/readonly/watch-dir", + r#"{"watch_dir":"/does/not/matter"}"#, + ), + ]; + for (method, uri, body) in cases { + let response = router + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .header("X-Project-ID", "read-only") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN, "{method} {uri}"); + } + + let delete = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/memories/1") + .header("X-Project-ID", "read-only") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(delete.status(), StatusCode::FORBIDDEN); + + let recall_web = router + .oneshot( + Request::builder() + .method("POST") + .uri("/recall/web") + .header("content-type", "application/json") + .header("X-Project-ID", "read-only") + .body(Body::from(r#"{"url":"not-a-url","query":"blocked","persist":true}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(recall_web.status(), StatusCode::FORBIDDEN); + } + + #[tokio::test] + async fn routes_cover_memory_project_and_ingestion_errors() { + let router = test_router(); + + let invalid_event_time = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"content":"bad timestamp","event_time":-1}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_event_time.status(), StatusCode::BAD_REQUEST); + + let missing_header = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .body(Body::from(r#"{"content":"missing project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_header.status(), StatusCode::BAD_REQUEST); + + let stored = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from( + r#"{"content":"source keyed","source_key":"source-1","minimal_response":true,"trace_timing":true}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(stored.status(), StatusCode::OK); + let stored_body = json_body(stored).await; + assert!(stored_body.get("timing").is_some()); + assert!(stored_body.get("cues").is_none()); + + for (uri, method) in [ + ("/memories/999999", "GET"), + ("/memories/999999", "DELETE"), + ] { + let response = router + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("X-Project-ID", "api-errors") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{method} {uri}"); + } + + let reinforce_missing = router + .clone() + .oneshot( + Request::builder() + .method("PATCH") + .uri("/memories/999999/reinforce") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"cues":[]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(reinforce_missing.status(), StatusCode::NOT_FOUND); + + let batch_empty = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories/batch") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"memories":[],"trace_timing":true}"#)) + .unwrap(), + ) + .await + .unwrap(); + let batch_empty_body = json_body(batch_empty).await; + assert_eq!(batch_empty_body["count"], 0); + assert_eq!(batch_empty_body["timings"].as_array().unwrap().len(), 0); + + let batch_failure = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories/batch") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from( + r#"{"memories":[{"content":"ok"},{"content":"bad","event_time":-1}]}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(batch_failure.status(), StatusCode::BAD_REQUEST); + assert_eq!(json_body(batch_failure).await["failed_index"], 1); + + let aliases_missing_cue = router + .clone() + .oneshot( + Request::builder() + .uri("/aliases") + .header("X-Project-ID", "api-errors") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(aliases_missing_cue.status(), StatusCode::BAD_REQUEST); + + let lexicon_missing = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/lexicon/entry/999999") + .header("X-Project-ID", "api-errors") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(lexicon_missing.status(), StatusCode::NOT_FOUND); + + let embedding_mismatch = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/content") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"content":"one sentence.","embeddings":[]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(embedding_mismatch.status(), StatusCode::BAD_REQUEST); + + let empty_content = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/content") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"content":""}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(empty_content.status(), StatusCode::BAD_REQUEST); + + let boundary = "api-errors-boundary"; + let missing_file = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/file") + .header("content-type", format!("multipart/form-data; boundary={boundary}")) + .header("X-Project-ID", "api-errors") + .body(Body::from(format!( + "--{boundary}\r\nContent-Disposition: form-data; name=\"note\"\r\n\r\nmissing file\r\n--{boundary}--\r\n" + ))) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_file.status(), StatusCode::BAD_REQUEST); + + let invalid_url = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/url") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"url":"not-a-url"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_url.status(), StatusCode::BAD_REQUEST); + + let invalid_web_url = router + .oneshot( + Request::builder() + .method("POST") + .uri("/recall/web") + .header("content-type", "application/json") + .header("X-Project-ID", "api-errors") + .body(Body::from(r#"{"url":"not-a-url","query":"test"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_web_url.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn routes_cover_remaining_guards_and_unconfigured_backup_paths() { + let router = test_router(); + + for (method, uri, body) in [ + ("POST", "/backup/upload", r#"{"project_id":"guard-project"}"#), + ("POST", "/backup/download", r#"{"project_id":"guard-project"}"#), + ] { + let response = router + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE, "{method} {uri}"); + } + let backup_delete = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/backup/guard-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(backup_delete.status(), StatusCode::SERVICE_UNAVAILABLE); + + let classify_invalid = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/intent/classify") + .header("content-type", "application/json") + .header("X-Project-ID", "bad!project") + .body(Body::from(r#"{"text":"classify this"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(classify_invalid.status(), StatusCode::BAD_REQUEST); + let classify_ok = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/intent/classify") + .header("content-type", "application/json") + .header("X-Project-ID", "guard-project") + .body(Body::from(r#"{"text":"What happened yesterday?","target":"query"}"#)) + .unwrap(), + ) + .await + .unwrap(); + #[cfg(feature = "semantic-encoder")] + assert_eq!(classify_ok.status(), StatusCode::OK); + #[cfg(not(feature = "semantic-encoder"))] + assert_eq!(classify_ok.status(), StatusCode::SERVICE_UNAVAILABLE); + + for (method, uri) in [ + ("GET", "/memories/1"), + ("DELETE", "/memories/1"), + ("PATCH", "/memories/1/reinforce"), + ] { + let response = router + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(if method == "PATCH" { r#"{"cues":[]}"# } else { "" })) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{method} {uri}"); + } + let global_stats = router + .clone() + .oneshot(Request::builder().uri("/stats").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(global_stats.status(), StatusCode::OK); + + let global_jobs = router + .clone() + .oneshot(Request::builder().uri("/jobs/status").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(global_jobs.status(), StatusCode::OK); + let global_grounded = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall/grounded") + .header("content-type", "application/json") + .body(Body::from(r#"{"query_text":"guard project","projects":[]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(global_grounded.status(), StatusCode::OK); + + let read_only = test_router_with_read_only(true); + let read_only_create = read_only + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"blocked-project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(read_only_create.status(), StatusCode::FORBIDDEN); + let read_only_delete = read_only + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/memories/1") + .header("X-Project-ID", "blocked-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(read_only_delete.status(), StatusCode::FORBIDDEN); + for (uri, body) in [ + ("/aliases/merge", r#"{"cues":["a"],"to":"b"}"#), + ("/lexicon/entry/1", ""), + ] { + let response = read_only + .clone() + .oneshot( + Request::builder() + .method(if uri == "/lexicon/entry/1" { "DELETE" } else { "POST" }) + .uri(uri) + .header("content-type", "application/json") + .header("X-Project-ID", "blocked-project") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN, "{uri}"); + } + + for (uri, method, body) in [ + ("/projects/not-created/artifacts", "GET", ""), + ("/projects/not-created/artifacts", "POST", ""), + ("/projects/not-created/export", "GET", ""), + ("/projects/not-created/watch-dir", "GET", ""), + ] { + let response = router + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert!( + matches!(response.status(), StatusCode::OK | StatusCode::NOT_FOUND | StatusCode::SERVICE_UNAVAILABLE), + "{method} {uri} returned {}", + response.status() + ); + } + + let missing_preview = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/directory/preview") + .header("content-type", "application/json") + .body(Body::from(r#"{"watch_dir":"/definitely/missing/cuemap-dir"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_preview.status(), StatusCode::BAD_REQUEST); + + let invalid_reload = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/projects/bad!id/artifacts") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_reload.status(), StatusCode::BAD_REQUEST); + + let logical = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/content") + .header("content-type", "application/json") + .header("X-Project-ID", "guard-project") + .body(Body::from( + r#"{"content":"First block.\n\nSecond block.","filename":"notes.md","segmenter":"logical_block","segment_window_size":2,"segment_overlap":1,"segment_min_chunk_chars":1,"segment_max_chunk_chars":100}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(logical.status(), StatusCode::OK); + + let debug_logical = router + .oneshot( + Request::builder() + .method("POST") + .uri("/debug/analyze-text") + .header("content-type", "application/json") + .header("X-Project-ID", "guard-project") + .body(Body::from( + r#"{"text":"First block.\n\nSecond block.","segmenter":"logical_block","segment_window_size":2,"segment_overlap":1,"segment_min_chunk_chars":1,"segment_max_chunk_chars":100}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(debug_logical.status(), StatusCode::OK); + } + + #[tokio::test] + async fn routes_cover_cuebridge_gap_expansion_in_recall() { + let snapshots = std::env::temp_dir().join(format!("cuemap-api-gap-{}", uuid::Uuid::new_v4())); + let artifact_dir = snapshots.join("artifacts").join("api-gap"); + std::fs::create_dir_all(&artifact_dir).unwrap(); + std::fs::write( + artifact_dir.join("gap.json"), + r#"{ + "artifact_type":"gap_pack", + "name":"api-gap-pack", + "entries":[{ + "id":"deployment-release", + "query_signature":{"required_any":["deployment"]}, + "expansions":[{"cue":"release","weight":1.0}], + "confidence":0.9, + "max_fanout":2 + }] + }"#, + ) + .unwrap(); + let router = test_router_with_snapshots(snapshots, false); + + let stored = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "api-gap") + .body(Body::from(r#"{"content":"Deployment release is ready.","cues":["deployment","release"]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(stored.status(), StatusCode::OK); + + let recalled = router + .oneshot( + Request::builder() + .method("POST") + .uri("/recall") + .header("content-type", "application/json") + .header("X-Project-ID", "api-gap") + .body(Body::from(r#"{"query_text":"deployment","cues":["deployment"],"disable_cuebridge_artifacts":false,"explain":true}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(recalled.status(), StatusCode::OK); + let body = json_body(recalled).await; + assert!(body["results"].is_array()); + } + + #[tokio::test] + async fn routes_cover_recall_modes_and_project_exports() { + let router = test_router(); + let stored = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "api-recall") + .body(Body::from( + r#"{"content":"The deployment decision moved the service to a regional cluster.","cues":["deployment","decision","service","regional","cluster"],"metadata":{"source_session_id":"session-api","source_turn_index":1,"source_role":"assistant","type":"answer"}}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(stored.status(), StatusCode::OK); + + let embedding = vec![0.01_f32; 384]; + let semantic_stored = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "api-recall") + .body(Body::from( + serde_json::json!({ + "content": "Semantic deployment vector", + "cues": ["semantic", "deployment"], + "embedding": embedding, + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(semantic_stored.status(), StatusCode::OK); + + let semantic_recall = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall") + .header("content-type", "application/json") + .header("X-Project-ID", "api-recall") + .body(Body::from( + serde_json::json!({ + "semantic_mode": "semantic", + "query_embedding": vec![0.01_f32; 384], + "limit": 3, + "explain": true, + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(semantic_recall.status(), StatusCode::OK); + assert!(json_body(semantic_recall).await["results"].is_array()); + + let recall = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall") + .header("content-type", "application/json") + .header("X-Project-ID", "api-recall") + .body(Body::from( + r#"{"query_text":"summarize the deployment decision","cues":["deployment"],"semantic_mode":"lexical","limit":5,"depth":2,"min_intersection":1,"explain":true,"trace_timing":true,"auto_reinforce":true,"ordered_reconstruction":"force","evidence_coverage":"force","parent_fusion":"force","disable_cuebridge_artifacts":true}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(recall.status(), StatusCode::OK); + let recall_body = json_body(recall).await; + assert!(recall_body.get("explain").is_some()); + assert!(recall_body.get("timing").is_some()); + + let cross_project = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall") + .header("content-type", "application/json") + .body(Body::from( + r#"{"projects":["api-recall","api-recall-other"],"query_text":"deployment","cues":["deployment"],"semantic_mode":"lexical","limit":3,"explain":true,"auto_reinforce":true}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(cross_project.status(), StatusCode::OK); + let cross_body = json_body(cross_project).await; + assert_eq!(cross_body["results"].as_array().unwrap().len(), 2); + + let artifacts = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/api-recall/artifacts") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(artifacts.status(), StatusCode::OK); + + let export_without_fields = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/api-recall/export?limit=1&include_content=false&include_cues=false&include_metadata=false") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(export_without_fields.status(), StatusCode::OK); + let export_body = json_body(export_without_fields).await; + assert_eq!(export_body["include_content"], false); + assert_eq!(export_body["include_cues"], false); + assert_eq!(export_body["include_metadata"], false); + assert!(export_body["memories"][0].get("content").is_none()); + + let export_after_cursor = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/api-recall/export?cursor=999999&limit=1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(export_after_cursor.status(), StatusCode::OK); + assert_eq!(json_body(export_after_cursor).await["count"], 0); + + let get_watch_meta = router + .clone() + .oneshot( + Request::builder() + .uri("/projects/api-recall/watch-dir") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(get_watch_meta.status(), StatusCode::OK); + + let invalid_watch = router + .oneshot( + Request::builder() + .method("POST") + .uri("/projects/api-recall/watch-dir") + .header("content-type", "application/json") + .body(Body::from(r#"{"watch_dir":"/does/not/exist"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_watch.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn routes_cover_local_cloud_backup_lifecycle() { + let router = test_router_with_local_backup().await; + let stored = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/memories") + .header("content-type", "application/json") + .header("X-Project-ID", "backup-project") + .body(Body::from(r#"{"content":"backup content","cues":["backup"]}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(stored.status(), StatusCode::OK); + + let invalid_upload = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/backup/upload") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"x"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_upload.status(), StatusCode::BAD_REQUEST); + + let uploaded = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/backup/upload") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"backup-project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(uploaded.status(), StatusCode::OK); + assert_eq!(json_body(uploaded).await["success"], true); + + let listed = router + .clone() + .oneshot(Request::builder().uri("/backup/list").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(listed.status(), StatusCode::OK); + assert!(json_body(listed).await["count"].as_u64().unwrap() >= 1); + + let downloaded = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/backup/download") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"backup-project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(downloaded.status(), StatusCode::OK); + assert_eq!(json_body(downloaded).await["success"], true); + + let deleted = router + .clone() + .oneshot( + Request::builder() + .method("DELETE") + .uri("/backup/backup-project") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(deleted.status(), StatusCode::OK); + + let missing_download = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/backup/download") + .header("content-type", "application/json") + .body(Body::from(r#"{"project_id":"backup-project"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(missing_download.status(), StatusCode::NOT_FOUND); + + let invalid_delete = router + .oneshot( + Request::builder() + .method("DELETE") + .uri("/backup/x") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(invalid_delete.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn routes_cover_local_url_ingestion_and_web_recall() { + let router = test_router(); + let url = local_http_url( + "<html><head><title>Deployment Notes

Deployment

The regional service uses a rolling release.

", + ) + .await; + + let ingested = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/url") + .header("content-type", "application/json") + .header("X-Project-ID", "api-url") + .body(Body::from(serde_json::json!({"url": url, "depth": 0}).to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(ingested.status(), StatusCode::OK); + let ingested_body = json_body(ingested).await; + assert_eq!(ingested_body["status"], "ingested"); + assert!(ingested_body["chunks"].as_u64().unwrap() >= 1); + + let recall_url = local_http_url( + "

Deployment

The regional service uses a rolling release.

", + ) + .await; + let web_recall = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall/web") + .header("content-type", "application/json") + .header("X-Project-ID", "api-url") + .body(Body::from( + serde_json::json!({"url": recall_url, "query": "deployment regional"}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(web_recall.status(), StatusCode::OK); + let web_body = json_body(web_recall).await; + assert_eq!(web_body["urls"].as_array().unwrap().len(), 1); + assert!(web_body["results"].is_array()); + + let persisted_url = local_http_url( + "

Persisted Deployment

The rolling release is persisted asynchronously.

", + ) + .await; + let persisted_web_recall = router + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/recall/web") + .header("content-type", "application/json") + .header("X-Project-ID", "api-url") + .body(Body::from( + serde_json::json!({"url": persisted_url, "query": "deployment", "persist": true}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(persisted_web_recall.status(), StatusCode::OK); + + let crawl_url = local_http_url( + "

Crawl Deployment

A single crawlable deployment page.

", + ) + .await; + let crawled = router + .oneshot( + Request::builder() + .method("POST") + .uri("/ingest/url") + .header("content-type", "application/json") + .header("X-Project-ID", "api-url") + .body(Body::from( + serde_json::json!({"url": crawl_url, "depth": 1, "same_domain_only": true}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(crawled.status(), StatusCode::OK); + } diff --git a/tests/unit/cuebridge.rs b/tests/unit/cuebridge.rs new file mode 100644 index 0000000..a674666 --- /dev/null +++ b/tests/unit/cuebridge.rs @@ -0,0 +1,19 @@ + use super::*; + + #[test] + fn gap_pack_rejects_bare_entry_without_gates() { + let entry = RuntimeGapEntry { + artifact: "test".to_string(), + artifact_hash: "hash".to_string(), + id: "gap".to_string(), + signature: RuntimeQuerySignature::default(), + expansions: vec![RawExpansion { + cue: "target".to_string(), + weight: 1.0, + }], + negative_gates: Vec::new(), + confidence: 1.0, + max_fanout: 1, + }; + assert!(!entry.matches(&HashSet::new(), &HashSet::new(), &HashSet::new())); + } diff --git a/tests/unit/facets.rs b/tests/unit/facets.rs new file mode 100644 index 0000000..6f1d238 --- /dev/null +++ b/tests/unit/facets.rs @@ -0,0 +1,65 @@ + use super::{compile_query_plan, extract_memory_facets_core}; + use serde_json::json; + use std::collections::HashMap; + + #[test] + fn structural_facets_keep_evidence_and_source_metadata() { + let mut metadata = HashMap::new(); + metadata.insert("source_role".to_string(), json!("user")); + metadata.insert("source_date".to_string(), json!("2023-04-21")); + let facets = extract_memory_facets_core( + "The meeting is next Friday at 7:30 PM and costs $20.", + Some(&metadata), + &[], + ); + for expected in [ + "source_role:user", + "source_time:dated", + "source_date:2023_04_21", + "source_week:2023_w16", + "has:number", + "has:money", + "has:time", + "time_of_day:evening", + "temporal:relative", + ] { + assert!(facets.iter().any(|facet| facet == expected), "missing {expected}: {facets:?}"); + } + } + + #[test] + fn semantic_language_does_not_create_ontology_facets() { + let facets = extract_memory_facets_core( + "I prefer tea, bought a new mug, and always want recommendations.", + None, + &[], + ); + assert!(!facets.iter().any(|facet| facet.starts_with("type:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("preference:"))); + assert!(!facets.iter().any(|facet| facet.starts_with("purchase:"))); + } + + #[test] + fn query_plan_only_emits_structural_or_retrieval_shape_signals() { + let intent = compile_query_plan("Summarize the events from yesterday", |cue| { + cue == "has:list" || cue.starts_with("temporal:") + }); + assert!(intent.labels.iter().any(|label| label == "multi_evidence_summary")); + assert!(intent.labels.iter().any(|label| label == "temporal_yesterday")); + assert!(!intent.labels.iter().any(|label| label.contains("preference"))); + } + + #[test] + fn may_modal_is_not_seen_as_a_temporal_month() { + assert!(!super::date_re().is_match("May I ask whether this may improve recall.")); + assert!(!super::has_temporal_month("May I ask whether this may improve recall.")); + } + + #[test] + fn quantity_regex_captures_short_units_and_ranges() { + let measurement = super::measurement_re().captures("5 ms").expect("measurement"); + assert_eq!(measurement.name("value").map(|value| value.as_str()), Some("5")); + assert_eq!(measurement.name("unit").map(|unit| unit.as_str()), Some("ms")); + assert_eq!(super::canonical_quantity_unit("ms"), Some("ms")); + assert!(super::between_range_re().is_match("between 4 and 6 kg")); + } diff --git a/tests/unit/metrics.rs b/tests/unit/metrics.rs new file mode 100644 index 0000000..cbb7c42 --- /dev/null +++ b/tests/unit/metrics.rs @@ -0,0 +1,48 @@ + use super::*; + + #[test] + fn test_ingestion_counter() { + let metrics = MetricsCollector::new(); + assert_eq!(metrics.ingestion_count.load(Ordering::Relaxed), 0); + + metrics.record_ingestion(); + metrics.record_ingestion(); + + assert_eq!(metrics.ingestion_count.load(Ordering::Relaxed), 2); + } + + #[test] + fn test_recall_counter_and_latency() { + let metrics = MetricsCollector::new(); + + metrics.record_recall(1.0); + metrics.record_recall(2.0); + metrics.record_recall(10.0); + + assert_eq!(metrics.recall_count.load(Ordering::Relaxed), 3); + + // With only 3 samples, P99 should be the max + let p99 = metrics.get_p99_latency(); + assert!((p99 - 10.0).abs() < 0.01); + } + + #[test] + fn test_avg_latency() { + let metrics = MetricsCollector::new(); + + metrics.record_recall(1.0); + metrics.record_recall(2.0); + metrics.record_recall(3.0); + + let avg = metrics.get_avg_latency(); + assert!((avg - 2.0).abs() < 0.01); + } + + #[test] + fn test_empty_latencies() { + let metrics = MetricsCollector::new(); + + assert_eq!(metrics.get_p99_latency(), 0.0); + assert_eq!(metrics.get_avg_latency(), 0.0); + assert_eq!(metrics.get_sample_count(), 0); + } diff --git a/tests/unit/nl.rs b/tests/unit/nl.rs new file mode 100644 index 0000000..53bfa09 --- /dev/null +++ b/tests/unit/nl.rs @@ -0,0 +1,13 @@ + use super::tokenize_to_cues; + + #[test] + fn temporal_connector_breaks_phrase_without_removing_token() { + let cues = tokenize_to_cues( + "Maya switched from coffee to mint tea after the April deploy.", + ); + + assert!(cues.contains(&"after".to_string())); + assert!(cues.contains(&"mint_tea".to_string())); + assert!(cues.contains(&"april_deploy".to_string())); + assert!(!cues.contains(&"mint_tea_after".to_string())); + } diff --git a/tests/unit/persistence.rs b/tests/unit/persistence.rs new file mode 100644 index 0000000..c38b5de --- /dev/null +++ b/tests/unit/persistence.rs @@ -0,0 +1,495 @@ + use super::*; + use crate::structures::MainStats; + + #[test] + fn test_config_from_args_s3() { + let config = CloudBackupConfig::from_args( + Some("s3"), + Some("my-bucket"), + Some("us-west-2"), + None, + "cuemap/", + true, + ) + .unwrap(); + + assert!(config.enabled); + assert!(config.auto_backup); + assert_eq!(config.prefix, "cuemap/"); + + match config.provider { + Some(CloudProvider::S3 { + bucket, + region, + endpoint, + }) => { + assert_eq!(bucket, "my-bucket"); + assert_eq!(region, "us-west-2"); + assert!(endpoint.is_none()); + } + _ => panic!("Expected S3 provider"), + } + } + + #[test] + fn test_config_from_args_s3_with_endpoint() { + let config = CloudBackupConfig::from_args( + Some("s3"), + Some("my-bucket"), + Some("us-east-1"), + Some("http://localhost:9000"), + "backups/", + false, + ) + .unwrap(); + + match config.provider { + Some(CloudProvider::S3 { endpoint, .. }) => { + assert_eq!(endpoint, Some("http://localhost:9000".to_string())); + } + _ => panic!("Expected S3 provider"), + } + } + + #[test] + fn test_config_from_args_gcs() { + let config = + CloudBackupConfig::from_args(Some("gcs"), Some("gcs-bucket"), None, None, "", false) + .unwrap(); + + match config.provider { + Some(CloudProvider::GCS { bucket }) => { + assert_eq!(bucket, "gcs-bucket"); + } + _ => panic!("Expected GCS provider"), + } + } + + #[test] + fn test_config_from_args_missing_bucket() { + let result = CloudBackupConfig::from_args(Some("s3"), None, None, None, "", false); + assert!(result.is_err()); + } + + #[test] + fn test_config_disabled_by_default() { + let config = CloudBackupConfig::from_args(None, None, None, None, "", false).unwrap(); + + assert!(!config.enabled); + assert!(config.provider.is_none()); + } + + #[test] + fn save_and_load_snapshot_round_trip_preserves_engine_indexes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("project.bin"); + let engine = CueMapEngine::::new(); + let mut metadata = std::collections::HashMap::new(); + metadata.insert("source".to_string(), serde_json::json!("notes.md")); + let memory_id = engine.add_memory_with_source_key( + "persisted content".to_string(), + vec!["persisted".to_string(), "content".to_string()], + Some(metadata), + MainStats::default(), + true, + Some("notes.md#1".to_string()), + ); + + PersistenceManager::save_to_path(&engine, &path).unwrap(); + let snapshot_bytes = std::fs::read(&path).unwrap(); + assert!(crate::crypto::is_compressed(&snapshot_bytes)); + assert!(snapshot_bytes.len() < zstd::stream::decode_all(std::io::Cursor::new(&snapshot_bytes)).unwrap().len()); + let (memories, source_keys, cues, next_id, counts) = + PersistenceManager::load_from_path::(&path).unwrap(); + assert_eq!(memories.len(), 1); + assert_eq!(source_keys.get("notes.md#1").map(|v| *v), Some(memory_id)); + assert!(cues.contains_key("persisted")); + assert!(next_id > memory_id); + assert!(counts.is_none()); + assert_eq!( + memories + .get(&memory_id) + .unwrap() + .access_content(None) + .unwrap(), + "persisted content" + ); + + let missing = + PersistenceManager::load_from_path::(&dir.path().join("missing.bin")); + assert!(missing.is_err()); + } + + #[test] + fn snapshot_listing_and_deletion_filter_auxiliary_files() { + let dir = tempfile::tempdir().unwrap(); + for name in ["b.bin", "a.bin", "a_aliases.bin", "a_lexicon.bin", "ignore.txt"] { + std::fs::write(dir.path().join(name), b"snapshot").unwrap(); + } + assert_eq!( + PersistenceManager::list_snapshots_in_dir(dir.path()), + vec!["a".to_string(), "b".to_string()] + ); + PersistenceManager::delete_snapshot(&dir.path().join("b.bin")).unwrap(); + PersistenceManager::delete_snapshot(&dir.path().join("b.bin")).unwrap(); + assert!(!dir.path().join("b.bin").exists()); + } + + #[test] + fn manager_save_and_load_state_handles_empty_and_populated_directories() { + let dir = tempfile::tempdir().unwrap(); + let manager = PersistenceManager::new(dir.path(), 1); + let (memories, source_keys, cues, next_id) = manager.load_state::().unwrap(); + assert!(memories.is_empty()); + assert!(source_keys.is_empty()); + assert!(cues.is_empty()); + assert_eq!(next_id, 1); + + let engine = CueMapEngine::::new(); + engine.add_memory( + "manager state".to_string(), + vec!["manager".to_string()], + None, + MainStats::default(), + true, + ); + manager.save_state(&engine).unwrap(); + let (memories, _, cues, _) = manager.load_state::().unwrap(); + assert_eq!(memories.len(), 1); + assert!(cues.contains_key("manager")); + } + + #[test] + fn loader_accepts_legacy_uncompressed_bincode_snapshots() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("legacy.bin"); + let state = PersistedState:: { + memories: std::collections::HashMap::new(), + source_key_to_id: std::collections::HashMap::new(), + cue_index: std::collections::HashMap::new(), + next_memory_id: 1, + version: LEGACY_PERSISTENCE_VERSION, + saved_at: 0, + cue_global_counts: None, + }; + std::fs::write(&path, bincode::serialize(&state).unwrap()).unwrap(); + let (_, _, _, next_id, _) = + PersistenceManager::load_from_path::(&path).unwrap(); + assert_eq!(next_id, 1); + } + + #[test] + fn configured_real_snapshot_can_be_checked_during_release_validation() { + let Ok(path) = std::env::var("CUEMAP_CHECK_SNAPSHOT") else { + return; + }; + let loaded = PersistenceManager::load_from_path::(std::path::Path::new(&path)) + .unwrap_or_else(|error| panic!("failed to load configured snapshot {path}: {error}")); + assert!(loaded.0.len() > 0 || loaded.3 >= 1); + } + + #[test] + fn snapshot_loader_rejects_unknown_versions_and_corrupt_payloads() { + let dir = tempfile::tempdir().unwrap(); + let state = PersistedState:: { + memories: std::collections::HashMap::new(), + source_key_to_id: std::collections::HashMap::new(), + cue_index: std::collections::HashMap::new(), + next_memory_id: 1, + version: 999, + saved_at: 0, + cue_global_counts: None, + }; + let unknown_path = dir.path().join("unknown-version.bin"); + std::fs::write(&unknown_path, serialize_state(&state).unwrap()).unwrap(); + let unknown = PersistenceManager::load_from_path::(&unknown_path).unwrap_err(); + assert!(unknown.to_string().contains("Unsupported snapshot version")); + + let corrupt_path = dir.path().join("corrupt.bin"); + std::fs::write(&corrupt_path, b"not a snapshot").unwrap(); + assert!(PersistenceManager::load_from_path::(&corrupt_path).is_err()); + } + + #[test] + fn snapshots_round_trip_global_cue_counts() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("counts.bin"); + let engine = CueMapEngine::::new(); + engine.add_memory( + "counted memory".to_string(), + vec!["counted".to_string()], + None, + MainStats::default(), + true, + ); + engine.cue_global_counts.insert("counted".to_string(), 7); + + PersistenceManager::save_to_path(&engine, &path).unwrap(); + let (_, _, _, _, counts) = PersistenceManager::load_from_path::(&path).unwrap(); + let counts = counts.expect("global cue counts should be persisted"); + assert_eq!(counts.get("counted").map(|value| *value), Some(7)); + } + + #[test] + fn local_cloud_backup_supports_all_snapshot_operations() { + let dir = tempfile::tempdir().unwrap(); + let config = CloudBackupConfig::from_args( + Some("local"), + Some(dir.path().to_str().unwrap()), + None, + None, + "release/", + true, + ) + .unwrap(); + assert!(config.enabled); + assert!(config.auto_backup); + let manager = futures::executor::block_on(CloudBackupManager::new(config)).unwrap(); + + let total = futures::executor::block_on(manager.upload_project_snapshot( + "project", + bytes::Bytes::from_static(b"main"), + Some(bytes::Bytes::from_static(b"aliases")), + Some(bytes::Bytes::from_static(b"lexicon")), + )) + .unwrap(); + assert_eq!(total, 18); + + let single = futures::executor::block_on( + manager.upload_snapshot("single", bytes::Bytes::from_static(b"one")), + ) + .unwrap(); + assert_eq!(single, 3); + assert_eq!( + futures::executor::block_on(manager.download_snapshot("single")) + .unwrap() + .as_ref(), + b"one" + ); + + let (main, aliases, lexicon) = + futures::executor::block_on(manager.download_project_snapshot("project")).unwrap(); + assert_eq!(main.as_ref(), b"main"); + assert_eq!(aliases.unwrap().as_ref(), b"aliases"); + assert_eq!(lexicon.unwrap().as_ref(), b"lexicon"); + + let entries = futures::executor::block_on(manager.list_snapshots()).unwrap(); + let project_ids: Vec<_> = entries.iter().map(|entry| entry.project_id.as_str()).collect(); + assert!(project_ids.contains(&"project")); + assert!(project_ids.contains(&"single")); + assert_eq!(entries.len(), 2); + + futures::executor::block_on(manager.delete_snapshot("project")).unwrap(); + assert!(futures::executor::block_on(manager.download_project_snapshot("project")).is_err()); + futures::executor::block_on(manager.delete_snapshot("missing")).unwrap(); + assert!(manager.is_auto_backup_enabled()); + } + + #[test] + fn cloud_config_reports_unknown_and_unconfigured_provider_errors() { + assert!(CloudBackupConfig::from_args(Some("unknown"), None, None, None, "", false) + .unwrap_err() + .contains("Unknown cloud provider")); + assert!(CloudBackupConfig::from_args(Some("azure"), Some("container"), None, None, "", false) + .is_err()); + assert!(futures::executor::block_on(CloudBackupManager::new(CloudBackupConfig::default())) + .is_err()); + } + + #[test] + fn persistence_private_serialization_and_version_guards_cover_failure_paths() { + assert!(check_snapshot_version(PERSISTENCE_VERSION).is_ok()); + assert!(check_snapshot_version(LEGACY_PERSISTENCE_VERSION).is_ok()); + let unsupported = check_snapshot_version(0).unwrap_err(); + assert!(unsupported.to_string().contains("Unsupported snapshot version")); + + let state = PersistedState:: { + memories: std::collections::HashMap::new(), + source_key_to_id: std::collections::HashMap::new(), + cue_index: std::collections::HashMap::new(), + next_memory_id: 0, + version: PERSISTENCE_VERSION, + saved_at: 123, + cue_global_counts: Some(std::collections::HashMap::from([("cue".to_string(), 4)])), + }; + let encoded = serialize_state(&state).unwrap(); + assert!(crate::crypto::is_compressed(&encoded)); + let decoded: PersistedState = deserialize_state(&encoded).unwrap(); + assert_eq!(decoded.saved_at, 123); + assert_eq!(decoded.cue_global_counts.unwrap().get("cue"), Some(&4)); + assert!(deserialize_state::(b"invalid snapshot").is_err()); + let compressed_invalid = zstd::stream::encode_all(std::io::Cursor::new(b"invalid json"), 3).unwrap(); + assert!(deserialize_state::(&compressed_invalid).is_err()); + } + + #[test] + fn persistence_load_state_handles_legacy_zero_ids_and_corrupt_files() { + let dir = tempfile::tempdir().unwrap(); + let manager = PersistenceManager::new(dir.path(), 1); + let state = PersistedState:: { + memories: std::collections::HashMap::new(), + source_key_to_id: std::collections::HashMap::new(), + cue_index: std::collections::HashMap::new(), + next_memory_id: 0, + version: PERSISTENCE_VERSION, + saved_at: 0, + cue_global_counts: None, + }; + std::fs::write(dir.path().join("cuemap.bin"), serialize_state(&state).unwrap()).unwrap(); + let (_, _, _, next_id) = manager.load_state::().unwrap(); + assert_eq!(next_id, 1); + + std::fs::write(dir.path().join("cuemap.bin"), b"corrupt").unwrap(); + assert!(manager.load_state::().is_err()); + + let legacy = PersistedState:: { + memories: std::collections::HashMap::new(), + source_key_to_id: std::collections::HashMap::new(), + cue_index: std::collections::HashMap::new(), + next_memory_id: 17, + version: LEGACY_PERSISTENCE_VERSION, + saved_at: 0, + cue_global_counts: None, + }; + std::fs::write(dir.path().join("cuemap.bin"), bincode::serialize(&legacy).unwrap()).unwrap(); + let (_, _, _, next_id) = manager.load_state::().unwrap(); + assert_eq!(next_id, 17); + } + + #[test] + fn persistence_save_errors_are_reported_and_snapshot_listing_is_tolerant() { + let dir = tempfile::tempdir().unwrap(); + let engine = CueMapEngine::::new(); + let missing_parent = dir.path().join("missing").join("snapshot.bin"); + assert!(PersistenceManager::save_to_path(&engine, &missing_parent).is_err()); + assert!(PersistenceManager::list_snapshots_in_dir(&dir.path().join("missing")).is_empty()); + + let directory_path = dir.path().join("directory"); + std::fs::create_dir(&directory_path).unwrap(); + assert!(PersistenceManager::delete_snapshot(&directory_path).is_err()); + assert!(PersistenceManager::delete_snapshot(&dir.path().join("does-not-exist")).is_ok()); + } + + #[tokio::test] + async fn background_snapshot_task_ticks_and_can_be_cancelled() { + let dir = tempfile::tempdir().unwrap(); + let manager = PersistenceManager::new(dir.path(), 1); + let engine = std::sync::Arc::new(CueMapEngine::::new()); + engine.add_memory("background".to_string(), vec!["background".to_string()], None, MainStats::default(), true); + let task = manager.start_background_snapshots(engine).await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(dir.path().join("cuemap.bin").exists()); + task.abort(); + let _ = task.await; + let cloned = manager.clone(); + assert!(cloned.load_state::().is_ok()); + } + + #[test] + fn local_cloud_backup_optional_files_and_config_accessors_are_covered() { + let dir = tempfile::tempdir().unwrap(); + let config = CloudBackupConfig::from_args( + Some("local"), + Some(dir.path().to_str().unwrap()), + None, + None, + "optional/", + false, + ) + .unwrap(); + let manager = futures::executor::block_on(CloudBackupManager::new(config)).unwrap(); + assert!(!manager.is_auto_backup_enabled()); + assert_eq!(manager.get_config().prefix, "optional/"); + futures::executor::block_on(manager.upload_snapshot("main-only", bytes::Bytes::from_static(b"main"))).unwrap(); + std::fs::create_dir_all(dir.path().join("optional/main-only_aliases.bin")).unwrap(); + std::fs::create_dir_all(dir.path().join("optional/main-only_lexicon.bin")).unwrap(); + let (main, aliases, lexicon) = futures::executor::block_on(manager.download_project_snapshot("main-only")).unwrap(); + assert_eq!(main.as_ref(), b"main"); + assert!(aliases.is_none()); + assert!(lexicon.is_none()); + assert!(futures::executor::block_on(manager.delete_snapshot("main-only")).is_err()); + std::fs::remove_dir(dir.path().join("optional/main-only_aliases.bin")).unwrap(); + std::fs::remove_dir(dir.path().join("optional/main-only_lexicon.bin")).unwrap(); + futures::executor::block_on(manager.delete_snapshot("main-only")).unwrap(); + } + + #[test] + fn persistence_constructor_and_provider_match_arms_are_exercised() { + let dir = tempfile::tempdir().unwrap(); + let blocker = dir.path().join("not-a-directory"); + std::fs::write(&blocker, b"file").unwrap(); + let _ = PersistenceManager::new(&blocker, 0); + + let s3 = CloudBackupConfig::from_args(Some("s3"), Some("bucket"), None, Some("http://127.0.0.1:9"), "s3/", false).unwrap(); + let _ = futures::executor::block_on(CloudBackupManager::new(s3)); + let gcs = CloudBackupConfig::from_args(Some("gcs"), Some("bucket"), None, None, "gcs/", false).unwrap(); + let _ = futures::executor::block_on(CloudBackupManager::new(gcs)); + + std::env::set_var("AZURE_STORAGE_ACCOUNT_NAME", "test-account"); + let azure = CloudBackupConfig::from_args(Some("azure"), Some("container"), None, None, "azure/", false).unwrap(); + match azure.provider.as_ref() { + Some(CloudProvider::Azure { account, container }) => { + assert_eq!(account, "test-account"); + assert_eq!(container, "container"); + } + other => panic!("unexpected provider: {other:?}"), + } + let _ = futures::executor::block_on(CloudBackupManager::new(azure)); + std::env::remove_var("AZURE_STORAGE_ACCOUNT_NAME"); + } + + #[test] + fn persistence_save_state_round_trips_global_counts_and_reports_serializer_errors() { + let dir = tempfile::tempdir().unwrap(); + let manager = PersistenceManager::new(dir.path(), 1); + let engine = CueMapEngine::::new(); + engine.cue_global_counts.insert("global".to_string(), 9); + manager.save_state(&engine).unwrap(); + let (memories, source_keys, cues, next_id) = manager.load_state::().unwrap(); + assert!(memories.is_empty()); + assert!(source_keys.is_empty()); + assert!(cues.is_empty()); + assert_eq!(next_id, 1); + + #[derive(Clone, Default, serde::Deserialize)] + struct FailingSerialize; + impl serde::Serialize for FailingSerialize { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + Err(serde::ser::Error::custom("intentional serializer failure")) + } + } + impl MemoryStats for FailingSerialize { + fn get_salience(&self) -> f64 { 0.0 } + fn get_effective_salience(&self, _now: u64) -> f64 { 0.0 } + fn get_reinforcement_count(&self) -> u64 { 0 } + fn manual_boost(&mut self) {} + } + let mut memories = std::collections::HashMap::new(); + memories.insert(1, Memory::::new(Vec::new(), None)); + let failing = PersistedState { + memories, + source_key_to_id: std::collections::HashMap::new(), + cue_index: std::collections::HashMap::new(), + next_memory_id: 1, + version: PERSISTENCE_VERSION, + saved_at: 0, + cue_global_counts: None, + }; + assert!(serialize_state(&failing).is_err()); + } + + #[tokio::test] + async fn background_snapshot_logs_failure_when_data_path_is_not_directory() { + let dir = tempfile::tempdir().unwrap(); + let blocker = dir.path().join("data-file"); + std::fs::write(&blocker, b"blocked").unwrap(); + let manager = PersistenceManager::new(&blocker, 1); + let engine = std::sync::Arc::new(CueMapEngine::::new()); + let task = manager.start_background_snapshots(engine).await; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + task.abort(); + let _ = task.await; + } diff --git a/tests/unit/semantic.rs b/tests/unit/semantic.rs new file mode 100644 index 0000000..939a4f5 --- /dev/null +++ b/tests/unit/semantic.rs @@ -0,0 +1,101 @@ + use super::*; + + fn config() -> SemanticConfig { + SemanticConfig { + enabled: true, + dimensions: 3, + ann_tables: 2, + ann_bits: 4, + ann_probes: 2, + candidate_limit: 8, + exact_fallback_max: 32, + ..SemanticConfig::default() + } + } + + #[test] + fn profiles_resolve_to_device_appropriate_defaults() { + let mut config = SemanticConfig::default(); + config.profile = SemanticProfile::Edge; + let resolved = config.resolved(); + assert!(resolved.enabled); + assert_eq!(resolved.dimensions, 384); + assert_eq!(resolved.storage, SemanticStorage::Int8); + assert_eq!(resolved.max_memory_mb, 32); + assert_eq!(resolved.model_id, "all-MiniLM-L3-v2"); + assert_eq!(resolved.model_version, "bundled-q4-minilm-l3"); + assert_eq!(resolved.max_tokens, 128); + } + + #[test] + fn quality_profile_uses_compact_hybrid_defaults() { + let resolved = SemanticConfig::default().resolved(); + + assert_eq!(resolved.profile, SemanticProfile::Quality); + assert_eq!(resolved.dimensions, 384); + assert_eq!(resolved.storage, SemanticStorage::Int8); + assert_eq!(resolved.model_id, "all-MiniLM-L3-v2"); + assert_eq!(resolved.model_version, "bundled-qint8-minilm-l3"); + assert_eq!(resolved.max_tokens, 128); + assert_eq!(resolved.index, SemanticIndexMode::Auto); + assert_eq!(resolved.semantic_rerank_weight, 0.60); + assert_eq!(resolved.semantic_rerank_candidate_limit, 200); + assert_eq!(resolved.query_embedding_cache_capacity, 256); + assert_eq!(resolved.intent_rerank_weight, 0.65); + assert_eq!(resolved.intent_rerank_max_delta, 64.0); + } + + #[test] + fn intent_delta_cap_is_non_negative_and_configurable() { + let mut config = SemanticConfig::default(); + config.intent_rerank_max_delta = -10.0; + assert_eq!(config.resolved().intent_rerank_max_delta, 0.0); + + config.intent_rerank_max_delta = 7.5; + assert_eq!(config.resolved().intent_rerank_max_delta, 7.5); + } + + #[test] + fn compact_storage_has_expected_size_and_similarity() { + let vector = [1.0, 0.0, 0.0, 0.25, 0.1, -0.2, 0.05, 0.3]; + let f32_vector = StoredSemanticVector::from_f32(&vector, SemanticStorage::F32).unwrap(); + let f16_vector = StoredSemanticVector::from_f32(&vector, SemanticStorage::F16).unwrap(); + let int8_vector = StoredSemanticVector::from_f32(&vector, SemanticStorage::Int8).unwrap(); + assert!(f16_vector.estimated_bytes() < f32_vector.estimated_bytes()); + assert!(int8_vector.estimated_bytes() < f16_vector.estimated_bytes()); + assert!(int8_vector.cosine_similarity(&vector).unwrap() > 0.99); + } + + #[test] + fn index_returns_nearest_vector() { + let mut index = SemanticIndex::new(config()); + let first = StoredSemanticVector::from_f32(&[1.0, 0.0, 0.0], SemanticStorage::F32).unwrap(); + let second = StoredSemanticVector::from_f32(&[0.0, 1.0, 0.0], SemanticStorage::F32).unwrap(); + let third = StoredSemanticVector::from_f32(&[0.9, 0.1, 0.0], SemanticStorage::F32).unwrap(); + index.insert(1, &first).unwrap(); + index.insert(2, &second).unwrap(); + index.insert(3, &third).unwrap(); + + let results = index.query_candidate_ids(&[1.0, 0.0, 0.0], 2).unwrap(); + assert_eq!(results.len(), 2); + assert!(results.contains(&1)); + } + + #[test] + fn dimension_mismatch_is_rejected() { + let mut index = SemanticIndex::new(config()); + let first = StoredSemanticVector::from_f32(&[1.0, 0.0, 0.0], SemanticStorage::F32).unwrap(); + let second = StoredSemanticVector::from_f32(&[1.0, 0.0], SemanticStorage::F32).unwrap(); + index.insert(1, &first).unwrap(); + assert!(index.insert(2, &second).is_err()); + assert!(index.query_candidate_ids(&[1.0, 0.0], 1).is_err()); + } + + #[test] + fn reranker_is_data_driven() { + let model = LinearReranker { + bias: 0.25, + weights: vec![2.0, -1.0], + }; + assert!((model.score(&[0.5, 0.25]) - 1.0).abs() < f32::EPSILON); + } diff --git a/tests/unit/semantic_encoder.rs b/tests/unit/semantic_encoder.rs new file mode 100644 index 0000000..e21e996 --- /dev/null +++ b/tests/unit/semantic_encoder.rs @@ -0,0 +1,41 @@ + use super::*; + + #[test] + fn bundled_minilm_encoder_produces_normalized_vectors() { + let config = SemanticConfig::default().resolved(); + assert_eq!(config.max_tokens, 128); + let encoder = OnnxSemanticEncoder::from_config(&config) + .expect("bundled MiniLM assets should load"); + let vector = encoder + .encode("A short local semantic encoder smoke test") + .expect("bundled MiniLM should encode text"); + + assert_eq!(vector.len(), 384); + let norm = vector + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + assert!((norm - 1.0).abs() < 1e-4, "unexpected vector norm: {norm}"); + } + + #[test] + fn bundled_edge_minilm_encoder_produces_normalized_vectors() { + let mut config = SemanticConfig::default(); + config.profile = crate::semantic::SemanticProfile::Edge; + let config = config.resolved(); + assert_eq!(config.max_tokens, 128); + let encoder = OnnxSemanticEncoder::from_config(&config) + .expect("bundled edge MiniLM assets should load"); + let vector = encoder + .encode("A short edge semantic encoder smoke test") + .expect("bundled edge MiniLM should encode text"); + + assert_eq!(vector.len(), 384); + let norm = vector + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + assert!((norm - 1.0).abs() < 1e-4, "unexpected edge vector norm: {norm}"); + }