diff --git a/src/basic_memory/repository/postgres_search_repository.py b/src/basic_memory/repository/postgres_search_repository.py index 723ffc12d..e5def3629 100644 --- a/src/basic_memory/repository/postgres_search_repository.py +++ b/src/basic_memory/repository/postgres_search_repository.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import List, Optional +import logfire from loguru import logger from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession @@ -998,36 +999,61 @@ async def search( logger.trace(f"Search {sql} params: {params}") + use_savepoint = session is not None or allow_relaxed + + async def execute_rows(active_session: AsyncSession, query_params: dict): + # PostgreSQL leaves a transaction unusable after invalid tsquery syntax. + # Scope retryable or caller-owned attempts to a savepoint so a relaxed + # retry—and any caller continuing to use its session—starts healthy. + if use_savepoint: + async with active_session.begin_nested(): + result = await active_session.execute(text(sql), query_params) + return result.fetchall() + result = await active_session.execute(text(sql), query_params) + return result.fetchall() + async def run_search(active_session: AsyncSession): - result = await active_session.execute(text(sql), params) - rows = result.fetchall() + relaxed = self._relaxed_tsquery_text(search_text) if allow_relaxed else None + strict_syntax_error = False + try: + rows = await execute_rows(active_session, params) + except Exception as exc: + if not (self._is_tsquery_syntax_error(exc) and relaxed and params.get("text")): + raise + strict_syntax_error = True + rows = [] + # Trigger: multi-word natural-language query matched nothing - # under the default all-terms-AND tsquery semantics. + # under the default all-terms-AND tsquery semantics, or its punctuation + # produced invalid strict tsquery syntax. # Why: questions rarely have every word in one document; - # without relaxation the FTS half of hybrid search contributes - # zero candidates (parity with the SQLite path). + # without relaxation the FTS half of hybrid search contributes zero + # candidates. The relaxed renderer also tokenizes punctuation safely. # Outcome: one retry with OR-joined prefix lexemes; ts_rank # still ranks multi-term matches first. - relaxed = ( - self._relaxed_tsquery_text(search_text) if allow_relaxed and not rows else None - ) - if relaxed and params.get("text"): - params["text"] = relaxed - result = await active_session.execute(text(sql), params) - rows = result.fetchall() + if relaxed and not rows and params.get("text"): + retry_reason = "invalid syntax" if strict_syntax_error else "0 results" + logger.debug( + f"Strict Postgres FTS returned {retry_reason}; retrying relaxed FTS query " + f"strict='{search_text}' relaxed='{relaxed}'" + ) + with logfire.span( + "search.relaxed_fts_retry", + backend="postgres", + reason="syntax_error" if strict_syntax_error else "empty_result", + token_count=len(relaxed_query_words(search_text) or ()), + limit=limit, + offset=offset, + ): + rows = await execute_rows( + active_session, + {**params, "text": relaxed}, + ) return rows try: if session is not None: - # Trigger: caller owns the session and keeps using it after this call. - # Why: a tsquery syntax error aborts the whole Postgres transaction, so - # swallowing it and returning [] would poison the caller-owned - # transaction and fail every later query on that session. Postgres - # SAVEPOINT (begin_nested) scopes the failure to the FTS attempt. - # Outcome: on syntax error the savepoint rolls back and the caller's - # outer transaction stays usable; the empty-result contract is kept. - async with session.begin_nested(): - rows = await run_search(session) + rows = await run_search(session) else: async with db.scoped_session(self.session_maker) as owned_session: rows = await run_search(owned_session) @@ -1087,6 +1113,7 @@ async def count( metadata_filters: Optional[dict] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, + allow_relaxed: bool = False, ) -> int: """Count indexed content matching the Postgres FTS query.""" if retrieval_mode != SearchRetrievalMode.FTS: @@ -1123,10 +1150,39 @@ async def count( ) sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}" logger.trace(f"Count {sql} params: {params}") + + async def execute_count(active_session: AsyncSession, query_params: dict) -> int: + if allow_relaxed: + async with active_session.begin_nested(): + result = await active_session.execute(text(sql), query_params) + return int(result.scalar_one()) + result = await active_session.execute(text(sql), query_params) + return int(result.scalar_one()) + try: async with db.scoped_session(self.session_maker) as session: - result = await session.execute(text(sql), params) - return int(result.scalar_one()) + relaxed = self._relaxed_tsquery_text(search_text) if allow_relaxed else None + strict_syntax_error = False + try: + total = await execute_count(session, params) + except Exception as exc: + if not (self._is_tsquery_syntax_error(exc) and relaxed and params.get("text")): + raise + strict_syntax_error = True + total = 0 + + if relaxed and total == 0 and params.get("text"): + with logfire.span( + "search.count.relaxed_fts_retry", + backend="postgres", + reason="syntax_error" if strict_syntax_error else "empty_result", + token_count=len(relaxed_query_words(search_text) or ()), + ): + total = await execute_count( + session, + {**params, "text": relaxed}, + ) + return total except Exception as e: if self._is_tsquery_syntax_error(e): logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}") diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index 4cc1019dd..47395a509 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -67,6 +67,7 @@ async def count( metadata_filters: Optional[dict] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, + allow_relaxed: bool = False, ) -> int: """Count indexed content matching the same filters as search.""" ... diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 06fd43044..43ede2b76 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -45,9 +45,45 @@ # that displace genuine results from the ranking window. RELAXATION_STOPWORDS = frozenset( "a an and are as at be but by did do does for from had has have how i in is it of on " - "or that the their they this to was we were what when where which who whom whose why " + "or our that the their they this to was we were what when where which who whom whose why " "will with you your".split() ) +RELAXATION_CJK_PATTERN = re.compile( + r"[" + r"\u1100-\u11ff" # Hangul Jamo + r"\u3040-\u30ff" # Hiragana and Katakana + r"\u3130-\u318f" # Hangul Compatibility Jamo + r"\u31f0-\u31ff" # Katakana Phonetic Extensions + r"\u3400-\u4dbf" # CJK Unified Ideographs Extension A + r"\u4e00-\u9fff" # CJK Unified Ideographs + r"\ua960-\ua97f" # Hangul Jamo Extended-A + r"\uac00-\ud7af" # Hangul Syllables + r"\ud7b0-\ud7ff" # Hangul Jamo Extended-B + r"\uf900-\ufaff" # CJK Compatibility Ideographs + r"\uff65-\uff9f" # Halfwidth Katakana + r"]" +) +RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+") +RELAXATION_EDGE_PUNCTUATION = "?!.,;:,。!?;:、" + + +def _dedupe_relaxation_words(words: list[str]) -> list[str]: + """Preserve first-seen relaxed terms while removing duplicates case-insensitively.""" + deduped_terms: list[str] = [] + seen_terms: set[str] = set() + for word in words: + key = word.lower() + if key in seen_terms: + continue + seen_terms.add(key) + deduped_terms.append(word) + return deduped_terms + + +def _split_relaxation_words(search_text: str) -> list[str]: + """Split whitespace-delimited relaxed terms, preserving CJK words.""" + words = [word.strip(RELAXATION_EDGE_PUNCTUATION) for word in search_text.split()] + return [word for word in words if word] def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]: @@ -62,6 +98,8 @@ def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]: - fewer than three alphanumeric tokens (short queries like "New Feature" over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize to 1.0 and can outrank the vector result the user wanted); + - CJK terms separated by whitespace can relax with two or more terms because + the ASCII token gate would otherwise suppress the fallback entirely; - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries over-broaden and create false positives under OR. """ @@ -72,16 +110,29 @@ def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]: return None # Eligibility checks run on raw alphanumeric tokens (parity with the # service), before stopword filtering. - tokens = re.findall(r"[A-Za-z0-9]+", stripped.lower()) + cjk_words = _split_relaxation_words(stripped) + has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words) + + if has_cjk_term: + if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words): + return None + pruned_words = [ + word + for word in cjk_words + if word.isalnum() and word.lower() not in RELAXATION_STOPWORDS + ] + relaxed_words = _dedupe_relaxation_words(pruned_words) + # Trigger: punctuation/stopword pruning or deduplication leaves only one term. + # Why: the raw whitespace count can make an identifier-like mixed query + # appear multi-term even though only one backend-safe CJK prefix remains. + # Outcome: preserve the short-query guard after pruning to avoid a broad retry. + return relaxed_words if len(relaxed_words) >= 2 else None + + tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower()) if len(tokens) < 3 or any(token.isdigit() for token in tokens): return None - words = [word.strip("?!.,;:") for word in stripped.split()] - words = [ - word - for word in words - if word and word.isalnum() and word.lower() not in RELAXATION_STOPWORDS - ] - return words or None + pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS] + return _dedupe_relaxation_words(pruned_words or tokens) or None # Entity, observation, and relation rows in search_index carry ids from independent @@ -312,6 +363,7 @@ async def count( metadata_filters: Optional[Dict[str, Any]] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, + allow_relaxed: bool = False, ) -> int: """Count results when a backend-specific COUNT query is available.""" if retrieval_mode != SearchRetrievalMode.FTS: diff --git a/src/basic_memory/repository/sqlite_search_repository.py b/src/basic_memory/repository/sqlite_search_repository.py index 79498a7ac..eade25393 100644 --- a/src/basic_memory/repository/sqlite_search_repository.py +++ b/src/basic_memory/repository/sqlite_search_repository.py @@ -6,6 +6,8 @@ from contextlib import asynccontextmanager from datetime import datetime from typing import List, Optional + +import logfire from loguru import logger from sqlalchemy import text from sqlalchemy.exc import OperationalError as SAOperationalError @@ -1064,8 +1066,19 @@ async def run_search(active_session: AsyncSession): relaxed = self._relaxed_fts_text(search_text) if allow_relaxed and not rows else None if relaxed and params.get("text"): params["text"] = relaxed - result = await active_session.execute(text(sql), params) - rows = result.fetchall() + logger.debug( + "Strict SQLite FTS returned 0 results; retrying relaxed FTS query " + f"strict='{search_text}' relaxed='{relaxed}'" + ) + with logfire.span( + "search.relaxed_fts_retry", + backend="sqlite", + token_count=len(relaxed_query_words(search_text) or ()), + limit=limit, + offset=offset, + ): + result = await active_session.execute(text(sql), params) + rows = result.fetchall() return rows try: @@ -1128,6 +1141,7 @@ async def count( metadata_filters: Optional[dict] = None, retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS, min_similarity: Optional[float] = None, + allow_relaxed: bool = False, ) -> int: """Count indexed content matching the SQLite FTS query.""" if retrieval_mode != SearchRetrievalMode.FTS: @@ -1161,7 +1175,20 @@ async def count( try: async with db.scoped_session(self.session_maker) as session: result = await session.execute(text(sql), params) - return int(result.scalar_one()) + total = int(result.scalar_one()) + relaxed = ( + self._relaxed_fts_text(search_text) if allow_relaxed and total == 0 else None + ) + if relaxed and params.get("text"): + params["text"] = relaxed + with logfire.span( + "search.count.relaxed_fts_retry", + backend="sqlite", + token_count=len(relaxed_query_words(search_text) or ()), + ): + result = await session.execute(text(sql), params) + total = int(result.scalar_one()) + return total except Exception as e: if self._is_fts5_syntax_error(e): # pragma: no cover logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}") diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index d4f5be525..e8efabce1 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -23,6 +23,7 @@ SearchRepository, VectorSyncBatchResult, ) +from basic_memory.repository.search_repository_base import relaxed_query_words from basic_memory.schemas.search import SearchQuery, SearchItemType, SearchRetrievalMode from basic_memory.services import FileService @@ -30,42 +31,6 @@ # We use 6000 characters to leave headroom for other indexed columns and overhead. MAX_CONTENT_STEMS_SIZE = 6000 -# Common glue words used to relax natural-language FTS queries after strict misses. -FTS_RELAXED_STOPWORDS = { - "a", - "an", - "and", - "are", - "as", - "at", - "be", - "by", - "for", - "from", - "how", - "in", - "is", - "it", - "of", - "on", - "or", - "our", - "the", - "their", - "this", - "to", - "was", - "we", - "what", - "when", - "where", - "who", - "why", - "with", - "you", - "your", -} - @dataclass(frozen=True) class _PreparedSearchQuery: @@ -240,6 +205,7 @@ async def _search_repository( search_text: str | None, limit: int, offset: int, + allow_relaxed: bool = False, session: AsyncSession | None = None, ) -> List[SearchIndexRow]: return await self.repository.search( @@ -256,6 +222,7 @@ async def _search_repository( min_similarity=prepared.min_similarity, limit=limit, offset=offset, + allow_relaxed=allow_relaxed, session=session, ) @@ -264,6 +231,7 @@ async def _count_repository( prepared: _PreparedSearchQuery, *, search_text: str | None, + allow_relaxed: bool = False, ) -> int: return await self.repository.count( search_text=search_text, @@ -277,6 +245,7 @@ async def _count_repository( metadata_filters=prepared.metadata_filters, retrieval_mode=prepared.retrieval_mode, min_similarity=prepared.min_similarity, + allow_relaxed=allow_relaxed, ) async def search( @@ -312,47 +281,23 @@ async def search( offset=offset, ): logger.trace(f"Searching with query: {query}") + # Repository backends own relaxed FTS rendering because SQLite and + # Postgres use different prefix syntax. Passing a service-built + # boolean OR string would be treated as explicit boolean input and + # lose the prefix matching that rescues compound CJK tokens. + allow_relaxed = self._is_relaxed_fts_fallback_eligible( + query, strict_search_text, prepared.retrieval_mode + ) results = await self._search_repository( prepared, search_text=strict_search_text, limit=limit, offset=offset, + allow_relaxed=allow_relaxed, session=session, ) - # Trigger: strict FTS with plain multi-term text returned no results. - # Why: natural-language queries often include stopwords that over-constrain implicit AND. - # Outcome: retry once with relaxed OR terms while preserving explicit boolean intent. - if results: - return results - if not self._is_relaxed_fts_fallback_eligible( - query, strict_search_text, prepared.retrieval_mode - ): - return results - - assert strict_search_text is not None - relaxed_search_text = self._build_relaxed_fts_query(strict_search_text) - if relaxed_search_text == strict_search_text: - return results - - logger.debug( - "Strict FTS returned 0 results; retrying relaxed FTS query " - f"strict='{strict_search_text}' relaxed='{relaxed_search_text}'" - ) - with logfire.span( - "search.relaxed_fts_retry", - retrieval_mode=prepared.retrieval_mode.value, - token_count=len(self._tokenize_fts_text(strict_search_text)), - limit=limit, - offset=offset, - ): - return await self._search_repository( - prepared, - search_text=relaxed_search_text, - limit=limit, - offset=offset, - session=session, - ) + return results async def count(self, query: SearchQuery) -> int: """Count all indexed rows matching a query.""" @@ -372,50 +317,14 @@ async def count(self, query: SearchQuery) -> int: has_query=has_query, has_filters=has_filters, ): - total = await self._count_repository(prepared, search_text=strict_search_text) - - if total > 0: - return total - if not self._is_relaxed_fts_fallback_eligible( - query, strict_search_text, prepared.retrieval_mode - ): - return total - - assert strict_search_text is not None - relaxed_search_text = self._build_relaxed_fts_query(strict_search_text) - if relaxed_search_text == strict_search_text: - return total - - with logfire.span( - "search.count.relaxed_fts_retry", - retrieval_mode=prepared.retrieval_mode.value, - token_count=len(self._tokenize_fts_text(strict_search_text)), - ): - return await self._count_repository(prepared, search_text=relaxed_search_text) - - @staticmethod - def _tokenize_fts_text(search_text: str) -> list[str]: - """Tokenize text into alphanumeric terms for relaxed FTS fallback.""" - return re.findall(r"[A-Za-z0-9]+", search_text.lower()) - - @classmethod - def _build_relaxed_fts_query(cls, search_text: str) -> str: - """Build a less strict OR query from natural-language input.""" - normalized_terms = cls._tokenize_fts_text(search_text) - if not normalized_terms: - return search_text - - deduped_terms: list[str] = [] - seen_terms: set[str] = set() - for term in normalized_terms: - if term in seen_terms: - continue - seen_terms.add(term) - deduped_terms.append(term) - - pruned_terms = [term for term in deduped_terms if term not in FTS_RELAXED_STOPWORDS] - relaxed_terms = pruned_terms or deduped_terms - return " OR ".join(relaxed_terms) + allow_relaxed = self._is_relaxed_fts_fallback_eligible( + query, strict_search_text, prepared.retrieval_mode + ) + return await self._count_repository( + prepared, + search_text=strict_search_text, + allow_relaxed=allow_relaxed, + ) @classmethod def _is_relaxed_fts_fallback_eligible( @@ -433,18 +342,12 @@ def _is_relaxed_fts_fallback_eligible( return False if query.has_boolean_operators(): return False - tokens = cls._tokenize_fts_text(search_text) - # Trigger: query has only one or two terms (e.g., link titles like "New Feature"). - # Why: OR-relaxing short queries can over-broaden and produce false positives. - # Outcome: require at least three tokens before enabling relaxed fallback. - if len(tokens) < 3: - return False - # Trigger: query contains explicit numeric identifiers (e.g., "root note 1"). - # Why: OR-relaxing identifier-like queries can over-broaden and create false positives. - # Outcome: preserve strict matching for these targeted queries. - if any(token.isdigit() for token in tokens): - return False - return True + # Trigger: query has too few safe relaxed terms, explicit numeric identifiers, + # or only terms that would over-broaden under OR. + # Why: the shared helper preserves the old English guard while allowing + # whitespace-separated CJK terms that ASCII tokenization cannot see. + # Outcome: retry only when there is a backend-safe relaxed OR query. + return relaxed_query_words(search_text) is not None @staticmethod def _generate_variants(text: str) -> Set[str]: diff --git a/tests/repository/test_postgres_search_repository.py b/tests/repository/test_postgres_search_repository.py index 8481edfbd..fcb100319 100644 --- a/tests/repository/test_postgres_search_repository.py +++ b/tests/repository/test_postgres_search_repository.py @@ -1070,7 +1070,7 @@ async def test_postgres_question_punctuation_and_relaxation(session_maker, test_ # Relaxation drops stopwords and OR-joins content terms. relaxed = repo._relaxed_tsquery_text("When did Melanie paint a sunrise?") - assert relaxed == "Melanie:* | paint:* | sunrise:*" + assert relaxed == "melanie:* | paint:* | sunrise:*" # User intent is not second-guessed. assert repo._relaxed_tsquery_text("alpha AND beta") is None @@ -1107,3 +1107,55 @@ async def test_postgres_multiword_query_relaxes_on_strict_miss(session_maker, te # The hybrid FTS branch opts in; OR-relaxation surfaces the partial match. results = await repo.search(search_text="Did Melanie go hiking at sunrise?", allow_relaxed=True) assert any(r.id == 77 for r in results) + + +@pytest.mark.asyncio +async def test_postgres_relaxes_after_strict_tsquery_syntax_error( + session_maker, + test_project, + monkeypatch: pytest.MonkeyPatch, +): + """Punctuated strict queries retry safely with the relaxed token rendering.""" + repo = PostgresSearchRepository(session_maker, project_id=test_project.id) + now = datetime.now(timezone.utc) + await repo.index_item( + SearchIndexRow( + project_id=test_project.id, + id=78, + title="Baz reference", + content_stems="a document containing baz", + content_snippet="A document containing baz.", + permalink="docs/baz-reference", + file_path="docs/baz-reference.md", + type="entity", + metadata={"note_type": "note"}, + created_at=now, + updated_at=now, + ) + ) + + syntax_errors: list[Exception] = [] + real_is_syntax_error = repo._is_tsquery_syntax_error + + def record_syntax_error(exc: Exception) -> bool: + is_syntax_error = real_is_syntax_error(exc) + if is_syntax_error: + syntax_errors.append(exc) + return is_syntax_error + + monkeypatch.setattr(repo, "_is_tsquery_syntax_error", record_syntax_error) + + query = "foo search_syntax_error_count diff --git a/tests/repository/test_search_relaxation.py b/tests/repository/test_search_relaxation.py new file mode 100644 index 000000000..3236a1b42 --- /dev/null +++ b/tests/repository/test_search_relaxation.py @@ -0,0 +1,35 @@ +"""Focused coverage for relaxed full-text query eligibility.""" + +import pytest + +from basic_memory.repository.search_repository_base import relaxed_query_words + + +@pytest.mark.parametrize( + ("query", "expected"), + [ + ("季度 报告", ["季度", "报告"]), + ("カタカナ レポート", ["カタカナ", "レポート"]), + ("분기 보고", ["분기", "보고"]), + ], +) +def test_relaxed_query_words_supports_whitespace_separated_cjk_scripts( + query: str, + expected: list[str], +) -> None: + """Han, kana, and Hangul terms all bypass the ASCII three-token gate.""" + assert relaxed_query_words(query) == expected + + +@pytest.mark.parametrize( + "query", + [ + "SPEC-16 设计", + "foo/bar 季度", + "季度 季度", + "the 季度", + ], +) +def test_relaxed_query_words_preserves_short_query_guard_after_cjk_pruning(query: str) -> None: + """Unsafe, duplicate, or stopword terms cannot pad a one-term CJK relaxation.""" + assert relaxed_query_words(query) is None diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 59c1f93e4..d47a40ab5 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -1149,10 +1149,36 @@ async def test_relaxed_query_drops_stopwords(search_repository): """Relaxation keys on content-bearing terms in each backend's syntax.""" if is_postgres_backend(search_repository): relaxed = search_repository._relaxed_tsquery_text("When did Melanie paint a sunrise?") - assert relaxed == "Melanie:* | paint:* | sunrise:*" + assert relaxed == "melanie:* | paint:* | sunrise:*" else: relaxed = search_repository._relaxed_fts_text("When did Melanie paint a sunrise?") - assert relaxed == "Melanie* OR paint* OR sunrise*" + assert relaxed == "melanie* OR paint* OR sunrise*" + + +@pytest.mark.asyncio +async def test_relaxed_query_preserves_punctuated_ascii_token_pieces(search_repository): + """Hyphenated and slashed ASCII terms should relax using their regex token pieces.""" + if is_postgres_backend(search_repository): + relaxed = search_repository._relaxed_tsquery_text("client-side state management") + assert relaxed == "client:* | side:* | state:* | management:*" + slashed = search_repository._relaxed_tsquery_text("foo/bar baz qux") + assert slashed == "foo:* | bar:* | baz:* | qux:*" + else: + relaxed = search_repository._relaxed_fts_text("client-side state management") + assert relaxed == "client* OR side* OR state* OR management*" + slashed = search_repository._relaxed_fts_text("foo/bar baz qux") + assert slashed == "foo* OR bar* OR baz* OR qux*" + + +@pytest.mark.asyncio +async def test_relaxed_query_supports_whitespace_separated_cjk_terms(search_repository): + """CJK terms separated by spaces should relax even when ASCII tokenization finds none.""" + if is_postgres_backend(search_repository): + relaxed = search_repository._relaxed_tsquery_text("季度 报告") + assert relaxed == "季度:* | 报告:*" + else: + relaxed = search_repository._relaxed_fts_text("季度 报告") + assert relaxed == "季度* OR 报告*" @pytest.mark.asyncio @@ -1205,3 +1231,34 @@ async def test_multiword_query_relaxes_to_or_when_strict_misses(search_repositor search_text="Did Melanie go hiking at sunrise?", allow_relaxed=True ) assert any(r.entity_id == search_entity.id for r in results) + + +@pytest.mark.asyncio +async def test_cjk_compound_query_relaxes_with_backend_prefix_terms( + search_repository, search_entity +): + """Whitespace-separated CJK terms should match indexed CJK compounds when relaxed.""" + row = SearchIndexRow( + project_id=search_repository.project_id, + id=search_entity.id, + type=SearchItemType.ENTITY.value, + title="季度报告总结", + content_snippet="季度报告总结已经完成。", + content_stems="季度报告总结已经完成", + permalink=search_entity.permalink, + file_path=search_entity.file_path, + entity_id=search_entity.id, + metadata={"note_type": search_entity.note_type}, + created_at=search_entity.created_at, + updated_at=search_entity.updated_at, + ) + await search_repository.index_item(row) + + strict = await search_repository.search(search_text="季度 报告") + assert strict == [] + + results = await search_repository.search(search_text="季度 报告", allow_relaxed=True) + assert any(r.entity_id == search_entity.id for r in results) + + total = await search_repository.count(search_text="季度 报告", allow_relaxed=True) + assert total == 1 diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 94d0eea82..32e0a3ace 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -557,11 +557,9 @@ async def test_boolean_operators_detection(search_service): @pytest.mark.asyncio -async def test_plain_multiterm_fts_retries_with_relaxed_or_when_strict_empty( - search_service, monkeypatch -): - """Plain multi-term FTS should retry with relaxed OR query after strict no-results.""" - call_texts: list[str | None] = [] +async def test_plain_multiterm_fts_enables_repository_relaxed_fallback(search_service, monkeypatch): + """Plain multi-term FTS should let the repository render relaxed backend syntax.""" + calls: list[dict] = [] now = datetime.now().astimezone() fallback_row = SearchIndexRow( @@ -578,10 +576,8 @@ async def test_plain_multiterm_fts_retries_with_relaxed_or_when_strict_empty( ) async def fake_search(**kwargs): - call_texts.append(kwargs.get("search_text")) - if len(call_texts) == 1: - return [] - return [fallback_row] + calls.append(kwargs) + return [fallback_row] if kwargs.get("allow_relaxed") else [] monkeypatch.setattr(search_service.repository, "search", fake_search) @@ -590,16 +586,69 @@ async def fake_search(**kwargs): ) assert len(results) == 1 - assert call_texts[0] == "fundraising venture capital" - assert call_texts[1] == "fundraising OR venture OR capital" - assert len(call_texts) == 2 + assert len(calls) == 1 + assert calls[0]["search_text"] == "fundraising venture capital" + assert calls[0]["allow_relaxed"] is True + + +@pytest.mark.asyncio +async def test_plain_cjk_multiterm_fts_enables_repository_relaxed_fallback( + search_service, monkeypatch +): + """Whitespace-separated CJK terms need backend prefix relaxed rendering.""" + calls: list[dict] = [] + + now = datetime.now().astimezone() + fallback_row = SearchIndexRow( + project_id=1, + id=1, + type=SearchItemType.ENTITY.value, + file_path="test/cjk-fallback.md", + created_at=now, + updated_at=now, + permalink="test/cjk-fallback", + metadata={"note_type": "note"}, + title="CJK Fallback Match", + score=1.0, + ) + + async def fake_search(**kwargs): + calls.append(kwargs) + return [fallback_row] if kwargs.get("allow_relaxed") else [] + + monkeypatch.setattr(search_service.repository, "search", fake_search) + + results = await search_service.search( + SearchQuery(text="季度 报告", retrieval_mode=SearchRetrievalMode.FTS) + ) + + assert len(results) == 1 + assert len(calls) == 1 + assert calls[0]["search_text"] == "季度 报告" + assert calls[0]["allow_relaxed"] is True @pytest.mark.asyncio -async def test_relaxed_query_prunes_stopwords(search_service): - """Relaxed query should remove stopwords and keep high-signal terms.""" - relaxed = search_service._build_relaxed_fts_query("who are our main competitors and partners?") - assert relaxed == "main OR competitors OR partners" +async def test_plain_cjk_multiterm_count_enables_repository_relaxed_fallback( + search_service, monkeypatch +): + """Count should use the same backend relaxed fallback as search.""" + calls: list[dict] = [] + + async def fake_count(**kwargs): + calls.append(kwargs) + return 1 if kwargs.get("allow_relaxed") else 0 + + monkeypatch.setattr(search_service.repository, "count", fake_count) + + total = await search_service.count( + SearchQuery(text="季度 报告", retrieval_mode=SearchRetrievalMode.FTS) + ) + + assert total == 1 + assert len(calls) == 1 + assert calls[0]["search_text"] == "季度 报告" + assert calls[0]["allow_relaxed"] is True @pytest.mark.asyncio diff --git a/tests/services/test_search_service_telemetry.py b/tests/services/test_search_service_telemetry.py index 1a32b88a7..820ae351a 100644 --- a/tests/services/test_search_service_telemetry.py +++ b/tests/services/test_search_service_telemetry.py @@ -44,7 +44,7 @@ async def test_search_service_wraps_repository_search(search_service, monkeypatc @pytest.mark.asyncio -async def test_search_service_emits_relaxed_retry_span(search_service, monkeypatch) -> None: +async def test_search_service_delegates_relaxed_retry(search_service, monkeypatch) -> None: import logfire spans, fake_span = _capture_spans() @@ -59,19 +59,7 @@ async def fake_repository_search(**kwargs): await search_service.search(SearchQuery(text="who are our main competitors and partners")) - assert [name for name, _ in spans] == [ - "search.execute", - "search.relaxed_fts_retry", - ] - assert spans[1] == ( - "search.relaxed_fts_retry", - { - "retrieval_mode": "fts", - "token_count": 7, - "limit": 10, - "offset": 0, - }, - ) - assert len(calls) == 2 + assert [name for name, _ in spans] == ["search.execute"] + assert len(calls) == 1 assert calls[0]["search_text"] == "who are our main competitors and partners" - assert calls[1]["search_text"] == "main OR competitors OR partners" + assert calls[0]["allow_relaxed"] is True