fix(core): support CJK terms in relaxed FTS fallback

src/basic_memory/repository/postgres_search_repository.py

@@ -6,6 +6,7 @@
 from datetime import datetime
 from typing import List, Optional
 
+import logfire
 from loguru import logger
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -998,36 +999,61 @@ async def search(
 
         logger.trace(f"Search {sql} params: {params}")
 
+        use_savepoint = session is not None or allow_relaxed
+
+        async def execute_rows(active_session: AsyncSession, query_params: dict):
+            # PostgreSQL leaves a transaction unusable after invalid tsquery syntax.
+            # Scope retryable or caller-owned attempts to a savepoint so a relaxed
+            # retry—and any caller continuing to use its session—starts healthy.
+            if use_savepoint:
+                async with active_session.begin_nested():
+                    result = await active_session.execute(text(sql), query_params)
+                    return result.fetchall()
+            result = await active_session.execute(text(sql), query_params)
+            return result.fetchall()
+
         async def run_search(active_session: AsyncSession):
-            result = await active_session.execute(text(sql), params)
-            rows = result.fetchall()
+            relaxed = self._relaxed_tsquery_text(search_text) if allow_relaxed else None
+            strict_syntax_error = False
+            try:
+                rows = await execute_rows(active_session, params)
+            except Exception as exc:
+                if not (self._is_tsquery_syntax_error(exc) and relaxed and params.get("text")):
+                    raise
+                strict_syntax_error = True
+                rows = []
+
             # Trigger: multi-word natural-language query matched nothing
-            # under the default all-terms-AND tsquery semantics.
+            # under the default all-terms-AND tsquery semantics, or its punctuation
+            # produced invalid strict tsquery syntax.
             # Why: questions rarely have every word in one document;
-            # without relaxation the FTS half of hybrid search contributes
-            # zero candidates (parity with the SQLite path).
+            # without relaxation the FTS half of hybrid search contributes zero
+            # candidates. The relaxed renderer also tokenizes punctuation safely.
             # Outcome: one retry with OR-joined prefix lexemes; ts_rank
             # still ranks multi-term matches first.
-            relaxed = (
-                self._relaxed_tsquery_text(search_text) if allow_relaxed and not rows else None
-            )
-            if relaxed and params.get("text"):
-                params["text"] = relaxed
-                result = await active_session.execute(text(sql), params)
-                rows = result.fetchall()
+            if relaxed and not rows and params.get("text"):
+                retry_reason = "invalid syntax" if strict_syntax_error else "0 results"
+                logger.debug(
+                    f"Strict Postgres FTS returned {retry_reason}; retrying relaxed FTS query "
+                    f"strict='{search_text}' relaxed='{relaxed}'"
+                )
+                with logfire.span(
+                    "search.relaxed_fts_retry",
+                    backend="postgres",
+                    reason="syntax_error" if strict_syntax_error else "empty_result",
+                    token_count=len(relaxed_query_words(search_text) or ()),
+                    limit=limit,
+                    offset=offset,
+                ):
+                    rows = await execute_rows(
+                        active_session,
+                        {**params, "text": relaxed},
+                    )
             return rows
 
         try:
             if session is not None:
-                # Trigger: caller owns the session and keeps using it after this call.
-                # Why: a tsquery syntax error aborts the whole Postgres transaction, so
-                #   swallowing it and returning [] would poison the caller-owned
-                #   transaction and fail every later query on that session. Postgres
-                #   SAVEPOINT (begin_nested) scopes the failure to the FTS attempt.
-                # Outcome: on syntax error the savepoint rolls back and the caller's
-                #   outer transaction stays usable; the empty-result contract is kept.
-                async with session.begin_nested():
-                    rows = await run_search(session)
+                rows = await run_search(session)
             else:
                 async with db.scoped_session(self.session_maker) as owned_session:
                     rows = await run_search(owned_session)
@@ -1087,6 +1113,7 @@ async def count(
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
+        allow_relaxed: bool = False,
     ) -> int:
         """Count indexed content matching the Postgres FTS query."""
         if retrieval_mode != SearchRetrievalMode.FTS:
@@ -1123,10 +1150,39 @@ async def count(
         )
         sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}"
         logger.trace(f"Count {sql} params: {params}")
+
+        async def execute_count(active_session: AsyncSession, query_params: dict) -> int:
+            if allow_relaxed:
+                async with active_session.begin_nested():
+                    result = await active_session.execute(text(sql), query_params)
+                    return int(result.scalar_one())
+            result = await active_session.execute(text(sql), query_params)
+            return int(result.scalar_one())
+
         try:
             async with db.scoped_session(self.session_maker) as session:
-                result = await session.execute(text(sql), params)
-                return int(result.scalar_one())
+                relaxed = self._relaxed_tsquery_text(search_text) if allow_relaxed else None
+                strict_syntax_error = False
+                try:
+                    total = await execute_count(session, params)
+                except Exception as exc:
+                    if not (self._is_tsquery_syntax_error(exc) and relaxed and params.get("text")):
+                        raise
+                    strict_syntax_error = True
+                    total = 0
+
+                if relaxed and total == 0 and params.get("text"):
+                    with logfire.span(
+                        "search.count.relaxed_fts_retry",
+                        backend="postgres",
+                        reason="syntax_error" if strict_syntax_error else "empty_result",
+                        token_count=len(relaxed_query_words(search_text) or ()),
+                    ):
+                        total = await execute_count(
+                            session,
+                            {**params, "text": relaxed},
+                        )
+                return total
         except Exception as e:
             if self._is_tsquery_syntax_error(e):
                 logger.warning(f"tsquery syntax error for search term: {search_text}, error: {e}")

src/basic_memory/repository/search_repository.py

@@ -67,6 +67,7 @@ async def count(
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
+        allow_relaxed: bool = False,
     ) -> int:
         """Count indexed content matching the same filters as search."""
         ...

src/basic_memory/repository/search_repository_base.py

@@ -45,9 +45,45 @@
 # that displace genuine results from the ranking window.
 RELAXATION_STOPWORDS = frozenset(
     "a an and are as at be but by did do does for from had has have how i in is it of on "
-    "or that the their they this to was we were what when where which who whom whose why "
+    "or our that the their they this to was we were what when where which who whom whose why "
     "will with you your".split()
 )
+RELAXATION_CJK_PATTERN = re.compile(
+    r"["
+    r"\u1100-\u11ff"  # Hangul Jamo
+    r"\u3040-\u30ff"  # Hiragana and Katakana
+    r"\u3130-\u318f"  # Hangul Compatibility Jamo
+    r"\u31f0-\u31ff"  # Katakana Phonetic Extensions
+    r"\u3400-\u4dbf"  # CJK Unified Ideographs Extension A
+    r"\u4e00-\u9fff"  # CJK Unified Ideographs
+    r"\ua960-\ua97f"  # Hangul Jamo Extended-A
+    r"\uac00-\ud7af"  # Hangul Syllables
+    r"\ud7b0-\ud7ff"  # Hangul Jamo Extended-B
+    r"\uf900-\ufaff"  # CJK Compatibility Ideographs
+    r"\uff65-\uff9f"  # Halfwidth Katakana
+    r"]"
+)
+RELAXATION_ASCII_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9]+")
+RELAXATION_EDGE_PUNCTUATION = "?!.,;:，。！？；：、"
+
+
+def _dedupe_relaxation_words(words: list[str]) -> list[str]:
+    """Preserve first-seen relaxed terms while removing duplicates case-insensitively."""
+    deduped_terms: list[str] = []
+    seen_terms: set[str] = set()
+    for word in words:
+        key = word.lower()
+        if key in seen_terms:
+            continue
+        seen_terms.add(key)
+        deduped_terms.append(word)
+    return deduped_terms
+
+
+def _split_relaxation_words(search_text: str) -> list[str]:
+    """Split whitespace-delimited relaxed terms, preserving CJK words."""
+    words = [word.strip(RELAXATION_EDGE_PUNCTUATION) for word in search_text.split()]
+    return [word for word in words if word]
 
 
 def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]:
@@ -62,6 +98,8 @@ def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]:
     - fewer than three alphanumeric tokens (short queries like "New Feature"
       over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize
       to 1.0 and can outrank the vector result the user wanted);
+    - CJK terms separated by whitespace can relax with two or more terms because
+      the ASCII token gate would otherwise suppress the fallback entirely;
     - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries
       over-broaden and create false positives under OR.
     """
@@ -72,16 +110,29 @@ def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]:
         return None
     # Eligibility checks run on raw alphanumeric tokens (parity with the
     # service), before stopword filtering.
-    tokens = re.findall(r"[A-Za-z0-9]+", stripped.lower())
+    cjk_words = _split_relaxation_words(stripped)
+    has_cjk_term = any(RELAXATION_CJK_PATTERN.search(word) for word in cjk_words)
+
+    if has_cjk_term:
+        if len(cjk_words) < 2 or any(word.isdigit() for word in cjk_words):
+            return None
+        pruned_words = [
+            word
+            for word in cjk_words
+            if word.isalnum() and word.lower() not in RELAXATION_STOPWORDS
+        ]
+        relaxed_words = _dedupe_relaxation_words(pruned_words)
+        # Trigger: punctuation/stopword pruning or deduplication leaves only one term.
+        # Why: the raw whitespace count can make an identifier-like mixed query
+        # appear multi-term even though only one backend-safe CJK prefix remains.
+        # Outcome: preserve the short-query guard after pruning to avoid a broad retry.
+        return relaxed_words if len(relaxed_words) >= 2 else None
+
+    tokens = RELAXATION_ASCII_TOKEN_PATTERN.findall(stripped.lower())
     if len(tokens) < 3 or any(token.isdigit() for token in tokens):
         return None
-    words = [word.strip("?!.,;:") for word in stripped.split()]
-    words = [
-        word
-        for word in words
-        if word and word.isalnum() and word.lower() not in RELAXATION_STOPWORDS
-    ]
-    return words or None
+    pruned_words = [token for token in tokens if token not in RELAXATION_STOPWORDS]
+    return _dedupe_relaxation_words(pruned_words or tokens) or None
 
 
 # Entity, observation, and relation rows in search_index carry ids from independent
@@ -312,6 +363,7 @@ async def count(
         metadata_filters: Optional[Dict[str, Any]] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
+        allow_relaxed: bool = False,
     ) -> int:
         """Count results when a backend-specific COUNT query is available."""
         if retrieval_mode != SearchRetrievalMode.FTS:

src/basic_memory/repository/sqlite_search_repository.py

@@ -6,6 +6,8 @@
 from contextlib import asynccontextmanager
 from datetime import datetime
 from typing import List, Optional
+
+import logfire
 from loguru import logger
 from sqlalchemy import text
 from sqlalchemy.exc import OperationalError as SAOperationalError
@@ -1064,8 +1066,19 @@ async def run_search(active_session: AsyncSession):
             relaxed = self._relaxed_fts_text(search_text) if allow_relaxed and not rows else None
             if relaxed and params.get("text"):
                 params["text"] = relaxed
-                result = await active_session.execute(text(sql), params)
-                rows = result.fetchall()
+                logger.debug(
+                    "Strict SQLite FTS returned 0 results; retrying relaxed FTS query "
+                    f"strict='{search_text}' relaxed='{relaxed}'"
+                )
+                with logfire.span(
+                    "search.relaxed_fts_retry",
+                    backend="sqlite",
+                    token_count=len(relaxed_query_words(search_text) or ()),
+                    limit=limit,
+                    offset=offset,
+                ):
+                    result = await active_session.execute(text(sql), params)
+                    rows = result.fetchall()
             return rows
 
         try:
@@ -1128,6 +1141,7 @@ async def count(
         metadata_filters: Optional[dict] = None,
         retrieval_mode: SearchRetrievalMode = SearchRetrievalMode.FTS,
         min_similarity: Optional[float] = None,
+        allow_relaxed: bool = False,
     ) -> int:
         """Count indexed content matching the SQLite FTS query."""
         if retrieval_mode != SearchRetrievalMode.FTS:
@@ -1161,7 +1175,20 @@ async def count(
         try:
             async with db.scoped_session(self.session_maker) as session:
                 result = await session.execute(text(sql), params)
-                return int(result.scalar_one())
+                total = int(result.scalar_one())
+                relaxed = (
+                    self._relaxed_fts_text(search_text) if allow_relaxed and total == 0 else None
+                )
+                if relaxed and params.get("text"):
+                    params["text"] = relaxed
+                    with logfire.span(
+                        "search.count.relaxed_fts_retry",
+                        backend="sqlite",
+                        token_count=len(relaxed_query_words(search_text) or ()),
+                    ):
+                        result = await session.execute(text(sql), params)
+                        total = int(result.scalar_one())
+                return total
         except Exception as e:
             if self._is_fts5_syntax_error(e):  # pragma: no cover
                 logger.warning(f"FTS5 syntax error for search term: {search_text}, error: {e}")