perf(webapp): select log search predicates by term shape

Use indexed whole-word matching for ASCII alphabetic search terms. Keep
substring matching for terms containing digits, punctuation, non-ASCII
text, or spaces.

Mono-RevId: 891d4b39cb78efd74ca986d55b2e287cc304747d

Author: carderne

apps/webapp/app/presenters/v3/LogsListPresenter.server.ts

@@ -21,9 +21,9 @@ import {
 } from "~/v3/eventRepository/clickhouseEventRepository.server";
 import { ServiceValidationError } from "~/v3/services/baseService.server";
 import {
-  escapeClickHouseLike,
   hasMinimumLogsSearchLength,
   logsSearchExpansionPeriod,
+  logsSearchPredicate,
   LOGS_SEARCH_RETRY_OVERFETCH_FACTOR,
   MIN_LOGS_SEARCH_LENGTH,
   normalizeLogsSearchTerm,
@@ -255,21 +255,26 @@ export class LogsListPresenter extends BasePresenter {
         `Log searches must be at least ${MIN_LOGS_SEARCH_LENGTH} characters.`
       );
     }
-    const searchTerm =
-      normalizedSearchTerm === "" ? undefined : escapeClickHouseLike(normalizedSearchTerm);
+    const searchPredicate =
+      normalizedSearchTerm === "" ? undefined : logsSearchPredicate(normalizedSearchTerm);
 
     // Run exactly one bounded query. Broadening a search window is an explicit user action;
     // silently rescanning the same recent rows makes absence queries needlessly expensive.
     const runQuery = () => {
       const queryBuilder = this.clickhouse.taskEventsSearch.logsListQueryBuilder({
         // Scoped to this query rather than the logs client, which other pages share. ClickHouse
         // skips lazy materialization when LIMIT exceeds the max, so pass this query's limit.
-        settings: env.CLICKHOUSE_LOGS_LIST_LAZY_MATERIALIZATION
-          ? {
-              query_plan_optimize_lazy_materialization: 1,
-              query_plan_max_limit_for_lazy_materialization: queryLimit,
-            }
-          : undefined,
+        settings: {
+          ...(env.CLICKHOUSE_LOGS_LIST_LAZY_MATERIALIZATION
+            ? {
+                query_plan_optimize_lazy_materialization: 1,
+                query_plan_max_limit_for_lazy_materialization: queryLimit,
+              }
+            : {}),
+          ...(searchPredicate && searchPredicate.kind !== "word"
+            ? { ignore_data_skipping_indices: "idx_search_text,idx_search_words" }
+            : {}),
+        },
       });
 
       // The projector excludes events without a trace_id.
@@ -308,9 +313,13 @@ export class LogsListPresenter extends BasePresenter {
         queryBuilder.where("run_id = {runId: String}", { runId });
       }
 
-      if (searchTerm !== undefined) {
+      if (searchPredicate?.kind === "word") {
+        queryBuilder.where("hasAllTokens(concat(search_text, ''), {searchTokens: String})", {
+          searchTokens: searchPredicate.term,
+        });
+      } else if (searchPredicate) {
         queryBuilder.where("search_text LIKE {searchPattern: String}", {
-          searchPattern: `%${searchTerm}%`,
+          searchPattern: searchPredicate.pattern,
         });
       }
 
@@ -378,7 +387,7 @@ export class LogsListPresenter extends BasePresenter {
     if (queryError) {
       if (isClickhouseResourceLimitError(queryError)) {
         throw new ServiceValidationError(
-          searchTerm === undefined
+          searchPredicate === undefined
             ? "These logs took too long to load. Try a shorter time range or add a filter."
             : "This search took too long. Try a shorter time range, a more specific search, or add a filter."
         );
@@ -440,7 +449,7 @@ export class LogsListPresenter extends BasePresenter {
     });
 
     const searchExpansion =
-      searchTerm !== undefined && time.isDefault && transformedLogs.length === 0
+      searchPredicate !== undefined && time.isDefault && transformedLogs.length === 0
         ? logsSearchExpansionPeriod(effectiveFrom, clampedTo, retentionLimitDays)
         : undefined;
 

apps/webapp/app/utils/logSearch.test.ts

@@ -3,6 +3,7 @@ import {
   escapeClickHouseLike,
   hasMinimumLogsSearchLength,
   logsSearchExpansionPeriod,
+  logsSearchPredicate,
   normalizeLogsSearchTerm,
   prepareLogsSearchPage,
 } from "./logSearch";
@@ -24,6 +25,53 @@ describe("log search normalization", () => {
     expect(escapeClickHouseLike("/api/a_b/100%")).toBe("/api/a\\_b/100\\%");
   });
 
+  it("uses token matching only for ASCII letters", () => {
+    expect(logsSearchPredicate("attempt")).toEqual({ kind: "word", term: "attempt" });
+  });
+
+  it("keeps partial ids and numeric terms on LIKE", () => {
+    expect(logsSearchPredicate("zahlung123")).toEqual({
+      kind: "id",
+      pattern: "%zahlung123%",
+    });
+    expect(logsSearchPredicate("123")).toEqual({ kind: "id", pattern: "%123%" });
+    expect(logsSearchPredicate("run_abc123")).toEqual({
+      kind: "id",
+      pattern: "%run\\_abc123%",
+    });
+    expect(logsSearchPredicate("api/orders:42@example.com+retry")).toEqual({
+      kind: "id",
+      pattern: "%api/orders:42@example.com+retry%",
+    });
+    expect(logsSearchPredicate("/api/orders/42")).toEqual({
+      kind: "id",
+      pattern: "%/api/orders/42%",
+    });
+  });
+
+  it("keeps non-ASCII terms, phrases, and irregular delimiter shapes on LIKE", () => {
+    expect(logsSearchPredicate("你好世界")).toEqual({
+      kind: "phrase",
+      pattern: "%你好世界%",
+    });
+    expect(logsSearchPredicate("café")).toEqual({
+      kind: "phrase",
+      pattern: "%café%",
+    });
+    expect(logsSearchPredicate("attempt failed")).toEqual({
+      kind: "phrase",
+      pattern: "%attempt failed%",
+    });
+    expect(logsSearchPredicate("run_abc123 other")).toEqual({
+      kind: "phrase",
+      pattern: "%run\\_abc123 other%",
+    });
+    expect(logsSearchPredicate("___")).toEqual({
+      kind: "phrase",
+      pattern: "%\\_\\_\\_%",
+    });
+  });
+
   it("requires at least three unicode characters after trimming", () => {
     expect(hasMinimumLogsSearchLength("ab")).toBe(false);
     expect(hasMinimumLogsSearchLength("  ab  ")).toBe(false);

apps/webapp/app/utils/logSearch.ts

@@ -58,6 +58,33 @@ export function escapeClickHouseLike(value: string): string {
   return value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
 }
 
+export type LogsSearchPredicate =
+  | { kind: "word"; term: string }
+  | { kind: "id"; pattern: string }
+  | { kind: "phrase"; pattern: string };
+
+const LOGS_SEARCH_WORD_PATTERN = /^[a-z]+$/;
+const LOGS_SEARCH_ID_PATTERN = /^[a-z0-9_\-.:/@+]+$/;
+const LOGS_SEARCH_ID_CONTENT_PATTERN = /[a-z0-9]/;
+const LOGS_SEARCH_ID_MARKER_PATTERN = /[0-9_\-.:/@+]/;
+
+export function logsSearchPredicate(normalizedTerm: string): LogsSearchPredicate {
+  if (LOGS_SEARCH_WORD_PATTERN.test(normalizedTerm)) {
+    return { kind: "word", term: normalizedTerm };
+  }
+
+  const pattern = `%${escapeClickHouseLike(normalizedTerm)}%`;
+  if (
+    LOGS_SEARCH_ID_PATTERN.test(normalizedTerm) &&
+    LOGS_SEARCH_ID_CONTENT_PATTERN.test(normalizedTerm) &&
+    LOGS_SEARCH_ID_MARKER_PATTERN.test(normalizedTerm)
+  ) {
+    return { kind: "id", pattern };
+  }
+
+  return { kind: "phrase", pattern };
+}
+
 export function normalizeLogsSearchTerm(value: string): string {
   return normalizeSearchText(value).trim();
 }

internal-packages/clickhouse/src/taskEvents.ts

@@ -536,8 +536,6 @@ export function getLogsSearchListQueryBuilder(ch: ClickhouseReader) {
     ],
     settings: {
       use_query_condition_cache: 1,
-      // The ngram text index covers every organization's rows in a part, so reading it costs more
-      // than scanning one environment's time range, often by seconds when it is not cached.
       ignore_data_skipping_indices: "idx_search_text",
       // Hold the response until the query finishes so a limit error arrives as an HTTP error the
       // client turns into a QueryError, not mid-stream. Pages are a few hundred small rows.