From 4467620b5ebcf1dfc3b80fe38a7a8333491e1cb8 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Wed, 12 Aug 2026 13:35:48 -0700 Subject: [PATCH 1/4] fix(web): build agent search queries as IR --- packages/web/src/features/tools/glob.ts | 37 ++--- packages/web/src/features/tools/grep.ts | 43 ++--- .../src/features/tools/searchQuery.test.ts | 148 ++++++++++++++++++ .../web/src/features/tools/searchQuery.ts | 128 +++++++++++++++ .../src/features/tools/searchTools.test.ts | 89 +++++++++++ 5 files changed, 388 insertions(+), 57 deletions(-) create mode 100644 packages/web/src/features/tools/searchQuery.test.ts create mode 100644 packages/web/src/features/tools/searchQuery.ts create mode 100644 packages/web/src/features/tools/searchTools.test.ts diff --git a/packages/web/src/features/tools/glob.ts b/packages/web/src/features/tools/glob.ts index 6e855a32c..bfcfb3f26 100644 --- a/packages/web/src/features/tools/glob.ts +++ b/packages/web/src/features/tools/glob.ts @@ -1,22 +1,17 @@ import { z } from "zod"; -import globToRegexp from "glob-to-regexp"; import { isServiceError } from "@/lib/utils"; import { search } from "@/features/search"; -import escapeStringRegexp from "escape-string-regexp"; +import type { QueryIR } from "@/features/search/ir"; import { Source, ToolDefinition } from "./types"; import { logger } from "./logger"; import description from "./glob.txt"; import { CodeHostType } from "@sourcebot/db"; import { getRepoInfoByName } from "@/actions"; +import { buildGlobSearchQuery } from "./searchQuery"; const DEFAULT_LIMIT = 100; const TRUNCATION_MESSAGE = `(Results truncated. Consider using a more specific pattern, specifying a repo, or increasing the limit.)`; -function globToFileRegexp(glob: string): string { - const re = globToRegexp(glob, { extended: true, globstar: true }); - return re.source.replace(/^\^/, ''); -} - const globShape = { pattern: z .string() @@ -55,7 +50,7 @@ export type GlobRepoInfo = { export type GlobMetadata = { files: GlobFile[]; pattern: string; - query: string; + query: QueryIR; fileCount: number; repoCount: number; repoInfoMap: Record; @@ -81,30 +76,20 @@ export const globDefinition: ToolDefinition<'glob', typeof globShape, GlobMetada logger.debug('glob', { pattern, repo, ref, path, limit }); - let query = `file:${globToFileRegexp(pattern)}`; - - if (path) { - query += ` file:${escapeStringRegexp(path)}`; - } - - if (repo) { - query += ` repo:${escapeStringRegexp(repo)}`; - } else if (context.selectedRepos && context.selectedRepos.length > 0) { - query += ` reposet:${context.selectedRepos.join(',')}`; - } - - if (ref) { - query += ` rev:${ref}`; - } + const query = buildGlobSearchQuery({ + pattern, + path, + repo, + ref, + selectedRepos: context.selectedRepos, + }); const response = await search({ - queryType: 'string', + queryType: 'ir', query, options: { matches: limit, contextLines: 0, - isCaseSensitivityEnabled: true, - isRegexEnabled: true, }, source: context.source, }); diff --git a/packages/web/src/features/tools/grep.ts b/packages/web/src/features/tools/grep.ts index 26b78afde..9bea9362a 100644 --- a/packages/web/src/features/tools/grep.ts +++ b/packages/web/src/features/tools/grep.ts @@ -1,13 +1,13 @@ import { z } from "zod"; -import globToRegexp from "glob-to-regexp"; import { isServiceError } from "@/lib/utils"; import { search } from "@/features/search"; -import escapeStringRegexp from "escape-string-regexp"; +import type { QueryIR } from "@/features/search/ir"; import { Source, ToolDefinition } from "./types"; import { logger } from "./logger"; import description from "./grep.txt"; import { CodeHostType } from "@sourcebot/db"; import { getRepoInfoByName } from "@/actions"; +import { buildGrepSearchQuery } from "./searchQuery"; const DEFAULT_LIMIT = 100; const DEFAULT_GROUP_BY_REPO_LIMIT = 10_000; @@ -15,11 +15,6 @@ const MAX_LINE_LENGTH = 2000; const MAX_LINE_SUFFIX = `... (line truncated to ${MAX_LINE_LENGTH} chars)`; const TRUNCATION_MESSAGE = `(Results truncated. Consider using a more specific path or pattern, specifying a repo, or increasing the limit.)`; -function globToFileRegexp(glob: string): string { - const re = globToRegexp(glob, { extended: true, globstar: true }); - return re.source.replace(/^\^/, ''); -} - const grepShape = { pattern: z .string() @@ -66,7 +61,7 @@ export type GrepRepoInfo = { export type GrepMetadata = { files: GrepFile[]; pattern: string; - query: string; + query: QueryIR; matchCount: number; repoCount: number; repoInfoMap: Record; @@ -95,35 +90,21 @@ export const grepDefinition: ToolDefinition<'grep', typeof grepShape, GrepMetada logger.debug('grep', { pattern, path, include, repo, ref, limit, groupByRepo }); - const quotedPattern = `"${pattern.replace(/"/g, '\\"')}"`; - let query = quotedPattern; - - if (path) { - query += ` file:${escapeStringRegexp(path)}`; - } - - if (include) { - query += ` file:${globToFileRegexp(include)}`; - } - - if (repo) { - query += ` repo:${escapeStringRegexp(repo)}`; - } else if (context.selectedRepos && context.selectedRepos.length > 0) { - query += ` reposet:${context.selectedRepos.join(',')}`; - } - - if (ref) { - query += ` rev:${ref}`; - } + const query = buildGrepSearchQuery({ + pattern, + path, + include, + repo, + ref, + selectedRepos: context.selectedRepos, + }); const response = await search({ - queryType: 'string', + queryType: 'ir', query, options: { matches: limit, contextLines: 0, - isCaseSensitivityEnabled: true, - isRegexEnabled: true, }, source: context.source, }); diff --git a/packages/web/src/features/tools/searchQuery.test.ts b/packages/web/src/features/tools/searchQuery.test.ts new file mode 100644 index 000000000..e60ad03ae --- /dev/null +++ b/packages/web/src/features/tools/searchQuery.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest'; +import { buildGlobSearchQuery, buildGrepSearchQuery } from './searchQuery'; + +describe('buildGrepSearchQuery', () => { + it('preserves spaces and quotes in structured search inputs', () => { + const query = buildGrepSearchQuery({ + pattern: 'call("hello world")', + path: 'src/my dir', + include: 'My Tests/**/*.test.ts', + repo: 'Acme, Platform', + ref: 'feature/my feature', + }); + + expect(query).toEqual({ + and: { + children: [ + { + regexp: { + regexp: 'call("hello world")', + case_sensitive: true, + file_name: false, + content: true, + }, + query: 'regexp', + }, + { + regexp: { + regexp: 'src/my dir', + case_sensitive: true, + file_name: true, + content: false, + }, + query: 'regexp', + }, + { + regexp: { + regexp: 'My Tests\\/((?:[^/]*(?:\\/|$))*)([^/]*)\\.test\\.ts$', + case_sensitive: true, + file_name: true, + content: false, + }, + query: 'regexp', + }, + { + repo_set: { + set: { 'Acme, Platform': true }, + }, + query: 'repo_set', + }, + { + branch: { + pattern: 'feature/my feature', + exact: false, + }, + query: 'branch', + }, + ], + }, + query: 'and', + }); + }); + + it('uses exact selected repository names when no explicit repo is provided', () => { + const query = buildGrepSearchQuery({ + pattern: 'needle', + selectedRepos: ['Repo, One', 'Repo Two'], + }); + + expect(query).toMatchObject({ + and: { + children: [ + { regexp: { regexp: 'needle' } }, + { + repo_set: { + set: { + 'Repo, One': true, + 'Repo Two': true, + }, + }, + }, + ], + }, + }); + }); + + it('keeps a bare content search as a single IR node', () => { + expect(buildGrepSearchQuery({ pattern: 'needle' })).toEqual({ + regexp: { + regexp: 'needle', + case_sensitive: true, + file_name: false, + content: true, + }, + query: 'regexp', + }); + }); +}); + +describe('buildGlobSearchQuery', () => { + it('keeps a glob containing spaces in one file predicate', () => { + const query = buildGlobSearchQuery({ + pattern: 'My Folder/**/*.ts', + path: 'packages/my-dir/[legacy] (copy)+', + }); + + expect(query).toEqual({ + and: { + children: [ + { + regexp: { + regexp: 'My Folder\\/((?:[^/]*(?:\\/|$))*)([^/]*)\\.ts$', + case_sensitive: true, + file_name: true, + content: false, + }, + query: 'regexp', + }, + { + regexp: { + regexp: 'packages/my-dir/\\[legacy\\] \\(copy\\)\\+', + case_sensitive: true, + file_name: true, + content: false, + }, + query: 'regexp', + }, + ], + }, + query: 'and', + }); + }); + + it('preserves the rev:* behavior for searching every branch', () => { + const query = buildGlobSearchQuery({ + pattern: '*.ts', + ref: '*', + }); + + expect(query).toMatchObject({ + and: { + children: [ + { regexp: { file_name: true } }, + { branch: { pattern: '', exact: false } }, + ], + }, + }); + }); +}); diff --git a/packages/web/src/features/tools/searchQuery.ts b/packages/web/src/features/tools/searchQuery.ts new file mode 100644 index 000000000..4446c3328 --- /dev/null +++ b/packages/web/src/features/tools/searchQuery.ts @@ -0,0 +1,128 @@ +import globToRegexp from "glob-to-regexp"; +import type { QueryIR } from "@/features/search/ir"; + +type SearchScope = { + repo?: string; + ref?: string; + selectedRepos?: string[]; +}; + +type GrepSearchQuery = SearchScope & { + pattern: string; + path?: string; + include?: string; +}; + +type GlobSearchQuery = SearchScope & { + pattern: string; + path?: string; +}; + +const createRegexpQuery = ({ + regexp, + fileName, + content, +}: { + regexp: string; + fileName: boolean; + content: boolean; +}): QueryIR => ({ + regexp: { + regexp, + case_sensitive: true, + file_name: fileName, + content, + }, + query: "regexp", +}); + +// Keep literal tool inputs compatible with Zoekt's RE2 parser. The generic +// escape-string-regexp package emits hex escapes for some characters. +const escapeRE2Regexp = (value: string): string => value.replace(/[\\.^$|?*+()[\]{}]/g, '\\$&'); + +const globToFileRegexp = (glob: string): string => { + const regexp = globToRegexp(glob, { extended: true, globstar: true }); + return regexp.source.replace(/^\^/, ''); +}; + +const createRepoScopeQuery = ({ repo, selectedRepos }: SearchScope): QueryIR | undefined => { + const repos = repo ? [repo] : selectedRepos; + if (!repos || repos.length === 0) { + return undefined; + } + + return { + repo_set: { + set: Object.fromEntries(repos.map((repoName) => [repoName, true])), + }, + query: "repo_set", + }; +}; + +const createBranchQuery = (ref?: string): QueryIR | undefined => ref ? { + branch: { + pattern: ref === '*' ? '' : ref, + exact: false, + }, + query: "branch", +} : undefined; + +const combineQueries = (queries: Array): QueryIR => { + const children = queries.filter((query): query is QueryIR => query !== undefined); + if (children.length === 1) { + return children[0]; + } + + return { + and: { children }, + query: "and", + }; +}; + +export const buildGrepSearchQuery = ({ + pattern, + path, + include, + repo, + ref, + selectedRepos, +}: GrepSearchQuery): QueryIR => combineQueries([ + createRegexpQuery({ + regexp: pattern, + fileName: false, + content: true, + }), + path ? createRegexpQuery({ + regexp: escapeRE2Regexp(path), + fileName: true, + content: false, + }) : undefined, + include ? createRegexpQuery({ + regexp: globToFileRegexp(include), + fileName: true, + content: false, + }) : undefined, + createRepoScopeQuery({ repo, selectedRepos }), + createBranchQuery(ref), +]); + +export const buildGlobSearchQuery = ({ + pattern, + path, + repo, + ref, + selectedRepos, +}: GlobSearchQuery): QueryIR => combineQueries([ + createRegexpQuery({ + regexp: globToFileRegexp(pattern), + fileName: true, + content: false, + }), + path ? createRegexpQuery({ + regexp: escapeRE2Regexp(path), + fileName: true, + content: false, + }) : undefined, + createRepoScopeQuery({ repo, selectedRepos }), + createBranchQuery(ref), +]); diff --git a/packages/web/src/features/tools/searchTools.test.ts b/packages/web/src/features/tools/searchTools.test.ts new file mode 100644 index 000000000..fb572ed7d --- /dev/null +++ b/packages/web/src/features/tools/searchTools.test.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + search: vi.fn(), + getRepoInfoByName: vi.fn(), +})); + +vi.mock('@/features/search', () => ({ + search: mocks.search, +})); + +vi.mock('@/actions', () => ({ + getRepoInfoByName: mocks.getRepoInfoByName, +})); + +vi.mock('@/lib/utils', () => ({ + isServiceError: () => false, +})); + +vi.mock('./logger', () => ({ + logger: { debug: vi.fn() }, +})); + +import { globDefinition } from './glob'; +import { grepDefinition } from './grep'; +import { buildGlobSearchQuery, buildGrepSearchQuery } from './searchQuery'; + +const emptySearchResponse = { + files: [], + repositoryInfo: [], + stats: { actualMatchCount: 0 }, + isSearchExhaustive: true, +}; + +describe('agent search tools', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.search.mockResolvedValue(emptySearchResponse); + }); + + it('executes grep with QueryIR', async () => { + await grepDefinition.execute({ + pattern: 'needle', + path: 'src/my dir', + limit: 25, + }, { + source: 'test', + selectedRepos: ['Repo One'], + }); + + expect(mocks.search).toHaveBeenCalledWith({ + queryType: 'ir', + query: buildGrepSearchQuery({ + pattern: 'needle', + path: 'src/my dir', + selectedRepos: ['Repo One'], + }), + options: { + matches: 25, + contextLines: 0, + }, + source: 'test', + }); + }); + + it('executes glob with QueryIR', async () => { + await globDefinition.execute({ + pattern: 'My Folder/**/*.ts', + ref: 'feature/my feature', + }, { + source: 'test', + selectedRepos: ['Repo One'], + }); + + expect(mocks.search).toHaveBeenCalledWith({ + queryType: 'ir', + query: buildGlobSearchQuery({ + pattern: 'My Folder/**/*.ts', + ref: 'feature/my feature', + selectedRepos: ['Repo One'], + }), + options: { + matches: 100, + contextLines: 0, + }, + source: 'test', + }); + }); +}); From e0995b9baddfed69d01d7a2e0976cf4d91fa9041 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Wed, 12 Aug 2026 13:36:24 -0700 Subject: [PATCH 2/4] chore: update changelog for #1573 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b90bc77a..cce0960e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed the web process being capped at a ~4GiB heap regardless of how much memory the container has, which caused multi-second garbage collection pauses on larger deployments. [#1569](https://github.com/sourcebot-dev/sourcebot/pull/1569) - Upgraded `@sentry/*` to `^10.70.0`, fixing memory leaks where spans retained request data indefinitely. [#1572](https://github.com/sourcebot-dev/sourcebot/pull/1572) +- Fixed the `grep` and `glob` agent tools mis-parsing structured search inputs containing spaces, commas, or quotes. [#1573](https://github.com/sourcebot-dev/sourcebot/pull/1573) ## [5.1.6] - 2026-08-10 From 2cd2edf354548ed0282cb6485e0d0ecdbf986982 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Wed, 12 Aug 2026 19:35:55 -0700 Subject: [PATCH 3/4] fix(web): omit search query from tool metadata --- packages/web/src/features/tools/glob.ts | 3 --- packages/web/src/features/tools/grep.ts | 3 --- packages/web/src/features/tools/searchTools.test.ts | 6 ++++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/web/src/features/tools/glob.ts b/packages/web/src/features/tools/glob.ts index bfcfb3f26..65b034539 100644 --- a/packages/web/src/features/tools/glob.ts +++ b/packages/web/src/features/tools/glob.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { isServiceError } from "@/lib/utils"; import { search } from "@/features/search"; -import type { QueryIR } from "@/features/search/ir"; import { Source, ToolDefinition } from "./types"; import { logger } from "./logger"; import description from "./glob.txt"; @@ -50,7 +49,6 @@ export type GlobRepoInfo = { export type GlobMetadata = { files: GlobFile[]; pattern: string; - query: QueryIR; fileCount: number; repoCount: number; repoInfoMap: Record; @@ -129,7 +127,6 @@ export const globDefinition: ToolDefinition<'glob', typeof globShape, GlobMetada const metadata: GlobMetadata = { files, pattern, - query, fileCount: files.length, repoCount: new Set(files.map((f) => f.repo)).size, repoInfoMap, diff --git a/packages/web/src/features/tools/grep.ts b/packages/web/src/features/tools/grep.ts index 9bea9362a..eda4acab8 100644 --- a/packages/web/src/features/tools/grep.ts +++ b/packages/web/src/features/tools/grep.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { isServiceError } from "@/lib/utils"; import { search } from "@/features/search"; -import type { QueryIR } from "@/features/search/ir"; import { Source, ToolDefinition } from "./types"; import { logger } from "./logger"; import description from "./grep.txt"; @@ -61,7 +60,6 @@ export type GrepRepoInfo = { export type GrepMetadata = { files: GrepFile[]; pattern: string; - query: QueryIR; matchCount: number; repoCount: number; repoInfoMap: Record; @@ -142,7 +140,6 @@ export const grepDefinition: ToolDefinition<'grep', typeof grepShape, GrepMetada const metadata: GrepMetadata = { files, pattern, - query, matchCount: response.stats.actualMatchCount, repoCount: new Set(files.map((f) => f.repo)).size, repoInfoMap, diff --git a/packages/web/src/features/tools/searchTools.test.ts b/packages/web/src/features/tools/searchTools.test.ts index fb572ed7d..fcb3588df 100644 --- a/packages/web/src/features/tools/searchTools.test.ts +++ b/packages/web/src/features/tools/searchTools.test.ts @@ -39,7 +39,7 @@ describe('agent search tools', () => { }); it('executes grep with QueryIR', async () => { - await grepDefinition.execute({ + const result = await grepDefinition.execute({ pattern: 'needle', path: 'src/my dir', limit: 25, @@ -61,10 +61,11 @@ describe('agent search tools', () => { }, source: 'test', }); + expect(result.metadata).not.toHaveProperty('query'); }); it('executes glob with QueryIR', async () => { - await globDefinition.execute({ + const result = await globDefinition.execute({ pattern: 'My Folder/**/*.ts', ref: 'feature/my feature', }, { @@ -85,5 +86,6 @@ describe('agent search tools', () => { }, source: 'test', }); + expect(result.metadata).not.toHaveProperty('query'); }); }); From 927d1ddb2a73ff97940b8ad44cda8582b2fe1194 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Wed, 12 Aug 2026 21:34:11 -0700 Subject: [PATCH 4/4] test: split search tool tests --- .../{searchTools.test.ts => glob.test.ts} | 33 +--------- packages/web/src/features/tools/grep.test.ts | 65 +++++++++++++++++++ 2 files changed, 68 insertions(+), 30 deletions(-) rename packages/web/src/features/tools/{searchTools.test.ts => glob.test.ts} (60%) create mode 100644 packages/web/src/features/tools/grep.test.ts diff --git a/packages/web/src/features/tools/searchTools.test.ts b/packages/web/src/features/tools/glob.test.ts similarity index 60% rename from packages/web/src/features/tools/searchTools.test.ts rename to packages/web/src/features/tools/glob.test.ts index fcb3588df..16ddff134 100644 --- a/packages/web/src/features/tools/searchTools.test.ts +++ b/packages/web/src/features/tools/glob.test.ts @@ -22,8 +22,7 @@ vi.mock('./logger', () => ({ })); import { globDefinition } from './glob'; -import { grepDefinition } from './grep'; -import { buildGlobSearchQuery, buildGrepSearchQuery } from './searchQuery'; +import { buildGlobSearchQuery } from './searchQuery'; const emptySearchResponse = { files: [], @@ -32,39 +31,13 @@ const emptySearchResponse = { isSearchExhaustive: true, }; -describe('agent search tools', () => { +describe('glob', () => { beforeEach(() => { vi.clearAllMocks(); mocks.search.mockResolvedValue(emptySearchResponse); }); - it('executes grep with QueryIR', async () => { - const result = await grepDefinition.execute({ - pattern: 'needle', - path: 'src/my dir', - limit: 25, - }, { - source: 'test', - selectedRepos: ['Repo One'], - }); - - expect(mocks.search).toHaveBeenCalledWith({ - queryType: 'ir', - query: buildGrepSearchQuery({ - pattern: 'needle', - path: 'src/my dir', - selectedRepos: ['Repo One'], - }), - options: { - matches: 25, - contextLines: 0, - }, - source: 'test', - }); - expect(result.metadata).not.toHaveProperty('query'); - }); - - it('executes glob with QueryIR', async () => { + it('executes with QueryIR', async () => { const result = await globDefinition.execute({ pattern: 'My Folder/**/*.ts', ref: 'feature/my feature', diff --git a/packages/web/src/features/tools/grep.test.ts b/packages/web/src/features/tools/grep.test.ts new file mode 100644 index 000000000..74bc04fe5 --- /dev/null +++ b/packages/web/src/features/tools/grep.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + search: vi.fn(), + getRepoInfoByName: vi.fn(), +})); + +vi.mock('@/features/search', () => ({ + search: mocks.search, +})); + +vi.mock('@/actions', () => ({ + getRepoInfoByName: mocks.getRepoInfoByName, +})); + +vi.mock('@/lib/utils', () => ({ + isServiceError: () => false, +})); + +vi.mock('./logger', () => ({ + logger: { debug: vi.fn() }, +})); + +import { grepDefinition } from './grep'; +import { buildGrepSearchQuery } from './searchQuery'; + +const emptySearchResponse = { + files: [], + repositoryInfo: [], + stats: { actualMatchCount: 0 }, + isSearchExhaustive: true, +}; + +describe('grep', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.search.mockResolvedValue(emptySearchResponse); + }); + + it('executes with QueryIR', async () => { + const result = await grepDefinition.execute({ + pattern: 'needle', + path: 'src/my dir', + limit: 25, + }, { + source: 'test', + selectedRepos: ['Repo One'], + }); + + expect(mocks.search).toHaveBeenCalledWith({ + queryType: 'ir', + query: buildGrepSearchQuery({ + pattern: 'needle', + path: 'src/my dir', + selectedRepos: ['Repo One'], + }), + options: { + matches: 25, + contextLines: 0, + }, + source: 'test', + }); + expect(result.metadata).not.toHaveProperty('query'); + }); +});