Skip to content

Commit 5d520ba

Browse files
fix(file-search): isolate and queue concurrent searches (#7952)
* fix(file-search): allow larger concurrent search bursts * fix(file-search): isolate and queue search transactions * fix(file-search): bound admission waits and propagate cancellation
1 parent 75156bc commit 5d520ba

14 files changed

Lines changed: 767 additions & 203 deletions
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/** @vitest-environment node */
2+
import { beforeEach, describe, expect, it, vi } from 'vitest'
3+
4+
const mocks = vi.hoisted(() => ({
5+
load: vi.fn(),
6+
permission: vi.fn(),
7+
search: vi.fn(),
8+
folders: vi.fn(),
9+
}))
10+
vi.mock('@sim/platform-authz/workspace', () => ({
11+
permissionSatisfies: (actual: string | null) => actual !== null,
12+
resolveEffectiveWorkspacePermission: mocks.permission,
13+
}))
14+
vi.mock('@/lib/uploads/contexts/workspace', () => ({ loadActiveWorkspaceContext: mocks.load }))
15+
vi.mock('@/lib/workspace-files/search/repository', () => ({
16+
searchWorkspaceFileIndex: mocks.search,
17+
}))
18+
vi.mock('@/lib/workspace-files/resolve-folder-scope', () => ({
19+
resolveWorkspaceFolderScope: mocks.folders,
20+
}))
21+
22+
import { searchWorkspaceFileContent } from '@/lib/workspace-files/application/search-workspace-file-content'
23+
24+
const principal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const
25+
const input = {
26+
workspaceId: 'workspace-1',
27+
query: 'needle',
28+
mode: 'exact',
29+
maxResults: 10,
30+
} as const
31+
32+
describe('searchWorkspaceFileContent cancellation', () => {
33+
beforeEach(() => {
34+
vi.clearAllMocks()
35+
mocks.load.mockResolvedValue({
36+
workspaceId: 'workspace-1',
37+
workspaceOrganizationId: null,
38+
allowPersonalApiKeys: true,
39+
billedAccountUserId: 'user-1',
40+
})
41+
mocks.permission.mockResolvedValue('read')
42+
mocks.search.mockResolvedValue({ results: [] })
43+
})
44+
45+
it.each(['request', 'input'] as const)(
46+
'propagates the %s signal through the authorized application operation',
47+
async (source) => {
48+
const controller = new AbortController()
49+
await searchWorkspaceFileContent.execute({
50+
principal,
51+
input: { ...input, ...(source === 'input' ? { signal: controller.signal } : {}) },
52+
request: {
53+
headers: new Headers(),
54+
...(source === 'request' ? { signal: controller.signal } : {}),
55+
},
56+
})
57+
expect(mocks.search).toHaveBeenCalledWith(
58+
expect.objectContaining({ signal: controller.signal })
59+
)
60+
}
61+
)
62+
63+
it('does not resolve folders or enqueue database work for a cancelled HTTP request', async () => {
64+
const signal = AbortSignal.abort(new Error('cancelled'))
65+
await expect(
66+
searchWorkspaceFileContent.execute({
67+
principal,
68+
input: { ...input, folderPaths: ['/notes'] },
69+
request: { headers: new Headers(), signal },
70+
})
71+
).rejects.toBe(signal.reason)
72+
expect(mocks.folders).not.toHaveBeenCalled()
73+
expect(mocks.search).not.toHaveBeenCalled()
74+
})
75+
})

apps/sim/lib/workspace-files/application/search-workspace-file-content.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@ import { loadActiveWorkspaceContext } from '@/lib/uploads/contexts/workspace'
33
import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case'
44
import { fileOperations } from '@/lib/workspace-files/application/operations'
55
import { resolveWorkspaceFolderScope } from '@/lib/workspace-files/resolve-folder-scope'
6+
import { WorkspaceFileSearchUnavailableError } from '@/lib/workspace-files/search/errors'
67
import {
78
compileFileSearchPattern,
89
type FileSearchMode,
910
FileSearchPatternError,
1011
} from '@/lib/workspace-files/search/pattern'
11-
import {
12-
searchWorkspaceFileIndex,
13-
WorkspaceFileSearchUnavailableError,
14-
} from '@/lib/workspace-files/search/repository'
12+
import { searchWorkspaceFileIndex } from '@/lib/workspace-files/search/repository'
1513

1614
export interface SearchWorkspaceFileContentInput {
1715
workspaceId: string
@@ -37,14 +35,16 @@ export const searchWorkspaceFileContent = defineAuthorizedWorkspaceFileUseCase({
3735
operation: fileOperations.searchContent,
3836
resolveContext: ({ input }: { input: SearchWorkspaceFileContentInput }) =>
3937
resolveSearchWorkspaceFileContext(input),
40-
execute: async ({ principal, input, context }) => {
41-
/*
38+
execute: async ({ principal, input, context, request }) => {
39+
const signal = input.signal ?? request?.signal
40+
signal?.throwIfAborted()
41+
/**
4242
* Resolved here rather than at the surface so every caller (the File
4343
* block, the v2 route) is confined by the same check. A folder tree
4444
* holding one subtree per user makes this scope the isolation boundary,
4545
* not a convenience filter.
4646
*/
47-
/*
47+
/**
4848
* `!== undefined`, not a length check: an explicitly empty list is a scope
4949
* that names no folder, which must match nothing. Treating it as "absent"
5050
* would answer a request for nothing with the whole workspace.
@@ -58,15 +58,15 @@ export const searchWorkspaceFileContent = defineAuthorizedWorkspaceFileUseCase({
5858
includeSubfolders: input.includeSubfolders,
5959
})
6060
: undefined
61-
input.signal?.throwIfAborted()
61+
signal?.throwIfAborted()
6262

6363
try {
6464
return await searchWorkspaceFileIndex({
6565
workspaceId: context.workspaceId,
6666
pattern: compileFileSearchPattern(input.query, input.mode),
6767
maxResults: input.maxResults,
6868
folderScope,
69-
signal: input.signal,
69+
signal,
7070
})
7171
} catch (error) {
7272
/**

apps/sim/lib/workspace-files/search/README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ Search joins the current file revision and resolved workspace/folder scope. A re
2222

2323
For regular chunks, PostgreSQL checks the pattern with newline-aware semantics, then verifies individual logical lines. Long-line fragments use only necessary three-character literals as a conservative prefilter, including all required alternation branches. Two-code-point overlap preserves those literals at every boundary. PostgreSQL reconstructs the complete candidate line and evaluates the original regex, so anchors, word boundaries, repetitions, and arbitrarily long match spans retain line semantics. Fixed overlap alone is never treated as proof of a match. The supported regex grammar and minimum literal requirement are unchanged.
2424

25-
Regular blocks are verified in batches of at most 16 (128 KiB of indexed text); long lines are reconstructed one at a time. Only bounded match-centered previews leave PostgreSQL: at most 201 rows to detect truncation, and at most 2 KiB per rendered result. A single search has a ten-second application deadline with per-statement guards. PostgreSQL 17 additionally enforces a total transaction timeout; PostgreSQL 16 uses the compatible idle-transaction guard. Transaction advisory locks admit at most two simultaneous searches per workspace and ten globally per database. Busy and timed-out searches fail explicitly; they never report an incomplete scan as an authoritative empty result. The reader uses the normal application database connection, so admission is coordinated on the same database as the index.
25+
Regular blocks are verified in batches of at most 16 (128 KiB of indexed text); long lines are reconstructed one at a time. Only bounded match-centered previews leave PostgreSQL: at most 201 rows to detect truncation, and at most 2 KiB per rendered result. A single search has a fifteen-second caller deadline covering queueing, connection acquisition, and execution; SQL runs for at most ten seconds within that budget, with per-statement guards. PostgreSQL 17 additionally enforces a total transaction timeout; PostgreSQL 16 uses the compatible idle-transaction guard. Transaction advisory locks admit at most 20 simultaneous searches per workspace and 5,000 globally per database. Search transactions use the dedicated `dbFor('search')` primary pool, with five connections per process, so they cannot occupy the application or execution client pools. Before acquiring a connection, a process admits five active searches and at most 100 waiting requests, capped at 20 waiting requests per workspace so one burst cannot fill the entire queue. Queued workspaces rotate after each grant; waiting requests expire after five seconds or leave immediately on cancellation. The local active budget comes from the same pool profile as the driver. A slot is released only when the transaction settles, including errors. Cancellation reaches this boundary from both HTTP requests and File-tool execution. The existing deadline helper ends the caller’s wait promptly even when connection acquisition stalls. A late transaction checks cancellation before running search SQL; its active slot remains reserved until the driver settles, so a timed-out request cannot start replacement work on top of a still-running query. The driver and upstream pooler still own physical connection cleanup; the application deadline does not cancel a queued PostgreSQL protocol command. Busy and timed-out searches fail explicitly; they never report an incomplete scan as an authoritative empty result. These bounds apply identically to exact and regex search and do not change result or line-number semantics.
26+
27+
The 20/workspace and 5,000/global advisory ceilings bound admitted transactions across processes; they are not promises of simultaneous execution or throughput. Local queues absorb short bursts without holding database connections. They are not durable jobs or a fleet-wide fair scheduler. The dedicated client pool isolates connection ownership, not PostgreSQL CPU, memory, I/O, or an upstream PgBouncer server pool. Its default URL is the process primary URL; any `DATABASE_URL_SEARCH` override must target the same primary database so current revisions and advisory admission remain coherent. Independent PgBouncer server budgets require separate database/user pool configuration. Total client connections can increase by five per participating process. Before raising execution capacity, measure the number of processes, backend pool budget, queue wait/rejection rates, search latency, and database resource headroom under representative exact and broad-regex workloads. More queueing cannot increase sustained throughput.
2628

2729
Arbitrary regex cannot have a fixed latency guarantee. Common terms, broad alternatives, and punctuation-only literals may require scanning significant scoped text. Larger capacity decisions need representative query plans and workload measurements; neither a per-file byte cap nor a PostgreSQL row-count claim establishes a total corpus capacity.
2830

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
/** @vitest-environment node */
2+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
3+
import { FileSearchAdmission } from '@/lib/workspace-files/search/admission'
4+
import { WorkspaceFileSearchUnavailableError } from '@/lib/workspace-files/search/errors'
5+
6+
describe('FileSearchAdmission', () => {
7+
beforeEach(() => vi.useFakeTimers())
8+
afterEach(() => vi.useRealTimers())
9+
10+
function createAdmission(concurrency = 1, maxPending = 3, maxPendingPerWorkspace = maxPending) {
11+
return new FileSearchAdmission({
12+
concurrency,
13+
maxPending,
14+
maxPendingPerWorkspace,
15+
timeoutMs: 5000,
16+
})
17+
}
18+
19+
it('queues a burst without exceeding the active budget and releases each slot once', async () => {
20+
const admission = createAdmission(2)
21+
const first = await admission.acquire('a')
22+
const second = await admission.acquire('a')
23+
const granted = vi.fn()
24+
const waiting = admission.acquire('a').then((release) => {
25+
granted()
26+
return release
27+
})
28+
await vi.advanceTimersByTimeAsync(1)
29+
expect(granted).not.toHaveBeenCalled()
30+
first()
31+
const third = await waiting
32+
first()
33+
const fourthGranted = vi.fn()
34+
const fourth = admission.acquire('b').then((release) => {
35+
fourthGranted()
36+
return release
37+
})
38+
await vi.advanceTimersByTimeAsync(1)
39+
expect(fourthGranted).not.toHaveBeenCalled()
40+
second()
41+
;(await fourth)()
42+
third()
43+
expect(vi.getTimerCount()).toBe(0)
44+
})
45+
46+
it('rotates waiting workspaces instead of draining one burst first', async () => {
47+
const admission = createAdmission()
48+
const first = await admission.acquire('a')
49+
const order: string[] = []
50+
const request = (workspace: string) =>
51+
admission.acquire(workspace).then((release) => {
52+
order.push(workspace)
53+
release()
54+
})
55+
const waiting = [request('a'), request('a'), request('b')]
56+
first()
57+
await Promise.all(waiting)
58+
expect(order).toEqual(['a', 'b', 'a'])
59+
})
60+
61+
it('rejects overflow and recovers when the queue drains', async () => {
62+
const admission = createAdmission(1, 1)
63+
const first = await admission.acquire('a')
64+
const waiting = admission.acquire('a')
65+
await expect(admission.acquire('b')).rejects.toBeInstanceOf(WorkspaceFileSearchUnavailableError)
66+
first()
67+
;(await waiting)()
68+
;(await admission.acquire('b'))()
69+
expect(vi.getTimerCount()).toBe(0)
70+
})
71+
72+
it('expires waiting requests without executing them or leaking queue capacity', async () => {
73+
const admission = createAdmission(1, 1)
74+
const first = await admission.acquire('a')
75+
const waiting = expect(admission.acquire('b')).rejects.toBeInstanceOf(
76+
WorkspaceFileSearchUnavailableError
77+
)
78+
await vi.advanceTimersByTimeAsync(5000)
79+
await waiting
80+
const next = admission.acquire('c')
81+
first()
82+
;(await next)()
83+
expect(vi.getTimerCount()).toBe(0)
84+
})
85+
86+
it('checks expiry when granting even if a busy event loop has delayed the timer', async () => {
87+
const admission = createAdmission()
88+
const first = await admission.acquire('a')
89+
const waiting = expect(admission.acquire('b')).rejects.toBeInstanceOf(
90+
WorkspaceFileSearchUnavailableError
91+
)
92+
vi.setSystemTime(Date.now() + 5000)
93+
first()
94+
await waiting
95+
;(await admission.acquire('c'))()
96+
expect(vi.getTimerCount()).toBe(0)
97+
})
98+
99+
it('removes cancelled waiters and their listeners without consuming a connection', async () => {
100+
const admission = createAdmission(1, 1)
101+
const first = await admission.acquire('a')
102+
const controller = new AbortController()
103+
const remove = vi.spyOn(controller.signal, 'removeEventListener')
104+
const reason = new Error('cancelled')
105+
const waiting = expect(admission.acquire('b', controller.signal)).rejects.toBe(reason)
106+
controller.abort(reason)
107+
await waiting
108+
expect(remove).toHaveBeenCalledWith('abort', expect.any(Function))
109+
expect(vi.getTimerCount()).toBe(0)
110+
const next = admission.acquire('c')
111+
first()
112+
;(await next)()
113+
})
114+
115+
it('rejects an already cancelled caller without occupying a slot', async () => {
116+
const admission = createAdmission()
117+
const reason = new Error('cancelled')
118+
await expect(admission.acquire('a', AbortSignal.abort(reason))).rejects.toBe(reason)
119+
;(await admission.acquire('b'))()
120+
})
121+
122+
it('does not recycle an active slot on cancellation until the database work settles', async () => {
123+
const admission = createAdmission()
124+
const controller = new AbortController()
125+
const first = await admission.acquire('a', controller.signal)
126+
controller.abort()
127+
const granted = vi.fn()
128+
const next = admission.acquire('b').then((release) => {
129+
granted()
130+
release()
131+
})
132+
await vi.advanceTimersByTimeAsync(1)
133+
expect(granted).not.toHaveBeenCalled()
134+
first()
135+
await next
136+
})
137+
it('leaves waiting capacity for another workspace when one burst hits its own cap', async () => {
138+
const admission = createAdmission(1, 4, 2)
139+
const release = await admission.acquire('hot')
140+
const hot = [admission.acquire('hot'), admission.acquire('hot')]
141+
await expect(admission.acquire('hot')).rejects.toBeInstanceOf(
142+
WorkspaceFileSearchUnavailableError
143+
)
144+
const quiet = admission.acquire('quiet')
145+
release()
146+
;(await hot[0])()
147+
;(await quiet)()
148+
;(await hot[1])()
149+
;(await admission.acquire('hot'))()
150+
expect(vi.getTimerCount()).toBe(0)
151+
})
152+
153+
it.each(['timeout', 'abort'] as const)(
154+
'keeps a stalled operation counted after caller %s until the operation settles',
155+
async (reason) => {
156+
const admission = createAdmission()
157+
const controller = new AbortController()
158+
const dependency = Promise.withResolvers<void>()
159+
const lateWork = vi.fn()
160+
const running = admission.run(
161+
'a',
162+
async (signal) => {
163+
await dependency.promise
164+
signal.throwIfAborted()
165+
lateWork()
166+
},
167+
controller.signal
168+
)
169+
const rejected = expect(running).rejects.toThrow(
170+
reason === 'timeout' ? /timed out/ : /cancelled/
171+
)
172+
await vi.advanceTimersByTimeAsync(1)
173+
if (reason === 'timeout') await vi.advanceTimersByTimeAsync(15000)
174+
else controller.abort(new Error('cancelled'))
175+
await rejected
176+
const nextWork = vi.fn().mockResolvedValue('done')
177+
const next = admission.run('b', nextWork)
178+
await vi.advanceTimersByTimeAsync(1)
179+
expect(nextWork).not.toHaveBeenCalled()
180+
dependency.resolve()
181+
await expect(next).resolves.toBe('done')
182+
expect(lateWork).not.toHaveBeenCalled()
183+
expect(vi.getTimerCount()).toBe(0)
184+
}
185+
)
186+
187+
it('releases a lease after an operation throws synchronously', async () => {
188+
const admission = createAdmission()
189+
await expect(
190+
admission.run('a', () => {
191+
throw new Error('operation failed')
192+
})
193+
).rejects.toThrow('operation failed')
194+
await expect(admission.run('b', async () => 'done')).resolves.toBe('done')
195+
expect(vi.getTimerCount()).toBe(0)
196+
})
197+
198+
it('does not start an operation cancelled while waiting', async () => {
199+
const admission = createAdmission()
200+
const release = await admission.acquire('a')
201+
const controller = new AbortController()
202+
const operation = vi.fn()
203+
const waiting = expect(admission.run('b', operation, controller.signal)).rejects.toThrow(
204+
'cancelled'
205+
)
206+
controller.abort(new Error('cancelled'))
207+
await waiting
208+
release()
209+
expect(operation).not.toHaveBeenCalled()
210+
expect(vi.getTimerCount()).toBe(0)
211+
})
212+
})

0 commit comments

Comments
 (0)