Skip to content

Commit 781c896

Browse files
authored
improvement(agent-cli): skip non-text files in grep instead of marking the search incomplete (#9635)
1 parent 36e1735 commit 781c896

4 files changed

Lines changed: 246 additions & 58 deletions

File tree

‎apps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,121 @@ describe('search authority and completeness', () => {
524524
expect(result.stderr).toContain('broken')
525525
})
526526

527+
it('skips files with no text extractor without fetching them or failing the search', async () => {
528+
const requested: string[] = []
529+
const result = await runEngine(
530+
'grep',
531+
['needle'],
532+
runtimeWith(
533+
{
534+
'/api/v2/files': {
535+
data: [
536+
{ id: 'notes', name: 'notes.md', type: 'text/markdown' },
537+
{ id: 'report', name: 'report.pdf', type: 'application/pdf' },
538+
{ id: 'font', name: 'brand.woff2', type: 'font/woff2' },
539+
{ id: 'clip', name: 'demo.mp4', type: 'video/mp4' },
540+
],
541+
nextCursor: null,
542+
},
543+
'/api/v2/files/notes/text': { data: { text: 'needle', degraded: false } },
544+
'/api/v2/files/report/text': { data: { text: 'a needle', degraded: false } },
545+
},
546+
requested
547+
),
548+
{ scope: 'files' }
549+
)
550+
expect(requested.filter((path) => path !== '/api/v2/files').sort()).toEqual([
551+
'/api/v2/files/notes/text',
552+
'/api/v2/files/report/text',
553+
])
554+
expect(result.exitCode).toBe(0)
555+
expect(result.stdout).toBe(
556+
'files/notes.md (notes):1: needle\nfiles/report.pdf (report):1: a needle'
557+
)
558+
expect(result.stderr).toBe(
559+
'Skipped 2 files with no searchable text (images, media, archives, and other binary formats).'
560+
)
561+
})
562+
563+
it('keeps real read failures incomplete beside skipped non-text files', async () => {
564+
const result = await runEngine(
565+
'grep',
566+
['needle'],
567+
runtimeWith({
568+
'/api/v2/files': {
569+
data: [
570+
{ id: 'broken', name: 'broken.txt', type: 'text/plain' },
571+
{ id: 'logo', name: 'logo.png', type: 'image/png' },
572+
],
573+
nextCursor: null,
574+
},
575+
}),
576+
{ scope: 'files' }
577+
)
578+
expect(result.exitCode).toBe(1)
579+
expect(result.stderr).toMatch(/^Search incomplete \(1 issue\)/)
580+
expect(result.stderr).toContain('files/broken.txt (broken): could not read text')
581+
expect(result.stderr).toContain('Skipped 1 file with no searchable text')
582+
})
583+
584+
it('names a selected non-text file as searched-and-skipped, not an unknown selector', async () => {
585+
const requested: string[] = []
586+
const result = await runEngine(
587+
'grep',
588+
['needle'],
589+
runtimeWith(
590+
{
591+
'/api/v2/files': {
592+
data: [{ id: 'logo', name: 'logo.png', type: 'image/png' }],
593+
nextCursor: null,
594+
},
595+
},
596+
requested
597+
),
598+
{ scope: 'files', in: 'files/logo' }
599+
)
600+
expect(requested).toEqual(['/api/v2/files'])
601+
expect(result.exitCode).toBe(0)
602+
expect(result.stdout).toContain('No matches')
603+
expect(result.stderr).toContain('Skipped 1 file with no searchable text')
604+
})
605+
606+
it('spends the file-read limit only on files with text', async () => {
607+
const files = [
608+
...Array.from({ length: 300 }, (_, i) => ({
609+
id: `img${i}`,
610+
name: `img${i}.png`,
611+
type: 'image/png',
612+
})),
613+
{ id: 'notes', name: 'notes.txt', type: 'text/plain' },
614+
]
615+
const result = await runEngine(
616+
'grep',
617+
['needle'],
618+
{
619+
...runtimeWith({}),
620+
client: {
621+
request: async <T>(
622+
path: string,
623+
options?: { query?: Record<string, string> }
624+
): Promise<T> => {
625+
if (path === '/api/v2/files') {
626+
const offset = Number(options?.query?.cursor ?? 0)
627+
return {
628+
data: files.slice(offset, offset + 100),
629+
nextCursor: offset + 100 < files.length ? String(offset + 100) : null,
630+
} as T
631+
}
632+
if (path === '/api/v2/files/notes/text') return { data: { text: 'needle' } } as T
633+
throw new Error(`Unexpected request: ${path}`)
634+
},
635+
},
636+
},
637+
{ scope: 'files', count: true }
638+
)
639+
expect(result).toMatchObject({ exitCode: 0, stdout: '1 (files=1)' })
640+
})
641+
527642
it('reports truncated extraction even when a returned prefix has a match', async () => {
528643
const result = await runEngine(
529644
'grep',

‎apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts‎

Lines changed: 56 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ import {
1111
type AgentCliFlags,
1212
type AgentCliRuntime,
1313
agentCliFail,
14-
agentCliOk,
1514
} from '@/lib/mothership/agent-cli/types'
1615
import { buildIntegrationToolSchemas } from '@/lib/mothership/chat/payload'
16+
import { workspaceFileTextFormat } from '@/lib/workspace-files/text-format'
1717
import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection'
1818
import { OPAQUE_RESOLVED_SECRET_REPLACEMENT } from '@/executor/utils/resolved-secret-matcher'
1919
import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
@@ -61,6 +61,7 @@ const PLATFORM_SCOPES: ReadonlySet<Scope> = new Set<Scope>(['blocks', 'tools'])
6161

6262
interface GrepRuntime extends AgentCliRuntime {
6363
reportIncomplete(message: string): void
64+
reportNonText(): void
6465
}
6566

6667
/**
@@ -370,22 +371,44 @@ const FETCHERS: Record<
370371
credentials: async (_runtime, item) => render('credentials', item.id, item.label, item.raw),
371372
}
372373

373-
function concurrencyFor(scope: Scope): number {
374-
return scope === 'files' ? FILE_READ_CONCURRENCY : FETCH_CONCURRENCY
374+
/**
375+
* A listed file the agent's text read has no extractor for (fonts, media, archives, images), by
376+
* the same gate that read applies. A listing without a name cannot say, so that file is read.
377+
*/
378+
function lacksText(file: unknown): boolean {
379+
if (!isRecordLike(file)) return false
380+
const name = str(file.name)
381+
return name !== undefined && !workspaceFileTextFormat({ name, type: str(file.type) ?? '' })
375382
}
376383

377384
async function fetchAll(
378385
runtime: GrepRuntime,
379386
scope: Scope,
380387
entries: IndexEntry[]
381388
): Promise<GrepMaterialized[]> {
382-
if (scope === 'files' && entries.length > MAX_GREP_FILES) {
389+
if (scope !== 'files') {
390+
return mapConcurrent(entries, FETCH_CONCURRENCY, (item) => FETCHERS[scope](runtime, item))
391+
}
392+
// Skipped files keep their identity, so `--in` naming one is not an unknown selector.
393+
const skipped: GrepMaterialized[] = []
394+
const readable: IndexEntry[] = []
395+
for (const item of entries) {
396+
if (lacksText(item.raw)) {
397+
runtime.reportNonText()
398+
skipped.push({ scope, id: item.id, label: item.label, text: null })
399+
} else readable.push(item)
400+
}
401+
if (readable.length > MAX_GREP_FILES) {
383402
runtime.reportIncomplete(
384-
`files: searched the first ${MAX_GREP_FILES} of ${entries.length} selected files; narrow with --in files/<name-or-id>.`
403+
`files: searched the first ${MAX_GREP_FILES} of ${readable.length} selected files; narrow with --in files/<name-or-id>.`
385404
)
386405
}
387-
const selected = scope === 'files' ? entries.slice(0, MAX_GREP_FILES) : entries
388-
return mapConcurrent(selected, concurrencyFor(scope), (item) => FETCHERS[scope](runtime, item))
406+
const read = await mapConcurrent(
407+
readable.slice(0, MAX_GREP_FILES),
408+
FILE_READ_CONCURRENCY,
409+
(item) => FETCHERS.files(runtime, item)
410+
)
411+
return [...read, ...skipped]
389412
}
390413

391414
/** A whole world, for a search with no `--in`: every resource the index lists. */
@@ -566,10 +589,15 @@ export interface GrepSource {
566589
registry?: AgentCliRuntime['resolvedSecretTraceRegistry']
567590
/** The only scopes this corpus holds, and the refusal for a search that names another. */
568591
scopeLimit?: { scopes: readonly Scope[]; refusal: string }
592+
/**
593+
* `reportNonText` counts a file skipped unread because it has no text to extract: a note,
594+
* not an incomplete search. The skipped file is still returned with null text.
595+
*/
569596
materialize(
570597
scopes: Scope[],
571598
nameFilter: string | undefined,
572-
reportIncomplete: (message: string) => void
599+
reportIncomplete: (message: string) => void,
600+
reportNonText: () => void
573601
): Promise<GrepMaterialized[]>
574602
}
575603

@@ -624,15 +652,24 @@ export async function executeGrep(positionals: string[], flags: AgentCliFlags, s
624652
issueCount++
625653
if (issues.length < 10) issues.push(clip(message))
626654
}
627-
const finish = (stdout: string) =>
628-
issueCount === 0
629-
? agentCliOk(stdout)
630-
: {
631-
exitCode: 1,
632-
stdout,
633-
stderr: `Search incomplete (${issueCount} issue${issueCount === 1 ? '' : 's'}); matches and counts cover only the content read.\n${issues.join('\n')}${issueCount > issues.length ? `\n${issueCount - issues.length} further issues omitted.` : ''}`,
634-
}
635-
const candidates = await source.materialize(searched, nameFilter, reportIncomplete)
655+
let nonTextCount = 0
656+
const reportNonText = () => {
657+
nonTextCount++
658+
}
659+
const finish = (stdout: string) => {
660+
const skipped = nonTextCount
661+
? `Skipped ${nonTextCount} file${nonTextCount === 1 ? '' : 's'} with no searchable text (images, media, archives, and other binary formats).`
662+
: ''
663+
const incomplete =
664+
issueCount &&
665+
`Search incomplete (${issueCount} issue${issueCount === 1 ? '' : 's'}); matches and counts cover only the content read.\n${issues.join('\n')}${issueCount > issues.length ? `\n${issueCount - issues.length} further issues omitted.` : ''}`
666+
return {
667+
exitCode: issueCount ? 1 : 0,
668+
stdout,
669+
stderr: [incomplete, skipped].filter(Boolean).join('\n'),
670+
}
671+
}
672+
const candidates = await source.materialize(searched, nameFilter, reportIncomplete, reportNonText)
636673
source.signal?.throwIfAborted()
637674
/**
638675
* A resource nothing in the searched worlds answers to is a wrong selector, not a
@@ -721,8 +758,8 @@ export const universalGrepCommand: AgentCliEngine = {
721758
executeGrep(positionals, flags, {
722759
signal: runtime.signal,
723760
registry: runtime.resolvedSecretTraceRegistry,
724-
async materialize(scopes, nameFilter, reportIncomplete) {
725-
const bounded = gatedRuntime({ ...runtime, reportIncomplete })
761+
async materialize(scopes, nameFilter, reportIncomplete, reportNonText) {
762+
const bounded = gatedRuntime({ ...runtime, reportIncomplete, reportNonText })
726763
return nameFilter
727764
? materializeWithin(bounded, scopes, nameFilter)
728765
: (await settleAll(scopes.map((scope) => materializeScope(bounded, scope)))).flat()

‎apps/sim/lib/mothership/agent-cli/project-file-grep.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,23 @@ describe('Project grep corpus', () => {
105105
expect(list).toHaveBeenCalledTimes(1)
106106
expect(result.stderr).toContain(`searched the first ${MAX_GREP_FILES} selected files`)
107107
})
108+
it('skips a file with no text extractor without reading it or failing the search', async () => {
109+
list.mockResolvedValue({
110+
files: [
111+
{ id: 'file', name: 'notes.txt', type: 'text/plain', folderPath: '/' },
112+
{ id: 'clip', name: 'notes-demo.mp4', type: 'video/mp4', folderPath: '/' },
113+
],
114+
nextKeys: null,
115+
})
116+
const result = await grepProjectFiles(invocation, context(), 'project')
117+
expect(textRead).toHaveBeenCalledTimes(1)
118+
expect(result).toEqual({
119+
exitCode: 0,
120+
stdout: 'files/notes.txt (file):2: needle',
121+
stderr:
122+
'Skipped 1 file with no searchable text (images, media, archives, and other binary formats).',
123+
})
124+
})
108125
it('marks truncated inventory incomplete instead of claiming a missing selector has no matches', async () => {
109126
list.mockResolvedValue({ files: [], nextKeys: [{ value: 'cursor', direction: 'asc' }] })
110127
const result = await grepProjectFiles(invocation, context(), 'project')

0 commit comments

Comments
 (0)