Skip to content

Commit 102ee20

Browse files
waleedlatif1waleedlatif1
committed
fix(cli): prioritize failure evidence in run summaries
1 parent 2579549 commit 102ee20

3 files changed

Lines changed: 177 additions & 18 deletions

File tree

‎packages/sim-cli/src/output/run-diagnostics.test.ts‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,141 @@ describe('compact run diagnostics', () => {
222222
expect(result.truncated).toBe(true)
223223
})
224224

225+
it.each(['finalOutput', 'files'])(
226+
'preserves failure metadata when %s exhausts the detail budget',
227+
(field) => {
228+
const result = summarizeRun({
229+
[field]: Array.from({ length: 12 }, () =>
230+
Array.from({ length: 12 }, () => Array.from({ length: 12 }, () => 'detail'))
231+
),
232+
traceSpans: [
233+
{
234+
blockId: 'failed-1',
235+
name: 'Send message',
236+
status: 'error',
237+
errorMessage: 'not_in_channel',
238+
errorHandled: true,
239+
},
240+
{
241+
blockId: 'agent-1',
242+
status: 'success',
243+
toolCalls: [{ name: 'lookup', status: 'error', error: 'Rate limited' }],
244+
},
245+
],
246+
})
247+
248+
expect(result.failures).toMatchObject([
249+
{
250+
blockId: 'failed-1',
251+
name: 'Send message',
252+
status: 'error',
253+
error: 'not_in_channel',
254+
handled: true,
255+
},
256+
{
257+
blockId: 'agent-1',
258+
name: 'lookup',
259+
status: 'error',
260+
error: 'Rate limited',
261+
handled: false,
262+
},
263+
])
264+
expect(result.truncated).toBe(true)
265+
}
266+
)
267+
268+
it('preserves later failure metadata when an earlier failure has large input and output', () => {
269+
const largeValue = Array.from({ length: 12 }, () =>
270+
Array.from({ length: 12 }, () => Array.from({ length: 12 }, () => 'detail'))
271+
)
272+
const result = summarizeRun({
273+
traceSpans: [
274+
{
275+
blockId: 'first',
276+
status: 'error',
277+
errorMessage: 'First failure',
278+
input: largeValue,
279+
output: largeValue,
280+
},
281+
{
282+
blockId: 'last',
283+
name: 'Last operation',
284+
status: 'error',
285+
errorMessage: 'Later failure',
286+
errorHandled: true,
287+
},
288+
],
289+
})
290+
291+
expect(result.failures).toMatchObject([
292+
{ blockId: 'first', status: 'error', error: 'First failure', handled: false },
293+
{
294+
blockId: 'last',
295+
name: 'Last operation',
296+
status: 'error',
297+
error: 'Later failure',
298+
handled: true,
299+
},
300+
])
301+
expect(result.truncated).toBe(true)
302+
})
303+
304+
it('preserves a late failure after many successful observed spans', () => {
305+
const result = summarizeRun({
306+
traceSpans: [
307+
...Array.from({ length: 99 }, (_, index) => ({
308+
blockId: `step-${index}`,
309+
name: `Step ${index}`,
310+
status: 'success',
311+
})),
312+
{
313+
blockId: 'last',
314+
name: 'Last operation',
315+
status: 'error',
316+
errorMessage: 'Late failure',
317+
},
318+
],
319+
})
320+
321+
expect(result.failures).toMatchObject([
322+
{ blockId: 'last', name: 'Last operation', status: 'error', error: 'Late failure' },
323+
])
324+
expect(result.observedBlocks).toHaveLength(100)
325+
expect(result.truncated).toBe(true)
326+
})
327+
328+
it.each([false, true])(
329+
'retains legacy status-only tool failures with explicit handled=%s',
330+
(errorHandled) => {
331+
const result = summarizeRun({
332+
traceSpans: [
333+
{
334+
blockId: 'agent-1',
335+
status: 'success',
336+
errorHandled,
337+
toolCalls: [
338+
{ name: 'lookup', status: 'error', input: { query: 'test' } },
339+
{ name: 'lookup', status: 'success', output: { error: 'Ordinary data' } },
340+
],
341+
},
342+
],
343+
})
344+
345+
expect(result.failures).toEqual([
346+
{
347+
blockId: 'agent-1',
348+
name: 'lookup',
349+
status: 'error',
350+
error: null,
351+
handled: errorHandled,
352+
input: { query: 'test' },
353+
output: null,
354+
},
355+
])
356+
expect(result.truncated).toBe(false)
357+
}
358+
)
359+
225360
it('bounds wide/deep traces, long text, and nested output values', () => {
226361
const result = summarizeRun({
227362
traceSpans: Array.from({ length: 1000 }, (_, i) => ({

‎packages/sim-cli/src/output/run-diagnostics.ts‎

Lines changed: 35 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,6 @@ export function summarizeRun(log: unknown): Record<string, unknown> {
5858
return Object.fromEntries(entries)
5959
}
6060

61-
const finalOutput = compact(log.finalOutput)
62-
const files = compact(log.files)
63-
6461
const failures: Record<string, unknown>[] = []
6562
const observedBlocks: Record<string, unknown>[] = []
6663
/** Iterator frames keep traversal memory proportional to depth, not trace width. */
@@ -77,19 +74,19 @@ export function summarizeRun(log: unknown): Record<string, unknown> {
7774
const span = next.value
7875
if (!isRecordLike(span)) continue
7976
const identity = {
80-
blockId: compact(span.blockId),
81-
name: compact(span.name),
82-
status: compact(span.status),
77+
blockId: span.blockId,
78+
name: span.name,
79+
status: span.status,
8380
}
8481
if (typeof span.blockId === 'string') observedBlocks.push(identity)
8582
if (span.errorMessage || span.status === 'error' || span.status === 'failed') {
8683
if (failures.length < MAX_FAILURES) {
8784
failures.push({
8885
...identity,
89-
error: compact(span.errorMessage),
86+
error: span.errorMessage,
9087
handled: span.errorHandled === true,
91-
input: compact(span.input),
92-
output: compact(span.output),
88+
input: span.input,
89+
output: span.output,
9390
})
9491
} else truncated = true
9592
} else if (Array.isArray(span.toolCalls)) {
@@ -101,32 +98,53 @@ export function summarizeRun(log: unknown): Record<string, unknown> {
10198
}
10299
visitedToolCalls++
103100
const call = span.toolCalls[index]
104-
if (!isRecordLike(call) || typeof call.error !== 'string' || !call.error) continue
101+
if (!isRecordLike(call)) continue
102+
if (call.status !== 'error' && (typeof call.error !== 'string' || !call.error)) continue
105103
if (failures.length === MAX_FAILURES) {
106104
truncated = true
107105
break
108106
}
109107
failures.push({
110108
blockId: identity.blockId,
111-
name: compact(call.name),
112-
status: compact(call.status),
113-
error: compact(call.error),
109+
name: call.name,
110+
status: call.status,
111+
error: call.error,
114112
handled: span.errorHandled === true,
115-
input: compact(call.input),
116-
output: compact(call.output),
113+
input: call.input,
114+
output: call.output,
117115
})
118116
}
119117
}
120118
if (Array.isArray(span.children)) pending.push(span.children[Symbol.iterator]())
121119
}
122120
if (pending.some((iterator) => !iterator.next().done)) truncated = true
123121

122+
/** Keep bounded failure evidence before arbitrary payloads consume the shared value budget. */
123+
const compactFailures: Record<string, unknown>[] = failures.map((failure) => ({
124+
blockId: compact(failure.blockId),
125+
name: compact(failure.name),
126+
status: compact(failure.status),
127+
error: compact(failure.error),
128+
handled: failure.handled,
129+
}))
130+
const finalOutput = compact(log.finalOutput)
131+
const files = compact(log.files)
132+
for (const [index, failure] of failures.entries()) {
133+
compactFailures[index].input = compact(failure.input)
134+
compactFailures[index].output = compact(failure.output)
135+
}
136+
const compactBlocks = observedBlocks.map((block) => ({
137+
blockId: compact(block.blockId),
138+
name: compact(block.name),
139+
status: compact(block.status),
140+
}))
141+
124142
return {
125143
runId: log.runId,
126144
executionStatus: log.status,
127145
finalOutput,
128-
failures,
129-
observedBlocks,
146+
failures: compactFailures,
147+
observedBlocks: compactBlocks,
130148
files,
131149
truncated,
132150
scope:

‎packages/sim-cli/src/runtime/execute.test.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,12 @@ it('runs compact log diagnostics through the generated authenticated read', asyn
127127
toolCalls: [{ name: 'lookup', error: 'Rate limited' }],
128128
},
129129
],
130-
finalOutput: { requiresClarification: true },
130+
finalOutput: {
131+
requiresClarification: true,
132+
details: Array.from({ length: 12 }, () =>
133+
Array.from({ length: 12 }, () => Array.from({ length: 12 }, () => 'detail'))
134+
),
135+
},
131136
workflowState: { source: 'hidden body' },
132137
},
133138
})
@@ -145,6 +150,7 @@ it('runs compact log diagnostics through the generated authenticated read', asyn
145150
executionStatus: 'completed',
146151
finalOutput: { requiresClarification: true },
147152
failures: [{ blockId: 'agent-1', name: 'lookup', error: 'Rate limited', handled: true }],
153+
truncated: true,
148154
})
149155
expect(String(stdout.mock.calls[0][0])).not.toContain('hidden body')
150156
})

0 commit comments

Comments
 (0)