diff --git a/.changeset/loop-stopped-signal.md b/.changeset/loop-stopped-signal.md new file mode 100644 index 0000000000..2be31fac10 --- /dev/null +++ b/.changeset/loop-stopped-signal.md @@ -0,0 +1,8 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-harness': minor +--- + +A run that the agent loop stops before the model finished now says so. When `agentLoopStrategy` (for example `maxIterations`) or a middleware `onShouldContinue` returns `false` while tool results wait for the next model call, `chat()` ends the stream with a `RUN_FINISHED` that has `metadata.tanstack.loopStopped: true`, and the last stored assistant message gets the same field. `finishReason` stays the reason of the last model call (`tool_calls`, or `length` for a cut answer). A run that the model ends has no `loopStopped`. + +In `@tanstack/ai-harness`, a turn that the loop stops settles with `{ outcome: 'completed', loopStopped: true }`. `session.settled()`, the `harness.input.settled` event of `createHarnessClient().events()`, and the transcript all carry it. diff --git a/docs/api/ai.md b/docs/api/ai.md index 7fd608dfce..158759cdf7 100644 --- a/docs/api/ai.md +++ b/docs/api/ai.md @@ -259,6 +259,8 @@ An iteration is one model turn, not one tool call. One turn can call many tools A strategy that goes on after a `length` answer makes the model send the [cut tool call](../chat/agentic-cycle#when-the-output-limit-cuts-a-tool-call) again. +When a strategy stops the loop while tool results wait for the model, the last `RUN_FINISHED` has `metadata.tanstack.loopStopped: true`. See [Know when the loop stopped the run](../chat/agentic-cycle#know-when-the-loop-stopped-the-run). + ## Response helpers Each helper turns a `chat()` stream into a response for one transport. Pick the one that matches the client connection adapter. See [Transports](../transports/overview). diff --git a/docs/chat/agentic-cycle.md b/docs/chat/agentic-cycle.md index 9169041571..55c04c587e 100644 --- a/docs/chat/agentic-cycle.md +++ b/docs/chat/agentic-cycle.md @@ -213,6 +213,66 @@ A strategy is a function. It gets the loop state and returns `true` to allow one - `toolCallCount`: the tool calls so far in this run. - `lastTurnToolCallCount`: the tool calls in the last batch. +### Know when the loop stopped the run + +Read `metadata.tanstack.loopStopped` on the last `RUN_FINISHED` event. It is `true` when the strategy, or a middleware `onShouldContinue`, stopped the loop while tool results waited for the model. In that case, the model did not finish its task. + +A run that the model ends has no `loopStopped`. The `finishReason` is always the reason of the last model call, often `tool_calls` for a stopped run. + +On the server: + +```typescript +// api/chat.ts +import { chat, maxIterations } from "@tanstack/ai"; +import { openaiText } from "@tanstack/ai-openai"; +import { getWeather } from "./tools"; + +const stream = chat({ + adapter: openaiText("gpt-6.1-sol"), + messages: [{ role: "user", content: "Plan my week of outfits." }], + threadId: "thread-1", + tools: [getWeather], + agentLoopStrategy: maxIterations(3), +}); + +for await (const chunk of stream) { + if (chunk.type === "RUN_FINISHED" && chunk.metadata?.tanstack?.loopStopped) { + console.log("The loop stopped before the model finished."); + } +} +``` + +On the client, read the same event in `onChunk`: + +```tsx +import { useState } from "react"; +import { useChat, fetchServerSentEvents } from "@tanstack/ai-react"; + +export function Chat() { + const [loopStopped, setLoopStopped] = useState(false); + const { sendMessage } = useChat({ + connection: fetchServerSentEvents("/api/chat"), + threadId: "support-chat", + onChunk: (chunk) => { + if (chunk.type === "RUN_FINISHED" && chunk.metadata?.tanstack?.loopStopped) { + setLoopStopped(true); + } + }, + }); + + return ( + <> + + {loopStopped ?

The agent stopped at its step limit.

: null} + + ); +} +``` + +The last assistant message that `chat()` stores on the server also gets `metadata.tanstack.loopStopped: true`. So a saved transcript keeps the signal. + ## Advanced ### When the output limit cuts a tool call @@ -225,7 +285,7 @@ A model can run out of output tokens in the middle of a tool call, for example w - It starts no client tool. - Each cut call gets an error tool result. -By default, the run then ends with the finish reason `length`. To let the model send the cut call again, use a strategy that does not stop on `length`: +By default, the run then ends with the finish reason `length` and [`loopStopped`](#know-when-the-loop-stopped-the-run). To let the model send the cut call again, use a strategy that does not stop on `length`: ```typescript // api/chat.ts diff --git a/docs/config.json b/docs/config.json index d1f636d5b8..8fe8b9c550 100644 --- a/docs/config.json +++ b/docs/config.json @@ -216,7 +216,7 @@ "label": "Agentic Cycle", "to": "chat/agentic-cycle", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-05" }, { "label": "Thinking & Reasoning", @@ -1133,7 +1133,7 @@ "label": "Send inputs safely", "to": "harness/inputs", "addedAt": "2026-10-13", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-05" }, { "label": "Run a tool without a model", @@ -2038,7 +2038,7 @@ "label": "@tanstack/ai", "to": "api/ai", "addedAt": "2026-04-15", - "updatedAt": "2026-11-02" + "updatedAt": "2026-11-05" }, { "label": "@tanstack/ai-client", diff --git a/docs/harness/inputs.md b/docs/harness/inputs.md index 5ce0dd0a07..1c9396a126 100644 --- a/docs/harness/inputs.md +++ b/docs/harness/inputs.md @@ -78,13 +78,16 @@ const settlement = await session.settled({ if (settlement.outcome === 'failed') { console.log(settlement.error?.code) // 'attempts_exhausted', 'timeout', or undefined } +if (settlement.outcome === 'completed' && settlement.loopStopped) { + console.log('The agent loop stopped the turn before the model finished.') +} ``` `signal` is optional. When it aborts first, `settled` rejects with its reason. The input keeps running. On a host that does not own the thread, the input waits for the owner. If the owner refuses it, `settled` rejects with `InputRejectedError`. The owner refuses an input when the thread is busy and the input has `busy: 'reject'`, or when the log has its id with another message. The log then has a `harness.offer.rejected` record. -- `completed`: the turn has its answer. +- `completed`: the turn has its answer. With `loopStopped: true`, the agent loop stopped the turn while tool results waited for the model, so the model did not finish. The default `agentLoopStrategy` of a harness is `maxIterations(50)`. See [Know when the loop stopped the run](../chat/agentic-cycle#know-when-the-loop-stopped-the-run). - `failed`: the turn failed. `error.code` is set for a limit. - `aborted`: someone cancelled the turn. The text that the model streamed before the cancel stays in the transcript as its answer. - `interrupted`: the turn waits for a person. The `resolve` that answers it is a new input. See [Wait for a person](./waits). diff --git a/packages/ai-client/src/chat-client.ts b/packages/ai-client/src/chat-client.ts index dce5dab219..73201ae930 100644 --- a/packages/ai-client/src/chat-client.ts +++ b/packages/ai-client/src/chat-client.ts @@ -220,6 +220,8 @@ function connectionDrainsOnSend(connection: ConnectionAdapter): boolean { function isIntermediateToolTurn(chunk: StreamChunk): boolean { if (chunk.type !== 'RUN_FINISHED') return false if (chunk.outcome?.type === 'interrupt') return false + // A loop stop keeps finishReason tool_calls. That event ends the stream. + if (tanstackMetadata(chunk)?.loopStopped === true) return false const extra = chunk as StreamChunk & { finishReason?: unknown } if (extra.finishReason !== undefined) { return extra.finishReason === 'tool_calls' diff --git a/packages/ai-client/tests/chat-client-loop-stopped.test.ts b/packages/ai-client/tests/chat-client-loop-stopped.test.ts new file mode 100644 index 0000000000..8b794d84d6 --- /dev/null +++ b/packages/ai-client/tests/chat-client-loop-stopped.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { EventType, type StreamChunk } from '@tanstack/ai/client' +import { ChatClient } from '../src/chat-client' +import type { SubscribeConnectionAdapter } from '../src/connection-adapters' + +function finished(loopStopped: boolean): StreamChunk { + return { + type: EventType.RUN_FINISHED, + runId: 'run-1', + threadId: 'thread-1', + timestamp: 1, + metadata: { + tanstack: { + finishReason: 'tool_calls', + ...(loopStopped ? { loopStopped: true as const } : {}), + }, + }, + } +} + +function subscribeConnection(chunk: StreamChunk): SubscribeConnectionAdapter { + return { + async *subscribe() { + yield { + type: EventType.RUN_STARTED, + runId: 'run-1', + threadId: 'thread-1', + timestamp: 0, + } + yield chunk + }, + send: async () => {}, + } +} + +describe('loopStopped on a subscribe connection', () => { + it('resolves sendMessage when the loop stops on tool_calls', async () => { + const client = new ChatClient({ + threadId: 'thread-1', + connection: subscribeConnection(finished(true)), + }) + + await client.sendMessage('hi') + + expect(client.getIsLoading()).toBe(false) + expect(client.getSnapshot().status).toBe('ready') + }) +}) diff --git a/packages/ai-harness/src/log.ts b/packages/ai-harness/src/log.ts index 14f15fc9bd..8596424a57 100644 --- a/packages/ai-harness/src/log.ts +++ b/packages/ai-harness/src/log.ts @@ -944,7 +944,12 @@ export function foldEntry( ...(record.operationId ? { operationId: record.operationId } : {}), ...(record.outcome === 'failed' ? { outcome: 'failed', error: record.error } - : { outcome: record.outcome }), + : { + outcome: record.outcome, + ...(record.outcome === 'completed' && record.loopStopped + ? { loopStopped: true as const } + : {}), + }), }, })) state.thread = settleRunNode(state.thread, record) diff --git a/packages/ai-harness/src/session/turn.ts b/packages/ai-harness/src/session/turn.ts index 330c9bc9c8..2aa178562b 100644 --- a/packages/ai-harness/src/session/turn.ts +++ b/packages/ai-harness/src/session/turn.ts @@ -678,6 +678,8 @@ export abstract class SessionTurn< let openMessageId: string | undefined let interrupts: Array | undefined let failure: string | undefined + /** The last RUN_FINISHED says the agent loop stopped the model. */ + let loopStopped = false /** With `outputSchema`: the value of the last call, or why its check failed. */ let structured: unknown let structuredError: { code: string; rawText?: string } | undefined @@ -1364,6 +1366,12 @@ export abstract class SessionTurn< ) { isRunOpen = false } + if ( + chunk.type === EventType.RUN_FINISHED && + chunk.metadata?.tanstack?.loopStopped === true + ) { + loopStopped = true + } if ( chunk.type === EventType.TEXT_MESSAGE_CONTENT || chunk.type === EventType.REASONING_MESSAGE_CONTENT || @@ -1853,7 +1861,10 @@ export abstract class SessionTurn< await settleTurn({ outcome: 'interrupted' }) operation.finish('interrupted', { text, interrupts: [], media: captured }) } else { - await settleTurn({ outcome: 'completed' }) + await settleTurn({ + outcome: 'completed', + ...(loopStopped ? { loopStopped: true } : {}), + }) operation.finish('completed', { text, media: captured, diff --git a/packages/ai-harness/src/types.ts b/packages/ai-harness/src/types.ts index 767ed48d52..01a522adcc 100644 --- a/packages/ai-harness/src/types.ts +++ b/packages/ai-harness/src/types.ts @@ -481,9 +481,19 @@ export type InputSettlement = { * `interrupted`: the turn stopped for human input. The `resolve` that * answers it is a new input with its own settlement. */ - outcome: 'completed' | 'aborted' | 'interrupted' + outcome: 'aborted' | 'interrupted' error?: never } + | { + outcome: 'completed' + error?: never + /** + * The agent loop stopped before the model was done: + * `agentLoopStrategy` (default `maxIterations(50)`) or a middleware + * `onShouldContinue` said stop while tool results waited. + */ + loopStopped?: true + } | { outcome: 'failed' /** diff --git a/packages/ai-harness/tests/loop-stopped.test.ts b/packages/ai-harness/tests/loop-stopped.test.ts new file mode 100644 index 0000000000..09e334403b --- /dev/null +++ b/packages/ai-harness/tests/loop-stopped.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { EventType, maxIterations, toolDefinition } from '@tanstack/ai' +import { memoryPersistence } from '@tanstack/ai-persistence' +import { HARNESS_EVENTS, createHarnessHost, defineHarness } from '../src' +import { createHarnessHandler } from '../src/server' +import { createHarnessClient } from '../src/client' +import { mockAdapter, text, toolCall } from './helpers' +import type { ModelMessage } from '@tanstack/ai' +import type { InputSettlement } from '../src' +import type { Reply } from './helpers' + +const lookup = toolDefinition({ + name: 'lookup', + description: 'Look something up', + inputSchema: z.object({ q: z.string() }), +}).server(async () => ({ found: true })) + +/** A harness with `maxIterations(2)`, its host session, and a client. */ +async function open(replies: Array) { + const { adapter, calls } = mockAdapter(replies) + const harness = defineHarness({ + name: 'test/loop-stopped', + adapter, + tools: [lookup], + agentLoopStrategy: maxIterations(2), + }) + const host = createHarnessHost({ + persistence: memoryPersistence(), + access: { + getUser: () => ({ id: 'u' }), + rules: { isAllowed: () => true }, + }, + }) + const handler = createHarnessHandler({ host, harness }) + const client = createHarnessClient({ + url: 'http://local/api/harness/', + threadId: 't1', + fetch: (input, init) => handler(new Request(input, init)), + }) + const session = await host.open(harness, { + threadId: 't1', + user: { id: 'u' }, + }) + return { host, session, client, calls } +} + +/** The first `harness.input.settled` value that the client reads. */ +async function clientSettlement( + client: Awaited>['client'], +): Promise { + for await (const { event } of client.events({ + from: '0', + signal: AbortSignal.timeout(2_000), + })) { + if ( + event.type === EventType.CUSTOM && + event.name === HARNESS_EVENTS.inputSettled + ) { + return event.value + } + } + return undefined +} + +const loopStoppedOf = (message: ModelMessage | undefined) => { + const metadata = message?.metadata + if (typeof metadata !== 'object' || metadata === null) return undefined + const tanstack: unknown = 'tanstack' in metadata ? metadata.tanstack : null + if (typeof tanstack !== 'object' || tanstack === null) return undefined + return 'loopStopped' in tanstack ? tanstack.loopStopped : undefined +} + +describe('a turn that the agent loop stops', () => { + it('settles completed with loopStopped, for the session and the client', async () => { + const { host, session, client, calls } = await open([ + () => toolCall('lookup', { q: 'a' }, 'call-1'), + () => toolCall('lookup', { q: 'b' }, 'call-2'), + () => text('never'), + ]) + + await session.send('look it up', { inputId: 'in-1' }) + + expect(calls).toHaveLength(2) + expect(await session.settled({ inputId: 'in-1' })).toMatchObject({ + outcome: 'completed', + loopStopped: true, + }) + expect(await clientSettlement(client)).toMatchObject({ + inputId: 'in-1', + outcome: 'completed', + loopStopped: true, + }) + const assistant = (await client.transcript()).findLast( + (message) => message.role === 'assistant', + ) + expect(loopStoppedOf(assistant)).toBe(true) + await host.close() + }) + + it('has no loopStopped when the model ends the turn', async () => { + const { host, session, client } = await open([ + () => toolCall('lookup', { q: 'a' }, 'call-1'), + () => text('Found it.'), + ]) + + await session.send('look it up', { inputId: 'in-1' }) + + const settlement = await session.settled({ inputId: 'in-1' }) + expect(settlement).toMatchObject({ outcome: 'completed' }) + expect(settlement).not.toHaveProperty('loopStopped') + expect(await clientSettlement(client)).not.toHaveProperty('loopStopped') + const transcript = await client.transcript() + expect(transcript.some((message) => loopStoppedOf(message))).toBe(false) + await host.close() + }) +}) diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index b2f82f23b6..2cfdcfe546 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -1123,6 +1123,13 @@ class TextEngine< private deferredModelRunFinishedChunks: Array = [] private earlyTermination = false + /** + * The strategy or `onShouldContinue` refused another model call while tool + * results waited. A length cut sets this too. The synthetic RUN_FINISHED + * is skipped when that cut already sent its own finish. + */ + private loopStopped = false + private loopStoppedFinishSent = false private toolPhase: ToolPhaseResult = 'continue' private cyclePhase: CyclePhase = 'processText' // Client state extracted from initial messages (before conversion to ModelMessage) @@ -1616,8 +1623,13 @@ class TextEngine< this.loopState(this.iterationCount), ), )) - if (isStoppedAfterResume && !this.finalStructuredOutput) { - yield* this.emitSuccessfulEarlyTermination() + if (isStoppedAfterResume) { + this.loopStopped = true + this.markLoopStoppedMessage() + if (!this.finalStructuredOutput) { + this.loopStoppedFinishSent = true + yield* this.emitSuccessfulEarlyTermination() + } } // Skip the agent loop entirely when there are no tools AND a separate @@ -1704,8 +1716,28 @@ class TextEngine< this.logger.agentLoop('run finished', { finishReason: this.lastFinishReason, + loopStopped: this.loopStopped, }) + // The `tool_calls` RUN_FINISHED went out before the tool results. A + // last one tells the client that the loop, not the model, ended the run. + // A streamed finalization sends the outer RUN_FINISHED, so it carries + // the flag instead. + if ( + this.loopStopped && + !this.loopStoppedFinishSent && + !this.isCancelled() + ) { + this.markLoopStoppedMessage() + if (!this.finalizationOwnsLifecycle()) { + yield* this.pipeThroughMiddleware( + withTanstackMetadata(this.createSyntheticFinishedEvent(), { + loopStopped: true, + }), + ) + } + } + // After the agent loop ends, if a structured-output finalization was // requested AND the run hasn't already errored/aborted, run it through // the middleware pipeline. The terminal hook fires once at the very @@ -2202,10 +2234,7 @@ class TextEngine< // // Native combined mode does NOT issue a second adapter stream — the // agent loop's lifecycle IS the outer pair the consumer sees. - const suppressAgentLifecycle = - !!this.finalStructuredOutput && - this.finalStructuredOutput.yieldChunks && - this.finalStructuredOutput.nativeCombined !== true + const suppressAgentLifecycle = this.finalizationOwnsLifecycle() for (const outputChunk of outputChunks) { this.recordPublicActivity(outputChunk, raw) for (const spec of normalizeStreamChunk( @@ -2950,12 +2979,24 @@ class TextEngine< // A stopped run ends on the `length` RUN_FINISHED, after the results. // Native combined structured output sends it after its harvest, and // drops it when the harvest fails. + if (!continues) { + this.loopStopped = true + this.markLoopStoppedMessage() + this.deferredToolCallRunFinishedChunks = + this.deferredToolCallRunFinishedChunks.map((chunk) => + chunk.type === EventType.RUN_FINISHED + ? withTanstackMetadata(chunk, { loopStopped: true }) + : chunk, + ) + } if (!continues && this.finalStructuredOutput?.nativeCombined) { + this.loopStoppedFinishSent = true this.deferredModelRunFinishedChunks.push( ...this.deferredToolCallRunFinishedChunks, ) this.deferredToolCallRunFinishedChunks = [] } else if (!continues) { + this.loopStoppedFinishSent = true yield* this.flushDeferredToolCallRunFinishedChunks() } this.toolCallManager.clear() @@ -3193,11 +3234,16 @@ class TextEngine< this.lastFinishReason = 'stop' const finishEvent = this.createSyntheticFinishedEvent('stop') yield* this.emitSyntheticRunStarted(finishEvent) - yield* this.pipeThroughMiddleware({ + const finished = { ...finishEvent, timestamp: Date.now(), - outcome: { type: 'success' }, - }) + outcome: { type: 'success' as const }, + } + yield* this.pipeThroughMiddleware( + this.loopStopped + ? withTanstackMetadata(finished, { loopStopped: true }) + : finished, + ) } private discardDeferredToolCallRunFinishedChunks(): void { @@ -3528,6 +3574,7 @@ class TextEngine< ? existingStructuredIndex : startedLength, ) + if (this.loopStopped) this.markLoopStoppedMessage() this.middlewareCtx.messages = this.messages } @@ -4567,10 +4614,38 @@ class TextEngine< const cutAnswerContinues = this.cutAnswerContinues this.cutAnswerContinues = undefined + const continues = + cutAnswerContinues ?? (await this.loopContinues(this.iterationCount)) + // Tool results wait for the next model call, and the loop says no. + if (!continues && this.toolPhase === 'continue') this.loopStopped = true + return continues && this.toolPhase === 'continue' + } + + /** + * A streamed structured-output finalization sends the one outer + * RUN_STARTED / RUN_FINISHED pair, not the agent loop. + */ + private finalizationOwnsLifecycle(): boolean { return ( - (cutAnswerContinues ?? (await this.loopContinues(this.iterationCount))) && - this.toolPhase === 'continue' + !!this.finalStructuredOutput && + this.finalStructuredOutput.yieldChunks && + this.finalStructuredOutput.nativeCombined !== true + ) + } + + /** + * Mark the last assistant message as stopped by the loop, so a stored + * transcript keeps the signal of the `RUN_FINISHED`. + */ + private markLoopStoppedMessage(): void { + const index = this.messages.findLastIndex((m) => m.role === 'assistant') + if (index < 0) return + this.messages = this.messages.map((message, at) => + at === index + ? withTanstackMetadata(message, { loopStopped: true }) + : message, ) + this.middlewareCtx.messages = this.messages } /** What the loop strategy and middleware `onShouldContinue` get. */ @@ -5041,6 +5116,13 @@ class TextEngine< await this.runOnUsageFromChunk(chunk) } + // This is the outer RUN_FINISHED, so it carries the loop stop. + if (chunk.type === EventType.RUN_FINISHED && this.loopStopped) { + outboundChunk = withTanstackMetadata(chunk, { + loopStopped: true, + }) + } + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { rawText += chunk.delta } diff --git a/packages/ai/src/activities/chat/stream/processor.ts b/packages/ai/src/activities/chat/stream/processor.ts index 917406236f..1d61040b87 100644 --- a/packages/ai/src/activities/chat/stream/processor.ts +++ b/packages/ai/src/activities/chat/stream/processor.ts @@ -2723,7 +2723,8 @@ export class StreamProcessor { this.completeAllToolCalls() const isIntermediateToolTurn = this.finishReason === 'tool_calls' && - chunk.outcome?.type !== 'interrupt' + chunk.outcome?.type !== 'interrupt' && + tanstackMetadata(chunk)?.loopStopped !== true if (isIntermediateToolTurn) { // The next model call is a new assistant message. Leave this // call's messages closed so its reasoning cannot land here. diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index b5e04c1a2c..764457ce6f 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -710,6 +710,8 @@ export interface TanStackMessageMetadata { * stopped at the output limit, so its tool calls are not complete. */ finishReason?: TanStackRunMetadata['finishReason'] + /** See `TanStackRunMetadata.loopStopped`. Set on the last assistant message of the run. */ + loopStopped?: boolean /** * The provider's answer items of this assistant message, in order, for a * same-model replay. The OpenAI Responses adapter keeps each output @@ -773,6 +775,14 @@ export interface TanStackRunMetadata { /** See `TanStackMessageMetadata.reasoningItem`. */ reasoningItem?: boolean finishReason?: 'stop' | 'length' | 'content_filter' | 'tool_calls' | null + /** + * `true` on the last `RUN_FINISHED` of a run that the agent loop stopped + * before the model was done: `agentLoopStrategy` (for example + * `maxIterations`) or a middleware `onShouldContinue` returned `false` + * while tool results waited for the next model call. `finishReason` stays + * the reason of the last model call. + */ + loopStopped?: boolean /** TokenUsage fields that have no AG-UI `usage[]` equivalent. */ usage?: TokenUsageLeftover interruptErrors?: ReadonlyArray diff --git a/packages/ai/tests/chat.test.ts b/packages/ai/tests/chat.test.ts index d55c42f73c..0e68466123 100644 --- a/packages/ai/tests/chat.test.ts +++ b/packages/ai/tests/chat.test.ts @@ -3397,9 +3397,11 @@ describe('chat()', () => { expect(lookup).not.toHaveBeenCalled() expect(afterToolCall).toHaveBeenCalledTimes(1) expect(calls).toHaveLength(1) - expect( - tanstackMetadata(expectSingleRunFinished(chunks))?.finishReason, - ).toBe('length') + expect(tanstackMetadata(expectSingleRunFinished(chunks))).toMatchObject({ + finishReason: 'length', + // The default strategy stops on `length`. + loopStopped: true, + }) // The result comes before the RUN_FINISHED that ends the run. expect( chunks @@ -3416,7 +3418,10 @@ describe('chat()', () => { content: 'Let me look.', toolCalls: [expect.objectContaining({ id: 'call_1' })], metadata: { - tanstack: expect.objectContaining({ finishReason: 'length' }), + tanstack: expect.objectContaining({ + finishReason: 'length', + loopStopped: true, + }), }, }), { @@ -3720,6 +3725,126 @@ describe('chat()', () => { expect(calls.length).toBe(2) }) + describe('loopStopped', () => { + const toolTurn = (id: string) => [ + ev.runStarted(), + ev.toolStart(id, 'repeater'), + ev.toolArgs(id, '{}'), + ev.runFinished('tool_calls'), + ] + const textTurn = [ + ev.runStarted(), + ev.textStart('msg-2'), + ev.textContent('Done.', 'msg-2'), + ev.textEnd('msg-2'), + ev.runFinished('stop'), + ] + const runFinished = (chunks: Array) => + chunks.filter((item) => item.type === EventType.RUN_FINISHED) + + async function run(iterations: Array>) { + const { adapter, calls } = createMockAdapter({ iterations }) + let transcript: Array = [] + const chunks = await collectChunks( + chat({ + threadId: 'thread-1', + adapter, + messages: [{ role: 'user', content: 'Repeat' }], + tools: [serverTool('repeater', () => ({ ok: true }))], + agentLoopStrategy: maxIterations(2), + middleware: [ + defineChatMiddleware({ + onFinish(ctx) { + transcript = [...ctx.messages] + }, + }), + ], + }) as AsyncIterable, + ) + const assistants = transcript.filter((m) => m.role === 'assistant') + return { calls, chunks, assistants } + } + + it('ends a run that maxIterations stops with a loopStopped RUN_FINISHED', async () => { + const { calls, chunks, assistants } = await run([ + toolTurn('call_1'), + toolTurn('call_2'), + textTurn, + ]) + + expect(calls).toHaveLength(2) + const finished = runFinished(chunks) + expect( + finished.map((item) => tanstackMetadata(item)?.loopStopped), + ).toEqual([undefined, undefined, true]) + // It comes last, after the tool results, and keeps the finish reason + // of the last model call. + expect(chunks.at(-1)).toBe(finished.at(-1)) + expect(tanstackMetadata(finished.at(-1))?.finishReason).toBe( + 'tool_calls', + ) + expect(assistants.map((m) => tanstackMetadata(m)?.loopStopped)).toEqual( + [undefined, true], + ) + }) + + it('does not mark a run that the model ends, also at the iteration limit', async () => { + const { calls, chunks, assistants } = await run([ + toolTurn('call_1'), + textTurn, + ]) + + expect(calls).toHaveLength(2) + const finished = runFinished(chunks) + expect(tanstackMetadata(finished.at(-1))?.finishReason).toBe('stop') + expect( + finished.some((item) => tanstackMetadata(item)?.loopStopped), + ).toBe(false) + expect(assistants.some((m) => tanstackMetadata(m)?.loopStopped)).toBe( + false, + ) + }) + + it('puts loopStopped on the RUN_FINISHED of a streamed structured-output finalization', async () => { + const { adapter } = createMockAdapter({ + iterations: [toolTurn('call_1'), toolTurn('call_2')], + structuredOutput: async () => ({ + data: { status: 'ok' }, + rawText: '{"status":"ok"}', + }), + }) + let transcript: Array = [] + const chunks = await collectChunks( + chat({ + threadId: 'thread-1', + adapter, + messages: [{ role: 'user', content: 'Repeat' }], + tools: [serverTool('repeater', () => ({ ok: true }))], + agentLoopStrategy: maxIterations(2), + outputSchema: { + type: 'object', + properties: { status: { type: 'string' } }, + required: ['status'], + }, + stream: true, + middleware: [ + defineChatMiddleware({ + onFinish(ctx) { + transcript = [...ctx.messages] + }, + }), + ], + }) as AsyncIterable, + ) + + const finished = expectSingleRunFinished(chunks) + expect(chunks.at(-1)).toBe(finished) + expect(tanstackMetadata(finished)?.loopStopped).toBe(true) + const lastAssistant = transcript.findLast((m) => m.role === 'assistant') + expect(tanstackMetadata(lastAssistant)?.loopStopped).toBe(true) + }) + }) + it('should respect toolCallCount on strategy state (counts tools, not turns)', async () => { const executeSpy = vi.fn().mockReturnValue({ data: 'result' }) diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index b7b3992915..57bafca86e 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -125,6 +125,7 @@ import { Route as ApiJsonTransportRouteImport } from './routes/api.json-transpor import { Route as ApiKeyedAdapterRouteImport } from './routes/api.keyed-adapter' import { Route as ApiLazyToolsWireRouteImport } from './routes/api.lazy-tools-wire' import { Route as ApiLoneSurrogatesWireRouteImport } from './routes/api.lone-surrogates-wire' +import { Route as ApiLoopStoppedRouteImport } from './routes/api.loop-stopped' import { Route as ApiMaxToolCallsWireRouteImport } from './routes/api.max-tool-calls-wire' import { Route as ApiMcpAppsCallRouteImport } from './routes/api.mcp-apps-call' import { Route as ApiMcpAppsChatRouteImport } from './routes/api.mcp-apps-chat' @@ -825,6 +826,11 @@ const ApiLoneSurrogatesWireRoute = ApiLoneSurrogatesWireRouteImport.update({ path: '/api/lone-surrogates-wire', getParentRoute: () => rootRouteImport, } as any) +const ApiLoopStoppedRoute = ApiLoopStoppedRouteImport.update({ + id: '/api/loop-stopped', + path: '/api/loop-stopped', + getParentRoute: () => rootRouteImport, +} as any) const ApiMaxToolCallsWireRoute = ApiMaxToolCallsWireRouteImport.update({ id: '/api/max-tool-calls-wire', path: '/api/max-tool-calls-wire', @@ -1426,6 +1432,7 @@ export interface FileRoutesByFullPath { '/api/keyed-adapter': typeof ApiKeyedAdapterRoute '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/lone-surrogates-wire': typeof ApiLoneSurrogatesWireRoute + '/api/loop-stopped': typeof ApiLoopStoppedRoute '/api/max-tool-calls-wire': typeof ApiMaxToolCallsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute '/api/mcp-apps-chat': typeof ApiMcpAppsChatRoute @@ -1636,6 +1643,7 @@ export interface FileRoutesByTo { '/api/keyed-adapter': typeof ApiKeyedAdapterRoute '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/lone-surrogates-wire': typeof ApiLoneSurrogatesWireRoute + '/api/loop-stopped': typeof ApiLoopStoppedRoute '/api/max-tool-calls-wire': typeof ApiMaxToolCallsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute '/api/mcp-apps-chat': typeof ApiMcpAppsChatRoute @@ -1847,6 +1855,7 @@ export interface FileRoutesById { '/api/keyed-adapter': typeof ApiKeyedAdapterRoute '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/lone-surrogates-wire': typeof ApiLoneSurrogatesWireRoute + '/api/loop-stopped': typeof ApiLoopStoppedRoute '/api/max-tool-calls-wire': typeof ApiMaxToolCallsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute '/api/mcp-apps-chat': typeof ApiMcpAppsChatRoute @@ -2059,6 +2068,7 @@ export interface FileRouteTypes { | '/api/keyed-adapter' | '/api/lazy-tools-wire' | '/api/lone-surrogates-wire' + | '/api/loop-stopped' | '/api/max-tool-calls-wire' | '/api/mcp-apps-call' | '/api/mcp-apps-chat' @@ -2269,6 +2279,7 @@ export interface FileRouteTypes { | '/api/keyed-adapter' | '/api/lazy-tools-wire' | '/api/lone-surrogates-wire' + | '/api/loop-stopped' | '/api/max-tool-calls-wire' | '/api/mcp-apps-call' | '/api/mcp-apps-chat' @@ -2479,6 +2490,7 @@ export interface FileRouteTypes { | '/api/keyed-adapter' | '/api/lazy-tools-wire' | '/api/lone-surrogates-wire' + | '/api/loop-stopped' | '/api/max-tool-calls-wire' | '/api/mcp-apps-call' | '/api/mcp-apps-chat' @@ -2690,6 +2702,7 @@ export interface RootRouteChildren { ApiKeyedAdapterRoute: typeof ApiKeyedAdapterRoute ApiLazyToolsWireRoute: typeof ApiLazyToolsWireRoute ApiLoneSurrogatesWireRoute: typeof ApiLoneSurrogatesWireRoute + ApiLoopStoppedRoute: typeof ApiLoopStoppedRoute ApiMaxToolCallsWireRoute: typeof ApiMaxToolCallsWireRoute ApiMcpAppsCallRoute: typeof ApiMcpAppsCallRoute ApiMcpAppsChatRoute: typeof ApiMcpAppsChatRoute @@ -3594,6 +3607,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiLoneSurrogatesWireRouteImport parentRoute: typeof rootRouteImport } + '/api/loop-stopped': { + id: '/api/loop-stopped' + path: '/api/loop-stopped' + fullPath: '/api/loop-stopped' + preLoaderRoute: typeof ApiLoopStoppedRouteImport + parentRoute: typeof rootRouteImport + } '/api/max-tool-calls-wire': { id: '/api/max-tool-calls-wire' path: '/api/max-tool-calls-wire' @@ -4420,6 +4440,7 @@ const rootRouteChildren: RootRouteChildren = { ApiKeyedAdapterRoute: ApiKeyedAdapterRoute, ApiLazyToolsWireRoute: ApiLazyToolsWireRoute, ApiLoneSurrogatesWireRoute: ApiLoneSurrogatesWireRoute, + ApiLoopStoppedRoute: ApiLoopStoppedRoute, ApiMaxToolCallsWireRoute: ApiMaxToolCallsWireRoute, ApiMcpAppsCallRoute: ApiMcpAppsCallRoute, ApiMcpAppsChatRoute: ApiMcpAppsChatRoute, diff --git a/testing/e2e/src/routes/api.loop-stopped.ts b/testing/e2e/src/routes/api.loop-stopped.ts new file mode 100644 index 0000000000..fa63746b59 --- /dev/null +++ b/testing/e2e/src/routes/api.loop-stopped.ts @@ -0,0 +1,52 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + maxIterations, + toServerSentEventsResponse, + toolDefinition, +} from '@tanstack/ai' +import { fakeText } from '@tanstack/ai/testing' +import { z } from 'zod' + +/** + * No network: `fakeText` answers from a script, and `maxIterations(2)` allows + * two model calls. The route sends the stream as SSE. + * + * - Default: both answers call a tool, so the loop stops the run. + * - `?done`: the second answer is text, so the model ends the run. + */ +export const Route = createFileRoute('/api/loop-stopped')({ + server: { + handlers: { + POST: ({ request }) => { + const done = new URL(request.url).searchParams.has('done') + const fake = fakeText() + fake.setResponses([ + { toolCalls: [{ name: 'getWeather', input: { city: 'Oslo' } }] }, + done + ? { text: 'Sunny in Oslo.' } + : { + toolCalls: [{ name: 'getWeather', input: { city: 'Bergen' } }], + }, + { text: 'The loop never asks for this answer.' }, + ]) + + const getWeather = toolDefinition({ + name: 'getWeather', + description: 'Get the weather for a city', + inputSchema: z.object({ city: z.string() }), + }).server(({ city }) => `sunny in ${city}`) + + return toServerSentEventsResponse( + chat({ + threadId: 'loop-stopped', + adapter: fake, + messages: [{ role: 'user', content: 'Weather in Oslo?' }], + tools: [getWeather], + agentLoopStrategy: maxIterations(2), + }), + ) + }, + }, + }, +}) diff --git a/testing/e2e/tests/loop-stopped.spec.ts b/testing/e2e/tests/loop-stopped.spec.ts new file mode 100644 index 0000000000..1b3c96eb09 --- /dev/null +++ b/testing/e2e/tests/loop-stopped.spec.ts @@ -0,0 +1,42 @@ +import { test, expect } from './fixtures' + +type Finished = { finishReason?: string; loopStopped?: boolean } + +/** The `metadata.tanstack` of each RUN_FINISHED in an SSE body. */ +function runFinished(body: string): Array { + return body + .split('\n') + .filter((line) => line.startsWith('data: ')) + .map((line) => JSON.parse(line.slice('data: '.length))) + .filter((event) => event.type === 'RUN_FINISHED') + .map((event) => { + const { finishReason, loopStopped } = event.metadata?.tanstack ?? {} + return { finishReason, loopStopped } + }) +} + +// The route is described at `src/routes/api.loop-stopped.ts`. +test.describe('agent loop stopped by maxIterations', () => { + test('the last RUN_FINISHED on the wire has loopStopped', async ({ + request, + }) => { + const res = await request.post('/api/loop-stopped') + expect(res.ok()).toBe(true) + + expect(runFinished(await res.text())).toEqual([ + { finishReason: 'tool_calls' }, + { finishReason: 'tool_calls' }, + { finishReason: 'tool_calls', loopStopped: true }, + ]) + }) + + test('a run that the model ends has no loopStopped', async ({ request }) => { + const res = await request.post('/api/loop-stopped?done') + expect(res.ok()).toBe(true) + + expect(runFinished(await res.text())).toEqual([ + { finishReason: 'tool_calls' }, + { finishReason: 'stop' }, + ]) + }) +}) diff --git a/testing/e2e/tests/max-tool-calls.spec.ts b/testing/e2e/tests/max-tool-calls.spec.ts index 1a12063944..ac429c4f84 100644 --- a/testing/e2e/tests/max-tool-calls.spec.ts +++ b/testing/e2e/tests/max-tool-calls.spec.ts @@ -40,8 +40,12 @@ test.describe('chat() tool-call budget middleware (#964)', () => { expect(skipped.length).toBe(5) // max: 5 — after the fat turn toolCallCount=8 (>= 5), so no further - // model turn runs. executeCount stays at 3 either way. + // model turn runs. The second RUN_FINISHED is the loop stop, not a + // second model call. executeCount stays at 3 either way. const runFinished = body.chunks.filter((c) => c.type === 'RUN_FINISHED') - expect(runFinished.length).toBe(1) + expect(runFinished).toHaveLength(2) + expect(runFinished[1]).toMatchObject({ + metadata: { tanstack: { loopStopped: true } }, + }) }) })