diff --git a/.changeset/ollama-stream-length-error.md b/.changeset/ollama-stream-length-error.md new file mode 100644 index 0000000000..16d2022d61 --- /dev/null +++ b/.changeset/ollama-stream-length-error.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-ollama': patch +--- + +Report `finishReason: "length"` on `RUN_FINISHED` when an Ollama stream ends with `done_reason: "length"`. This includes a reply cut by `num_predict`, and a tool call cut the same way. A normal stop stays `"stop"`. A finished tool call stays `"tool_calls"`. diff --git a/docs/adapters/ollama.md b/docs/adapters/ollama.md index 2f8e695b6b..73fdad1e62 100644 --- a/docs/adapters/ollama.md +++ b/docs/adapters/ollama.md @@ -188,7 +188,7 @@ Ollama does not make images. For images, use [OpenAI](./openai) or [Gemini](./ge Ollama nests its sampling and runner options in an `options` object **inside** `modelOptions`. This includes `temperature`, `top_p`, and `num_predict` (the token limit): -```typescript +```typescript group=ollama-options import { chat } from "@tanstack/ai"; import { ollamaText } from "@tanstack/ai-ollama"; @@ -212,6 +212,18 @@ const stream = chat({ Sampling options do not go at the root of `chat()`. For Ollama, they go to `modelOptions.options.temperature`, `modelOptions.options.top_p`, and `modelOptions.options.num_predict`. See [Moving Sampling Options into modelOptions](../migration/sampling-options-to-model-options). The full option list is in [All runner options](#all-runner-options). +### Cut-off responses + +When Ollama stops a reply at `num_predict`, it sends `done_reason: "length"`. The stream still ends with `RUN_FINISHED`. `metadata.tanstack.finishReason` is `"length"`. A tool call that stops the same way also uses `"length"`, so that call does not run. To get the rest of the reply, raise `modelOptions.options.num_predict`: + +```typescript group=ollama-options +for await (const chunk of stream) { + if (chunk.type === "RUN_FINISHED" && chunk.metadata?.tanstack?.finishReason === "length") { + console.warn("The reply was cut off."); + } +} +``` + ### Reasoning Running a thinking model like Qwen 3 or gpt-oss? Set `reasoning` on `chat()`: diff --git a/packages/ai-ollama/src/adapters/text.ts b/packages/ai-ollama/src/adapters/text.ts index 25f08e5f2f..eca6cab7e2 100644 --- a/packages/ai-ollama/src/adapters/text.ts +++ b/packages/ai-ollama/src/adapters/text.ts @@ -456,7 +456,12 @@ export class OllamaTextAdapter extends BaseTextAdapter< threadId, model: chunk.model, timestamp: Date.now(), - finishReason: toolCallsEmitted.size > 0 ? 'tool_calls' : 'stop', + finishReason: + chunk.done_reason === 'length' + ? 'length' + : toolCallsEmitted.size > 0 + ? 'tool_calls' + : 'stop', // usage is optional under exactOptionalPropertyTypes; omit the key // entirely when Ollama reported no token counts. ...(finishUsage && { usage: finishUsage }), diff --git a/packages/ai-ollama/tests/text-adapter.test.ts b/packages/ai-ollama/tests/text-adapter.test.ts index 77b1f1030d..e43ecc3355 100644 --- a/packages/ai-ollama/tests/text-adapter.test.ts +++ b/packages/ai-ollama/tests/text-adapter.test.ts @@ -133,6 +133,110 @@ describe('OllamaTextAdapter.chatStream (content streaming)', () => { .join('') expect(contents).toContain('Hello') expect(contents).toContain('world') + expect(chunks.find((c) => c.type === 'RUN_FINISHED')).toMatchObject({ + finishReason: 'stop', + }) + }) + + it('reports finishReason stop when the final chunk has no done_reason', async () => { + chatMock.mockResolvedValueOnce( + asyncIterable([ + { + message: { role: 'assistant', content: 'Hello' }, + done: true, + }, + ]), + ) + + const chunks = await collectStream( + createOllamaChat('llama3.2').chatStream({ + threadId: 'thread-1', + logger: testLogger, + model: 'llama3.2', + messages: [{ role: 'user', content: 'hi' }], + }), + ) + + expect(chunks.find((c) => c.type === 'RUN_FINISHED')).toMatchObject({ + finishReason: 'stop', + }) + }) +}) + +describe('OllamaTextAdapter.chatStream (truncated responses)', () => { + it('reports finishReason length on RUN_FINISHED when done_reason is length', async () => { + chatMock.mockResolvedValueOnce( + asyncIterable([ + { + model: 'llama3.2', + message: { role: 'assistant', content: 'Once upon' }, + done: false, + }, + { + model: 'llama3.2', + message: { role: 'assistant', content: '' }, + done: true, + done_reason: 'length', + }, + ]), + ) + + const adapter = createOllamaChat('llama3.2') + const chunks = await collectStream( + adapter.chatStream({ + threadId: 'thread-1', + logger: testLogger, + model: 'llama3.2', + messages: [{ role: 'user', content: 'hi' }], + }), + ) + + const types = chunks.map((c) => c.type) + expect(types).toContain('TEXT_MESSAGE_END') + expect(types).not.toContain('RUN_ERROR') + expect(chunks.find((c) => c.type === 'RUN_FINISHED')).toMatchObject({ + finishReason: 'length', + }) + }) + + it('reports finishReason length when a tool call is cut off', async () => { + chatMock.mockResolvedValueOnce( + asyncIterable([ + { + message: { + role: 'assistant', + content: '', + tool_calls: [ + { + id: 'tc-123', + function: { name: 'search', arguments: { q: 'cats' } }, + }, + ], + }, + done: false, + }, + { + message: { role: 'assistant', content: '' }, + done: true, + done_reason: 'length', + }, + ]), + ) + + const chunks = await collectStream( + createOllamaChat('llama3.2').chatStream({ + threadId: 'thread-1', + logger: testLogger, + model: 'llama3.2', + messages: [{ role: 'user', content: 'find cats' }], + tools: [searchTool], + }), + ) + + expect(chunks.map((c) => c.type)).toContain('TOOL_CALL_END') + expect(chunks.find((c) => c.type === 'RUN_FINISHED')).toMatchObject({ + finishReason: 'length', + }) }) }) @@ -185,6 +289,9 @@ describe('OllamaTextAdapter.chatStream (tool calls)', () => { | undefined expect(startChunk!.toolCallName).toBe('search') expect(startChunk!.toolCallId).toBe('tc-123') + expect(chunks.find((c) => c.type === 'RUN_FINISHED')).toMatchObject({ + finishReason: 'tool_calls', + }) }) it('emits parentMessageId on tool-first tool calls matching the assistant message id', async () => { diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 3988d3e0bc..bb13fc9dfb 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -154,6 +154,7 @@ import { Route as ApiMoonshotUsageDetailsRouteImport } from './routes/api.moonsh import { Route as ApiMultimodalToolResultWireRouteImport } from './routes/api.multimodal-tool-result-wire' import { Route as ApiNeonWireRouteImport } from './routes/api.neon-wire' import { Route as ApiNonStreamingRunErrorRouteImport } from './routes/api.non-streaming-run-error' +import { Route as ApiOllamaLengthTruncationRouteImport } from './routes/api.ollama-length-truncation' import { Route as ApiOpenaiApplyPatchWireRouteImport } from './routes/api.openai-apply-patch-wire' import { Route as ApiOpenaiCompletedResponseTextRouteImport } from './routes/api.openai-completed-response-text' import { Route as ApiOpenaiComputerUseWireRouteImport } from './routes/api.openai-computer-use-wire' @@ -972,6 +973,12 @@ const ApiNonStreamingRunErrorRoute = ApiNonStreamingRunErrorRouteImport.update({ path: '/api/non-streaming-run-error', getParentRoute: () => rootRouteImport, } as any) +const ApiOllamaLengthTruncationRoute = + ApiOllamaLengthTruncationRouteImport.update({ + id: '/api/ollama-length-truncation', + path: '/api/ollama-length-truncation', + getParentRoute: () => rootRouteImport, + } as any) const ApiOpenaiApplyPatchWireRoute = ApiOpenaiApplyPatchWireRouteImport.update({ id: '/api/openai-apply-patch-wire', path: '/api/openai-apply-patch-wire', @@ -1449,6 +1456,7 @@ export interface FileRoutesByFullPath { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/neon-wire': typeof ApiNeonWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute + '/api/ollama-length-truncation': typeof ApiOllamaLengthTruncationRoute '/api/openai-apply-patch-wire': typeof ApiOpenaiApplyPatchWireRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute @@ -1658,6 +1666,7 @@ export interface FileRoutesByTo { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/neon-wire': typeof ApiNeonWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute + '/api/ollama-length-truncation': typeof ApiOllamaLengthTruncationRoute '/api/openai-apply-patch-wire': typeof ApiOpenaiApplyPatchWireRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute @@ -1868,6 +1877,7 @@ export interface FileRoutesById { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/neon-wire': typeof ApiNeonWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute + '/api/ollama-length-truncation': typeof ApiOllamaLengthTruncationRoute '/api/openai-apply-patch-wire': typeof ApiOpenaiApplyPatchWireRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute @@ -2079,6 +2089,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/neon-wire' | '/api/non-streaming-run-error' + | '/api/ollama-length-truncation' | '/api/openai-apply-patch-wire' | '/api/openai-completed-response-text' | '/api/openai-computer-use-wire' @@ -2288,6 +2299,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/neon-wire' | '/api/non-streaming-run-error' + | '/api/ollama-length-truncation' | '/api/openai-apply-patch-wire' | '/api/openai-completed-response-text' | '/api/openai-computer-use-wire' @@ -2497,6 +2509,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/neon-wire' | '/api/non-streaming-run-error' + | '/api/ollama-length-truncation' | '/api/openai-apply-patch-wire' | '/api/openai-completed-response-text' | '/api/openai-computer-use-wire' @@ -2707,6 +2720,7 @@ export interface RootRouteChildren { ApiMultimodalToolResultWireRoute: typeof ApiMultimodalToolResultWireRoute ApiNeonWireRoute: typeof ApiNeonWireRoute ApiNonStreamingRunErrorRoute: typeof ApiNonStreamingRunErrorRoute + ApiOllamaLengthTruncationRoute: typeof ApiOllamaLengthTruncationRoute ApiOpenaiApplyPatchWireRoute: typeof ApiOpenaiApplyPatchWireRoute ApiOpenaiCompletedResponseTextRoute: typeof ApiOpenaiCompletedResponseTextRoute ApiOpenaiComputerUseWireRoute: typeof ApiOpenaiComputerUseWireRoute @@ -3784,6 +3798,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiNonStreamingRunErrorRouteImport parentRoute: typeof rootRouteImport } + '/api/ollama-length-truncation': { + id: '/api/ollama-length-truncation' + path: '/api/ollama-length-truncation' + fullPath: '/api/ollama-length-truncation' + preLoaderRoute: typeof ApiOllamaLengthTruncationRouteImport + parentRoute: typeof rootRouteImport + } '/api/openai-apply-patch-wire': { id: '/api/openai-apply-patch-wire' path: '/api/openai-apply-patch-wire' @@ -4429,6 +4450,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMultimodalToolResultWireRoute: ApiMultimodalToolResultWireRoute, ApiNeonWireRoute: ApiNeonWireRoute, ApiNonStreamingRunErrorRoute: ApiNonStreamingRunErrorRoute, + ApiOllamaLengthTruncationRoute: ApiOllamaLengthTruncationRoute, ApiOpenaiApplyPatchWireRoute: ApiOpenaiApplyPatchWireRoute, ApiOpenaiCompletedResponseTextRoute: ApiOpenaiCompletedResponseTextRoute, ApiOpenaiComputerUseWireRoute: ApiOpenaiComputerUseWireRoute, diff --git a/testing/e2e/src/routes/api.ollama-length-truncation.ts b/testing/e2e/src/routes/api.ollama-length-truncation.ts new file mode 100644 index 0000000000..b546b1052a --- /dev/null +++ b/testing/e2e/src/routes/api.ollama-length-truncation.ts @@ -0,0 +1,76 @@ +import { createServer } from 'node:http' +import { createFileRoute } from '@tanstack/react-router' +import { chat } from '@tanstack/ai' +import { createOllamaChat } from '@tanstack/ai-ollama' + +/** + * Streams one Ollama chat that stops at the `num_predict` limit + * (`done_reason: 'length'`). aimock always answers Ollama with + * `done_reason: 'stop'`, so this route serves the NDJSON stream itself from a + * throwaway local server. Returns the `finishReason` of the + * `RUN_FINISHED` chunk. + */ +export const Route = createFileRoute('/api/ollama-length-truncation')({ + server: { + handlers: { + POST: async () => { + const stub = createServer((req, res) => { + req.resume() + res.writeHead(200, { 'content-type': 'application/x-ndjson' }) + const base = { + model: 'mistral', + created_at: new Date().toISOString(), + } + res.write( + JSON.stringify({ + ...base, + message: { role: 'assistant', content: 'Hello there, my' }, + done: false, + }) + '\n', + ) + res.end( + JSON.stringify({ + ...base, + message: { role: 'assistant', content: '' }, + done: true, + done_reason: 'length', + prompt_eval_count: 13, + eval_count: 3, + }) + '\n', + ) + }) + await new Promise((resolve) => + stub.listen(0, '127.0.0.1', () => resolve()), + ) + const { port } = stub.address() as { port: number } + + let runError: Record | undefined + let finishReason: string | null | undefined + try { + const adapter = createOllamaChat('mistral', { + baseURL: `http://127.0.0.1:${port}`, + }) + for await (const chunk of chat({ + threadId: 'ollama-length-truncation', + adapter, + messages: [{ role: 'user', content: 'say hello in five words' }], + })) { + if (chunk.type === 'RUN_ERROR') runError = { code: chunk.code } + if (chunk.type === 'RUN_FINISHED') { + finishReason = chunk.metadata?.tanstack?.finishReason + } + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } finally { + stub.close() + } + + return Response.json({ ok: true, runError, finishReason }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/ollama-length-truncation.spec.ts b/testing/e2e/tests/ollama-length-truncation.spec.ts new file mode 100644 index 0000000000..07207fa729 --- /dev/null +++ b/testing/e2e/tests/ollama-length-truncation.spec.ts @@ -0,0 +1,26 @@ +import { test, expect } from './fixtures' + +/** + * An Ollama stream that stops at the `num_predict` limit + * (`done_reason: 'length'`) must end in `RUN_FINISHED` with + * `finishReason: 'length'`, not `'stop'`, so callers can tell the answer is + * incomplete. + */ +test.describe('ollama — num_predict truncation', () => { + test('RUN_FINISHED carries finishReason length', async ({ request }) => { + const res = await request.post('/api/ollama-length-truncation') + expect(res.ok()).toBe(true) + + const { ok, runError, finishReason, error } = (await res.json()) as { + ok: boolean + error?: string + runError?: { code?: string } + finishReason?: string + } + + expect(error ?? null).toBeNull() + expect(ok).toBe(true) + expect(runError).toBeUndefined() + expect(finishReason).toBe('length') + }) +})