From f02ed613a585685ee91ac2566a1f9b2b970cd725 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:44:16 +1100 Subject: [PATCH] feat(ai-anthropic): support native on-demand compaction withCompaction passes tools, model options, and reasoning into compact. The Anthropic adapter posts a summarize request and replays the signed block. --- .changeset/anthropic-native-compaction.md | 7 + docs/adapters/anthropic-advanced.md | 32 ++ docs/adapters/anthropic.md | 4 +- docs/advanced/compaction-advanced.md | 2 +- docs/advanced/compaction.md | 13 +- docs/config.json | 8 +- .../interfaces/TextCompactOptions.md | 52 ++- packages/ai-anthropic/src/adapters/text.ts | 354 +++++++++++++++--- .../ai-anthropic/tests/compaction.test.ts | 266 +++++++++++++ packages/ai-compaction/src/index.test.ts | 39 ++ packages/ai-compaction/src/index.ts | 33 +- packages/ai/src/types.ts | 11 + testing/e2e/src/routes/api.compaction-wire.ts | 136 ++++++- testing/e2e/tests/compaction-wire.spec.ts | 79 ++++ 14 files changed, 969 insertions(+), 67 deletions(-) create mode 100644 .changeset/anthropic-native-compaction.md create mode 100644 packages/ai-anthropic/tests/compaction.test.ts diff --git a/.changeset/anthropic-native-compaction.md b/.changeset/anthropic-native-compaction.md new file mode 100644 index 0000000000..b22cae5f3f --- /dev/null +++ b/.changeset/anthropic-native-compaction.md @@ -0,0 +1,7 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-compaction': minor +'@tanstack/ai-anthropic': minor +--- + +`withCompaction({ native })` now passes the tools, the model options, and the reasoning of the chat call into `compact`. The Anthropic text adapter implements `compact` as an on-demand Messages request (`compaction: { type: "summarize" }`, beta `compact-2026-09-04`). The signed compaction block stays on the assistant message and goes back on later requests to the same model. A result with no signed block uses the configured strategy. diff --git a/docs/adapters/anthropic-advanced.md b/docs/adapters/anthropic-advanced.md index aea48f7a43..2d5993909b 100644 --- a/docs/adapters/anthropic-advanced.md +++ b/docs/adapters/anthropic-advanced.md @@ -256,6 +256,38 @@ export const throughProxy = anthropicText("claude-opus-5-5", { If that endpoint sends the request and the `anthropic-beta` header on to Anthropic without changes, set `true`. See [Mid-Conversation Changes](../advanced/mid-conversation-changes). +## Native compaction + +Anthropic can compact a long conversation on its side. `createAnthropicChat` and `anthropicText` have a `compact` method for that call. Pass the adapter as `native`: + +```typescript +import { chat } from "@tanstack/ai"; +import { createAnthropicChat } from "@tanstack/ai-anthropic"; +import { withCompaction } from "@tanstack/ai-compaction"; + +const adapter = createAnthropicChat( + "claude-opus-5-5", + process.env.ANTHROPIC_API_KEY!, +); + +const stream = chat({ + adapter, + messages: [{ role: "user", content: "Hello!" }], + threadId: "thread-1", + middleware: [withCompaction({ maxTokens: 100_000, native: adapter })], +}); +``` + +The call is a Messages request. It sets `compaction` to `{ type: "summarize" }` and sends the beta `compact-2026-09-04`. + +- The request includes the system prompts, the tools, and `max_tokens` from the chat. The call does not run a tool. +- The result is one assistant message. The message text is the summary. +- `metadata.anthropicCompaction` holds the signed block. The block has `content` and `signature`. +- The next request to the same model includes that block and the same beta. +- A result with no signed block uses `strategy`. After a 404, a 405, or a 501, that adapter and model use `strategy` until the process restarts. + +See [Let the provider compact](../advanced/compaction#let-the-provider-compact). + ## Provider tools Anthropic has tools of its own, next to the function tools that you define. Import them from `@tanstack/ai-anthropic/tools` and pass them into `chat({ tools: [...] })`. diff --git a/docs/adapters/anthropic.md b/docs/adapters/anthropic.md index 7538e2a2d3..9ff3661699 100644 --- a/docs/adapters/anthropic.md +++ b/docs/adapters/anthropic.md @@ -356,8 +356,8 @@ Now a declined request gets an answer from a fallback model. When no model can a These topics are on [Anthropic: Advanced](./anthropic-advanced): - Bearer and OAuth tokens, gateways, and your own Anthropic client. -- Claude on Vertex AI. -- The `max_tokens` default and prompt caching. +- Claude on Vertex AI, the `max_tokens` default, and prompt caching. +- [Native compaction](./anthropic-advanced#native-compaction). - Every provider tool, with a full sample. - The API reference. diff --git a/docs/advanced/compaction-advanced.md b/docs/advanced/compaction-advanced.md index 98dec1f7e2..d6a0df8b07 100644 --- a/docs/advanced/compaction-advanced.md +++ b/docs/advanced/compaction-advanced.md @@ -91,7 +91,7 @@ These rules apply when you pass `native` (see [Let the provider compact](./compa - When `compact` fails, `strategy` runs in its place. `onCompact` and `compaction:ended` get the failure in `error`. Some providers on the OpenAI Responses base have no compaction endpoint. - After a 404, 405, or 501 (the provider has no compaction endpoint), that adapter object and model use `strategy` until the process restarts. Create the adapter once, outside the request handler, so that no more compaction calls go out. - After any other failure, such as a 400, a 429, or a network error, the next compaction tries `compact` again. -- The compaction call gets the system prompts of the `chat()` call. It does not get the tools, because the compaction endpoint does not accept them. +- The compaction call gets the system prompts, the tools, the model options, and the reasoning of the `chat()` call. - The compacted messages can hold an encrypted provider item. The adapter sends it back unchanged to the same model. Another model does not get it. ## Give the retry something to cut diff --git a/docs/advanced/compaction.md b/docs/advanced/compaction.md index 06965c522a..040575cdc8 100644 --- a/docs/advanced/compaction.md +++ b/docs/advanced/compaction.md @@ -238,7 +238,18 @@ export async function POST(request: Request) { - An adapter with no `compact` method uses `strategy`. - When `compact` fails, `strategy` runs in its place. -The OpenAI Responses adapters have `compact`. See [Native compaction](../adapters/openai-advanced#native-compaction). For what happens on each kind of failure, see [Native compaction failures](./compaction-advanced#native-compaction-failures). +The Anthropic adapters have `compact` too. Create the adapter once: + +```typescript +import { createAnthropicChat } from "@tanstack/ai-anthropic"; + +const adapter = createAnthropicChat( + "claude-opus-5-5", + process.env.ANTHROPIC_API_KEY!, +); +``` + +Pass that adapter to `chat()` and to `native`, as in the sample above. See [OpenAI native compaction](../adapters/openai-advanced#native-compaction) and [Anthropic native compaction](../adapters/anthropic-advanced#native-compaction). For each kind of failure, see [Native compaction failures](./compaction-advanced#native-compaction-failures). ## When the chat is still too long diff --git a/docs/config.json b/docs/config.json index f75826e68f..48e1b95415 100644 --- a/docs/config.json +++ b/docs/config.json @@ -450,13 +450,13 @@ "label": "Compaction", "to": "advanced/compaction", "addedAt": "2026-08-24", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-09" }, { "label": "Compaction: Advanced", "to": "advanced/compaction-advanced", "addedAt": "2026-10-21", - "updatedAt": "2026-11-07" + "updatedAt": "2026-11-09" } ], "tab": "guides" @@ -1724,13 +1724,13 @@ "label": "Anthropic", "to": "adapters/anthropic", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-09" }, { "label": "Anthropic: Advanced", "to": "adapters/anthropic-advanced", "addedAt": "2026-10-21", - "updatedAt": "2026-11-02" + "updatedAt": "2026-11-09" }, { "label": "Google Gemini", diff --git a/docs/reference/interfaces/TextCompactOptions.md b/docs/reference/interfaces/TextCompactOptions.md index f9ba2d9f94..572aa89d33 100644 --- a/docs/reference/interfaces/TextCompactOptions.md +++ b/docs/reference/interfaces/TextCompactOptions.md @@ -3,7 +3,7 @@ id: TextCompactOptions title: TextCompactOptions --- -Defined in: [packages/ai/src/types.ts:1570](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1570) +Defined in: [packages/ai/src/types.ts:1584](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1584) What `TextAdapter.compact` receives. @@ -62,6 +62,54 @@ The system prompts of the chat call. optional wrapFetch?: FetchWrapper; ``` -Defined in: [packages/ai/src/types.ts:1576](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1576) +Defined in: [packages/ai/src/types.ts:1591](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1591) Wraps the fetch of the compaction request. + +*** + +### tools? + +```ts +optional tools?: AnyTool[]; +``` + +Defined in: [packages/ai/src/types.ts:1598](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1598) + +The tools of the chat call. The provider reads the definitions. The compaction call does not run a tool. + +*** + +### modelOptions? + +```ts +optional modelOptions?: Record; +``` + +Defined in: [packages/ai/src/types.ts:1600](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1600) + +Provider model options of the chat call, including `max_tokens`. + +*** + +### reasoning? + +```ts +optional reasoning?: ReasoningRequest; +``` + +Defined in: [packages/ai/src/types.ts:1602](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1602) + +How hard the model thinks on the chat call. + +*** + +### addUsage? + +```ts +optional addUsage?: (usage: TokenUsage) => void; +``` + +Defined in: [packages/ai/src/types.ts:1604](https://github.com/TanStack/ai/blob/main/packages/ai/src/types.ts#L1604) + +Report the token usage of the compaction call. diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index e06a9ac7b8..58e3a725f4 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -73,6 +73,8 @@ import type { import type { BetaBase64ImageSource, BetaBase64PDFSource, + BetaCompactionBlock, + BetaCompactionBlockParam, BetaContentBlockParam, BetaFileDocumentSource, BetaFileImageSource, @@ -95,9 +97,11 @@ import type { ModelMessage, AdapterYieldChunk, NormalizedSystemPrompt, + TextCompactOptions, TextOptions, ToolChoice, ToolCall, + TokenUsage, } from '@tanstack/ai' import type { AnthropicSystemPromptMetadata, @@ -208,6 +212,110 @@ export function messagesHaveFileSource(messages: Array): boolean { ) } +/** Beta for on-demand compaction (`compaction: { type: "summarize" }`). */ +const COMPACT_BETA = 'compact-2026-09-04' satisfies AnthropicBeta + +/** Signed compaction block stored on the assistant message that replaces the history. */ +const COMPACTION_METADATA_KEY = 'anthropicCompaction' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** The signed block to send back, or `undefined` when the message has none. */ +function readStoredCompaction( + message: ModelMessage, +): BetaCompactionBlockParam | undefined { + const raw = message.metadata?.[COMPACTION_METADATA_KEY] + if (!isRecord(raw)) return undefined + if (typeof raw.content !== 'string' || raw.content.length === 0) + return undefined + if (typeof raw.signature !== 'string' || raw.signature.length === 0) { + return undefined + } + return { + type: 'compaction', + content: raw.content, + signature: raw.signature, + ...(typeof raw.encrypted_content === 'string' && { + encrypted_content: raw.encrypted_content, + }), + ...(Array.isArray(raw.tool_changes) && { + tool_changes: + raw.tool_changes as BetaCompactionBlockParam['tool_changes'], + }), + } +} + +function messagesHaveCompaction(messages: Array): boolean { + return messages.some((message) => readStoredCompaction(message) !== undefined) +} + +function withCompactBeta( + betas: Array | undefined, + needed: boolean, +): Array | undefined { + if (!needed) return betas + return [...new Set([...(betas ?? []), COMPACT_BETA])] +} + +/** + * On-demand compaction reports its tokens in `usage.iterations`, with the + * top-level counts at zero. Fold those iteration tokens into the totals. + * A response that already has top-level tokens keeps them. + */ +function usageOfCompaction( + usage: Parameters[0], +): TokenUsage | undefined { + const built = buildAnthropicUsage(usage) + if (!built) return undefined + if (built.promptTokens !== 0 || built.completionTokens !== 0) return built + let prompt = 0 + let completion = 0 + for (const item of built.providerUsageDetails?.iterations ?? []) { + if (item.type !== 'compaction') continue + prompt += + item.inputTokens + + item.cacheCreationInputTokens + + item.cacheReadInputTokens + completion += item.outputTokens + } + if (prompt === 0 && completion === 0) return built + return { + ...built, + promptTokens: prompt, + completionTokens: completion, + totalTokens: prompt + completion, + } +} + +/** + * The block to store and send back. A missing summary or signature is + * unusable: the caller falls back to its strategy. + */ +function compactionBlockFromResponse( + block: BetaCompactionBlock, +): BetaCompactionBlockParam | undefined { + if (typeof block.content !== 'string' || block.content.length === 0) { + return undefined + } + if (typeof block.signature !== 'string' || block.signature.length === 0) { + return undefined + } + return { + type: 'compaction', + content: block.content, + signature: block.signature, + ...(typeof block.encrypted_content === 'string' && { + encrypted_content: block.encrypted_content, + }), + ...(block.tool_changes != null && { + tool_changes: + block.tool_changes as BetaCompactionBlockParam['tool_changes'], + }), + } +} + /** * Maps `chat({ toolChoice })` to the Messages `tool_choice`. When the * request cannot force a tool, a forced choice falls back to `auto`. @@ -704,15 +812,18 @@ export class AnthropicTextAdapter< // The request, not modelOptions: `reasoning` can turn budget // thinking (and so the interleaved-thinking beta) on. const betas = this.requestBetas( - computeAnthropicBetas( - options.tools, - requestParams, - messagesHaveFileSource(options.messages), - // Mid-conversation tool mode puts the deferred placeholder in `tools`. - requestParams.tools?.some( - (tool) => - 'name' in tool && tool.name === DEFERRED_TOOL_PLACEHOLDER.name, + withCompactBeta( + computeAnthropicBetas( + options.tools, + requestParams, + messagesHaveFileSource(options.messages), + // Mid-conversation tool mode puts the deferred placeholder in `tools`. + requestParams.tools?.some( + (tool) => + 'name' in tool && tool.name === DEFERRED_TOOL_PLACEHOLDER.name, + ), ), + messagesHaveCompaction(options.messages), ), ) @@ -830,10 +941,13 @@ export class AnthropicTextAdapter< { provider: 'anthropic', model: this.model }, ) const betas = this.requestBetas( - computeAnthropicBetas( - chatOptions.tools, - requestParams, - messagesHaveFileSource(chatOptions.messages), + withCompactBeta( + computeAnthropicBetas( + chatOptions.tools, + requestParams, + messagesHaveFileSource(chatOptions.messages), + ), + messagesHaveCompaction(chatOptions.messages), ), ) // Make non-streaming request with tool_choice forced to our structured output tool @@ -915,6 +1029,165 @@ export class AnthropicTextAdapter< } } + /** + * Compact the history with a Messages request that sets + * `compaction: { type: "summarize" }`. The result is one assistant + * message whose metadata holds the signed compaction block. The next + * request sends that block first, in place of the summarized messages. + * A result that is not `stop_reason: "compaction"`, or that has no + * signed block, throws so the caller can use another strategy. + */ + async compact(options: TextCompactOptions): Promise> { + const tools = options.tools?.map((tool) => + resolveComputerUseTool(tool, options.model), + ) + const anthropicTools = + tools && tools.length > 0 + ? convertToolsToProviderFormat(tools) + : undefined + const messages = this.formatMessages( + this.replayMessages(options.messages, options.model), + undefined, + tools?.some(isComputerToolset) ?? false, + ) + const system = this.toSystemBlocks( + normalizeSystemPrompts( + options.systemPrompts, + ), + ) + const thinkingFields = anthropicThinking( + options.model, + options.reasoning, + ANTHROPIC_MODEL_REASONING[options.model], + ) + const budget = + thinkingFields.thinking?.type === 'enabled' + ? thinkingFields.thinking.budget_tokens + : undefined + const requested = options.modelOptions?.max_tokens + const base = + typeof requested === 'number' + ? requested + : getAnthropicDefaultMaxTokens(options.model, { stream: false }) + const maxTokens = budget !== undefined && budget >= base ? budget + 1 : base + const betas = this.requestBetas( + withCompactBeta( + computeAnthropicBetas( + tools, + thinkingFields.thinking + ? { thinking: thinkingFields.thinking } + : undefined, + messagesHaveFileSource(options.messages), + ), + true, + ), + ) + const response = await this.clientFor( + options.wrapFetch, + ).beta.messages.create( + { + model: options.model, + max_tokens: maxTokens, + messages, + stream: false, + compaction: { type: 'summarize' }, + ...(system !== undefined && { system }), + ...(anthropicTools !== undefined && { tools: anthropicTools }), + ...thinkingFields, + ...(betas && { betas }), + }, + { + ...(options.signal ? { signal: options.signal } : {}), + headers: this.requestHeaders(undefined, betas), + }, + ) + const usage = usageOfCompaction(response.usage) + if (usage) options.addUsage?.(usage) + if (response.stop_reason !== 'compaction') { + throw new Error( + `The compact result stop_reason is "${response.stop_reason ?? 'none'}", not "compaction".`, + ) + } + const block = response.content + .map((item) => + item.type === 'compaction' + ? compactionBlockFromResponse(item) + : undefined, + ) + .find((item) => item !== undefined) + if (!block) { + throw new Error('The compact result has no compaction block.') + } + return [ + { + role: 'assistant', + content: block.content ?? '', + metadata: { + tanstack: { + source: { + provider: this.provider ?? this.name, + api: this.api, + model: options.model, + }, + }, + [COMPACTION_METADATA_KEY]: { + content: block.content, + signature: block.signature, + ...(block.encrypted_content !== undefined && { + encrypted_content: block.encrypted_content, + }), + ...(block.tool_changes !== undefined && { + tool_changes: block.tool_changes, + }), + }, + }, + }, + ] + } + + /** Rewrite tool ids from another model to the shape Anthropic accepts. */ + private replayMessages(messages: Array, model: string) { + return transformMessagesForReplay( + messages, + { provider: this.provider, api: this.api, model }, + (id, { attempt }) => { + const clean = id.replace(/[^a-zA-Z0-9_-]/g, '_') + const suffix = attempt === 0 ? '' : `_${attempt}` + return clean.slice(0, 64 - suffix.length) + suffix + }, + ).messages + } + + /** + * System text blocks for a Messages request. An OAuth token also gets + * the Claude Code identity block. + */ + private toSystemBlocks( + prompts: ReadonlyArray< + NormalizedSystemPrompt + >, + ): Array | undefined { + if (prompts.length === 0 && !this.oauth) return undefined + const blocks = prompts.map( + (p): TextBlockParam => ({ + type: 'text', + text: p.content, + ...(p.metadata?.cache_control && { + cache_control: p.metadata.cache_control, + }), + }), + ) + return this.oauth + ? [ + { + type: 'text', + text: "You are Claude Code, Anthropic's official CLI for Claude.", + }, + ...blocks, + ] + : blocks + } + private mapCommonOptionsToAnthropic( options: TextOptions, { stream = true }: { stream?: boolean } = {}, @@ -924,15 +1197,9 @@ export class AnthropicTextAdapter< // Another model's history loses its signatures, and its tool IDs are // rewritten to the shape Anthropic accepts. Plan indexes match this // array when the second replay keeps the same length. - const replay = transformMessagesForReplay( - options.messages, - { provider: this.provider, api: this.api, model: options.model }, - (id, { attempt }) => { - const clean = id.replace(/[^a-zA-Z0-9_-]/g, '_') - const suffix = attempt === 0 ? '' : `_${attempt}` - return clean.slice(0, 64 - suffix.length) + suffix - }, - ) + const replay = { + messages: this.replayMessages(options.messages, options.model), + } const mid = this.midConversationRequest({ ...options, messages: replay.messages, @@ -1044,35 +1311,14 @@ export class AnthropicTextAdapter< // `string | Array` (no `| undefined`), so build it // outside the literal and spread it conditionally rather than // assigning `undefined` under exactOptionalPropertyTypes. - const systemBlocks = ((): Array | undefined => { - // With a prompt channel, `system` holds only the start prompts. They - // keep their own `cache_control` metadata. - const normalized = - mid?.systemPrompts ?? + // With a prompt channel, `system` holds only the start prompts. They + // keep their own `cache_control` metadata. + const systemBlocks = this.toSystemBlocks( + mid?.systemPrompts ?? normalizeSystemPrompts( options.systemPrompts, - ) - if (normalized.length === 0 && !this.oauth) return undefined - const blocks = normalized.map( - (p): TextBlockParam => ({ - type: 'text', - text: p.content, - ...(p.metadata?.cache_control && { - cache_control: p.metadata.cache_control, - }), - }), - ) - // Anthropic accepts an OAuth token only with the Claude Code identity. - return this.oauth - ? [ - { - type: 'text', - text: "You are Claude Code, Anthropic's official CLI for Claude.", - }, - ...blocks, - ] - : blocks - })() + ), + ) // Wire engine-threaded outputSchema into Messages `output_config.format` // alongside any `tools` so the model emits tool calls during the agent // loop and a single schema-constrained JSON message on its final turn. @@ -1391,6 +1637,18 @@ export class AnthropicTextAdapter< continue } + // A native compaction result replaces the messages it summarizes. + // Send the signed block, not the summary text beside it. + const storedCompaction = + role === 'assistant' ? readStoredCompaction(message) : undefined + if (storedCompaction) { + formattedMessages.push({ + role: 'assistant', + content: [storedCompaction], + }) + continue + } + // A valid block order map: send the blocks in the order the model // sent them. Unsigned thinking is still skipped at its place. const ordered = diff --git a/packages/ai-anthropic/tests/compaction.test.ts b/packages/ai-anthropic/tests/compaction.test.ts new file mode 100644 index 0000000000..4ecebe3cc2 --- /dev/null +++ b/packages/ai-anthropic/tests/compaction.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it } from 'vitest' +import { resolveDebugOption } from '@tanstack/ai/adapter-internals' +import { createAnthropicChat } from '../src/adapters/text' +import { getAnthropicDefaultMaxTokens } from '../src/model-meta' + +const logger = resolveDebugOption(false) +const model = 'claude-opus-5-5' + +const summary = { + id: 'msg_compact_1', + type: 'message', + role: 'assistant', + model, + content: [ + { + type: 'compaction', + content: 'The user is naming recipe entities.', + signature: 'EuYBCkQY', + encrypted_content: 'sealed', + }, + ], + stop_reason: 'compaction', + stop_sequence: null, + usage: { + input_tokens: 0, + output_tokens: 0, + iterations: [ + { + type: 'compaction', + input_tokens: 144, + output_tokens: 276, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + ], + }, +} + +const message = { + id: 'msg_1', + type: 'message', + role: 'assistant', + model, + content: [], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 2, output_tokens: 1 }, +} + +function sse(events: Array<[string, unknown]>) { + return new Response( + events + .map( + ([event, data]) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`, + ) + .join(''), + { headers: { 'content-type': 'text/event-stream' } }, + ) +} + +/** A fetch that records JSON bodies and answers a compact or a chat stream. */ +function recordingFetch() { + const bodies: Array = [] + const headers: Array = [] + const fetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init) + if (request.url.startsWith('data:')) return new Response('') + headers.push(request.headers) + const body = (await request.json()) as { compaction?: unknown } + bodies.push(body) + if (body.compaction) return Response.json(summary) + return sse([ + ['message_start', { type: 'message_start', message }], + ['message_stop', { type: 'message_stop' }], + ]) + } + return { fetch, bodies, headers } +} + +describe('Anthropic adapter compact', () => { + it('posts a summarize request and keeps the signed block', async () => { + const { fetch, bodies, headers } = recordingFetch() + const usages: Array = [] + const adapter = createAnthropicChat(model, 'test-key', { fetch }) + + const messages = await adapter.compact({ + model, + messages: [ + { role: 'user', content: 'Name the entities.' }, + { role: 'assistant', content: 'Recipe and Ingredient.' }, + ], + systemPrompts: ['Be brief.'], + tools: [ + { + name: 'echo', + description: 'Echo a message', + inputSchema: { + type: 'object', + properties: { msg: { type: 'string' } }, + required: ['msg'], + }, + }, + ], + modelOptions: { max_tokens: 4096, context_management: { edits: [] } }, + wrapFetch: (next) => (input, init) => { + const requestHeaders = new Headers(init?.headers) + requestHeaders.set('x-trace-id', 'trace-1') + return next(input, { ...init, headers: requestHeaders }) + }, + addUsage: (usage) => { + usages.push(usage) + }, + }) + + expect(bodies).toHaveLength(1) + expect(headers[0]?.get('x-trace-id')).toBe('trace-1') + expect(headers[0]?.get('anthropic-beta')).toContain('compact-2026-09-04') + expect(bodies[0]).toMatchObject({ + model, + max_tokens: 4096, + stream: false, + compaction: { type: 'summarize' }, + system: [{ type: 'text', text: 'Be brief.' }], + tools: [ + { + name: 'echo', + input_schema: { + type: 'object', + properties: { msg: { type: 'string' } }, + required: ['msg'], + }, + }, + ], + messages: [ + { role: 'user', content: 'Name the entities.' }, + { + role: 'assistant', + content: [{ type: 'text', text: 'Recipe and Ingredient.' }], + }, + ], + }) + // A compaction request cannot carry context_management. + expect(bodies[0]).not.toHaveProperty('context_management') + expect(messages).toEqual([ + { + role: 'assistant', + content: 'The user is naming recipe entities.', + metadata: { + tanstack: { + source: { + provider: 'anthropic', + api: 'anthropic-messages', + model, + }, + }, + anthropicCompaction: { + content: 'The user is naming recipe entities.', + signature: 'EuYBCkQY', + encrypted_content: 'sealed', + }, + }, + }, + ]) + expect(usages[0]).toMatchObject({ + promptTokens: 144, + completionTokens: 276, + totalTokens: 420, + }) + }) + + it('uses the non-streaming output ceiling when max_tokens is absent', async () => { + const { fetch, bodies } = recordingFetch() + const adapter = createAnthropicChat(model, 'test-key', { fetch }) + + await adapter.compact({ + model, + messages: [{ role: 'user', content: 'Name the entities.' }], + }) + + expect(bodies[0]).toMatchObject({ + max_tokens: getAnthropicDefaultMaxTokens(model, { stream: false }), + }) + }) + + it('throws when the result is not a signed compaction block', async () => { + const cases = [ + { ...summary, stop_reason: 'end_turn' }, + { + ...summary, + content: [ + { + type: 'compaction', + content: '', + signature: 'EuYBCkQY', + }, + ], + }, + { + ...summary, + content: [ + { + type: 'compaction', + content: 'Summary', + signature: null, + }, + ], + }, + ] + for (const body of cases) { + const fetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init) + if (request.url.startsWith('data:')) return new Response('') + return Response.json(body) + } + const adapter = createAnthropicChat(model, 'test-key', { fetch }) + const error = await adapter + .compact({ + model, + messages: [{ role: 'user', content: 'Name the entities.' }], + }) + .then( + () => undefined, + (thrown: unknown) => thrown, + ) + expect(error).toBeInstanceOf(Error) + expect(error).not.toHaveProperty('status') + } + }) + + it('sends the signed block back first on the next call', async () => { + const { fetch, bodies, headers } = recordingFetch() + const adapter = createAnthropicChat(model, 'test-key', { fetch }) + const compacted = await adapter.compact({ + model, + messages: [{ role: 'user', content: 'Name the entities.' }], + }) + + for await (const _ of adapter.chatStream({ + threadId: 'thread-1', + logger, + model, + messages: [...compacted, { role: 'user', content: 'Go on' }], + })) { + // Drain the stream. + } + + expect(bodies[1]).toMatchObject({ + messages: [ + { + role: 'assistant', + content: [ + { + type: 'compaction', + content: 'The user is naming recipe entities.', + signature: 'EuYBCkQY', + encrypted_content: 'sealed', + }, + ], + }, + { role: 'user', content: 'Go on' }, + ], + }) + expect(JSON.stringify(bodies[1])).not.toContain('Name the entities.') + expect(headers[1]?.get('anthropic-beta')).toContain('compact-2026-09-04') + }) +}) diff --git a/packages/ai-compaction/src/index.test.ts b/packages/ai-compaction/src/index.test.ts index 8e15b00f3d..5f0a3790b0 100644 --- a/packages/ai-compaction/src/index.test.ts +++ b/packages/ai-compaction/src/index.test.ts @@ -3374,6 +3374,45 @@ describe('withCompaction native', () => { systemPrompts: ['Be brief.'], }) }) + + it('passes the chat tools, model options, reasoning, and addUsage', async () => { + const compact = vi.fn((options: TextCompactOptions) => { + options.addUsage?.({ + promptTokens: 3, + completionTokens: 1, + totalTokens: 4, + }) + return Promise.resolve([text('user', 'kept')]) + }) + const echo = { + name: 'echo', + description: 'Echo', + inputSchema: { type: 'object' as const, properties: {} }, + } + const recorded = recordingContext('beforeModel', { model: 'gpt-5.5' }) + await withCompaction({ maxTokens: 50, native: { compact } }).onConfig?.( + recorded.ctx, + { + messages: [big('user'), big('assistant'), big('user')], + systemPrompts: ['Be brief.'], + tools: [echo], + modelOptions: { max_tokens: 4096 }, + reasoning: { level: 'low', summary: false }, + }, + ) + + expect(compact.mock.calls[0]?.[0]).toMatchObject({ + tools: [echo], + modelOptions: { max_tokens: 4096 }, + reasoning: { level: 'low', summary: false }, + }) + const ended = recorded.events.find( + (event) => event.name === COMPACTION_ENDED_EVENT, + ) + expect(ended?.value).toMatchObject({ + usage: { promptTokens: 3, completionTokens: 1, totalTokens: 4 }, + }) + }) }) describe('limits as functions of the model', () => { diff --git a/packages/ai-compaction/src/index.ts b/packages/ai-compaction/src/index.ts index 690fe8f456..bcca352e37 100644 --- a/packages/ai-compaction/src/index.ts +++ b/packages/ai-compaction/src/index.ts @@ -809,6 +809,17 @@ export function withCompaction( const configured = options.strategy ?? evictOldest() const native = options.native const nativeCompact = native?.compact?.bind(native) + /** + * The chat fields of the last `onConfig`, so a compact after the turn + * (no config argument) still sends the tools and the system prompts. + */ + const nativeInputs = new Map< + string, + Pick< + ChatMiddlewareConfig, + 'systemPrompts' | 'tools' | 'modelOptions' | 'reasoning' | 'wrapFetch' + > + >() /** * The strategy of a call: the adapter's `compact` when `native` has it. * When `compact` fails, the configured strategy runs, and `spent.error` @@ -825,14 +836,23 @@ export function withCompaction( if (failedNative.get(native)?.has(ctx.model)) { return configured(messages, compaction) } + const input = config ?? nativeInputs.get(ctx.threadId) + const systemPrompts = input?.systemPrompts ?? ctx.systemPrompts + const modelOptions = input?.modelOptions ?? ctx.modelOptions try { return await nativeCompact({ messages: [...messages], model: ctx.model, // A background summary has no signal, so the run does not stop it. ...(compaction.signal ? { signal: compaction.signal } : {}), - ...(config?.wrapFetch ? { wrapFetch: config.wrapFetch } : {}), - ...(config ? { systemPrompts: config.systemPrompts } : {}), + ...(input?.wrapFetch ? { wrapFetch: input.wrapFetch } : {}), + ...(systemPrompts ? { systemPrompts } : {}), + ...(input?.tools && input.tools.length > 0 + ? { tools: input.tools } + : {}), + ...(modelOptions ? { modelOptions } : {}), + ...(input?.reasoning ? { reasoning: input.reasoning } : {}), + addUsage: compaction.addUsage, }) } catch (error) { if (compaction.signal?.aborted) throw error @@ -1107,6 +1127,15 @@ export function withCompaction( ctx.retry() }, async onConfig(ctx, config) { + nativeInputs.set(ctx.threadId, { + systemPrompts: config.systemPrompts, + tools: config.tools, + ...(config.modelOptions !== undefined && { + modelOptions: config.modelOptions, + }), + ...(config.reasoning !== undefined && { reasoning: config.reasoning }), + ...(config.wrapFetch !== undefined && { wrapFetch: config.wrapFetch }), + }) // init is discarded by the engine rebuild and can run before persistence // hydrates the thread. Compact only on model-bound phases. if (ctx.phase === 'init') return diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index f1d4acdd50..955947e1af 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -1591,6 +1591,17 @@ export interface TextCompactOptions { wrapFetch?: FetchWrapper /** The system prompts of the chat call. */ systemPrompts?: Array + /** + * The tools of the chat call. The provider reads the definitions. + * The compaction call does not run a tool. + */ + tools?: Array + /** Provider model options of the chat call, including `max_tokens`. */ + modelOptions?: Record + /** How hard the model thinks on the chat call. */ + reasoning?: ReasoningRequest + /** Report the token usage of the compaction call. */ + addUsage?: (usage: TokenUsage) => void } // ============================================================================ diff --git a/testing/e2e/src/routes/api.compaction-wire.ts b/testing/e2e/src/routes/api.compaction-wire.ts index bfa3a65c6a..a1ef83ec9f 100644 --- a/testing/e2e/src/routes/api.compaction-wire.ts +++ b/testing/e2e/src/routes/api.compaction-wire.ts @@ -1,5 +1,6 @@ import { createFileRoute } from '@tanstack/react-router' import { chat, createChatOptions, maxIterations } from '@tanstack/ai' +import { createAnthropicChat } from '@tanstack/ai-anthropic' import { createOpenaiChat } from '@tanstack/ai-openai' import { clearToolResults, @@ -68,6 +69,85 @@ function makeTextStream(callNumber: number): ReadableStream { }) } +/** Minimal Anthropic Messages stream with one text block. */ +function makeAnthropicTextStream(): ReadableStream { + const encoder = new TextEncoder() + const events = [ + { + type: 'message_start', + message: { + id: 'msg_anthropic_compaction', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-opus-5-5', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'ok' }, + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: null }, + usage: { output_tokens: 2 }, + }, + { type: 'message_stop' }, + ] + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ) + } + controller.close() + }, + }) +} + +const anthropicCompacted = { + id: 'msg_compact_e2e', + type: 'message', + role: 'assistant', + model: 'claude-opus-5-5', + content: [ + { + type: 'compaction', + content: 'Kept the last turn.', + signature: 'SIG_E2E', + encrypted_content: 'SEALED_E2E', + }, + ], + stop_reason: 'compaction', + stop_sequence: null, + usage: { + input_tokens: 0, + output_tokens: 0, + iterations: [ + { + type: 'compaction', + input_tokens: 50, + output_tokens: 5, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + ], + }, +} + const FILLER = 'x'.repeat(160) // evict: oldest message carries SECRET_ALPHA_ONE, newest carries KEEP_ME_LAST. @@ -131,6 +211,9 @@ const compactedResponse = { * `?strategy=native` turns on the adapter's own compaction. * `?strategy=native-empty` does too, but the compact result has no compaction * item, so `evictOldest` runs in its place. + * `?strategy=anthropic-native` turns on Anthropic on-demand compaction. + * `?strategy=anthropic-native-empty` does too, but the result has no signed + * block, so `evictOldest` runs in its place. * Anything else uses `evictOldest` on a plain chat history. */ export const Route = createFileRoute('/api/compaction-wire')({ @@ -139,15 +222,37 @@ export const Route = createFileRoute('/api/compaction-wire')({ POST: async ({ request }) => { const strategyParam = new URL(request.url).searchParams.get('strategy') const clear = strategyParam === 'clear' + const anthropic = strategyParam?.startsWith('anthropic-native') ?? false const requestBodies: Array = [] const compactRequestBodies: Array = [] + const requestBetas: Array = [] + const compactBetas: Array = [] const mockFetch: typeof fetch = async (input, init) => { const req = input instanceof Request ? input : new Request(input, init) - if (req.url.endsWith('/responses/compact')) { - compactRequestBodies.push(JSON.parse(await req.text())) + if (req.url.startsWith('data:')) { + return new Response(null, { status: 204 }) + } + const body = JSON.parse(await req.text()) as unknown + const isCompact = + req.url.endsWith('/responses/compact') || + (typeof body === 'object' && body !== null && 'compaction' in body) + if (isCompact) { + compactRequestBodies.push(body) + compactBetas.push(req.headers.get('anthropic-beta')) + if (anthropic) { + return Response.json( + strategyParam === 'anthropic-native-empty' + ? { + ...anthropicCompacted, + content: [{ type: 'text', text: 'nope' }], + stop_reason: 'compaction', + } + : anthropicCompacted, + ) + } return Response.json( strategyParam === 'native-empty' ? { @@ -159,7 +264,13 @@ export const Route = createFileRoute('/api/compaction-wire')({ : compactedResponse, ) } - requestBodies.push(JSON.parse(await req.text())) + requestBodies.push(body) + requestBetas.push(req.headers.get('anthropic-beta')) + if (anthropic) { + return new Response(makeAnthropicTextStream(), { + headers: { 'Content-Type': 'text/event-stream' }, + }) + } return new Response(makeTextStream(requestBodies.length), { headers: { 'Content-Type': 'text/event-stream' }, }) @@ -175,9 +286,17 @@ export const Route = createFileRoute('/api/compaction-wire')({ }) : evictOldest({ keepRecentTokens: 45 }) - const adapter = createOpenaiChat('gpt-5.2', DUMMY_KEY, { - fetch: mockFetch, - }) + const adapter = anthropic + ? createAnthropicChat( + 'claude-opus-5-5', + 'sk-ant-e2e-test-dummy-key', + { + fetch: mockFetch, + }, + ) + : createOpenaiChat('gpt-5.2', DUMMY_KEY, { + fetch: mockFetch, + }) const persistence = memoryPersistence() let compactionCount = 0 const compactionErrors: Array = [] @@ -185,7 +304,8 @@ export const Route = createFileRoute('/api/compaction-wire')({ compactionCount++ if (info.error) compactionErrors.push(info.error.message) } - const native = strategyParam?.startsWith('native') ? adapter : undefined + const native = + strategyParam?.startsWith('native') || anthropic ? adapter : undefined try { for await (const _ of chat({ @@ -240,6 +360,8 @@ export const Route = createFileRoute('/api/compaction-wire')({ firstRequestBody: requestBodies[0], secondRequestBody: requestBodies[1], compactRequestBodies, + compactBetas, + requestBetas, canonicalMessages, compactionCount, compactionErrors, diff --git a/testing/e2e/tests/compaction-wire.spec.ts b/testing/e2e/tests/compaction-wire.spec.ts index f55a56d84e..1d67c76c8e 100644 --- a/testing/e2e/tests/compaction-wire.spec.ts +++ b/testing/e2e/tests/compaction-wire.spec.ts @@ -147,4 +147,83 @@ test.describe('withCompaction — wire format', () => { expect(wire).not.toContain('SECRET_ALPHA_ONE') expect(wire).not.toContain('SEALED_E2E') }) + + test('anthropic native compaction posts a summarize request and replays the block', async ({ + request, + }) => { + const response = await request.post( + '/api/compaction-wire?strategy=anthropic-native', + ) + expect(response.ok()).toBe(true) + const result = (await response.json()) as { + ok: boolean + error?: string + firstRequestBody: unknown + secondRequestBody: unknown + compactRequestBodies: Array + compactBetas: Array + requestBetas: Array + compactionCount: number + } + if (!result.ok) throw new Error(`Route failed: ${result.error}`) + + expect(result.compactRequestBodies).toHaveLength(1) + expect(result.compactRequestBodies[0]).toMatchObject({ + compaction: { type: 'summarize' }, + stream: false, + }) + expect(JSON.stringify(result.compactRequestBodies[0])).toContain( + 'SECRET_ALPHA_ONE', + ) + expect(result.compactBetas[0]).toContain('compact-2026-09-04') + expect(result.firstRequestBody).toMatchObject({ + messages: [ + { + role: 'assistant', + content: [ + { + type: 'compaction', + content: 'Kept the last turn.', + signature: 'SIG_E2E', + encrypted_content: 'SEALED_E2E', + }, + ], + }, + ], + }) + expect(JSON.stringify(result.firstRequestBody)).not.toContain( + 'SECRET_ALPHA_ONE', + ) + expect(result.requestBetas[0]).toContain('compact-2026-09-04') + expect(JSON.stringify(result.secondRequestBody)).toContain('SEALED_E2E') + expect(result.compactionCount).toBe(1) + }) + + test('anthropic native compaction with no compaction block falls back', async ({ + request, + }) => { + const response = await request.post( + '/api/compaction-wire?strategy=anthropic-native-empty', + ) + expect(response.ok()).toBe(true) + const result = (await response.json()) as { + ok: boolean + error?: string + firstRequestBody: unknown + compactRequestBodies: Array + compactionErrors: Array + } + if (!result.ok) throw new Error(`Route failed: ${result.error}`) + + expect(result.compactRequestBodies).toHaveLength(2) + expect(result.compactionErrors).toEqual([ + 'The compact result has no compaction block.', + 'The compact result has no compaction block.', + ]) + const wire = JSON.stringify(result.firstRequestBody) + expect(wire).toContain('omitted to save context') + expect(wire).toContain('KEEP_ME_LAST') + expect(wire).not.toContain('SECRET_ALPHA_ONE') + expect(wire).not.toContain('SEALED_E2E') + }) })